5efd0947026d6cc51aa9d59bb1cb2ef3b5ef57d4
[controller.git] / opendaylight / md-sal / sal-distributed-datastore / src / test / java / org / opendaylight / controller / cluster / datastore / shardmanager / ShardManagerTest.java
1 /*
2  * Copyright (c) 2014, 2015 Cisco Systems, Inc. and others.  All rights reserved.
3  *
4  * This program and the accompanying materials are made available under the
5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6  * and is available at http://www.eclipse.org/legal/epl-v10.html
7  */
8
9 package org.opendaylight.controller.cluster.datastore.shardmanager;
10
11 import static org.junit.Assert.assertEquals;
12 import static org.junit.Assert.assertNotNull;
13 import static org.junit.Assert.assertNull;
14 import static org.junit.Assert.assertSame;
15 import static org.junit.Assert.assertTrue;
16 import static org.junit.Assert.fail;
17 import static org.mockito.Mockito.mock;
18 import static org.mockito.Mockito.never;
19 import static org.mockito.Mockito.times;
20 import static org.mockito.Mockito.verify;
21
22 import akka.actor.ActorRef;
23 import akka.actor.ActorSystem;
24 import akka.actor.AddressFromURIString;
25 import akka.actor.Props;
26 import akka.actor.Status;
27 import akka.actor.Status.Failure;
28 import akka.actor.Status.Success;
29 import akka.cluster.Cluster;
30 import akka.cluster.ClusterEvent;
31 import akka.cluster.Member;
32 import akka.dispatch.Dispatchers;
33 import akka.japi.Creator;
34 import akka.pattern.Patterns;
35 import akka.persistence.RecoveryCompleted;
36 import akka.serialization.Serialization;
37 import akka.testkit.JavaTestKit;
38 import akka.testkit.TestActorRef;
39 import akka.util.Timeout;
40 import com.google.common.base.Function;
41 import com.google.common.base.Stopwatch;
42 import com.google.common.collect.ImmutableMap;
43 import com.google.common.collect.ImmutableSet;
44 import com.google.common.collect.Lists;
45 import com.google.common.collect.Sets;
46 import com.google.common.util.concurrent.Uninterruptibles;
47 import java.net.URI;
48 import java.util.AbstractMap;
49 import java.util.Arrays;
50 import java.util.Collection;
51 import java.util.Collections;
52 import java.util.HashMap;
53 import java.util.List;
54 import java.util.Map;
55 import java.util.Map.Entry;
56 import java.util.Set;
57 import java.util.concurrent.CountDownLatch;
58 import java.util.concurrent.TimeUnit;
59 import java.util.concurrent.TimeoutException;
60 import java.util.stream.Collectors;
61 import org.junit.Test;
62 import org.mockito.Mockito;
63 import org.opendaylight.controller.cluster.access.concepts.MemberName;
64 import org.opendaylight.controller.cluster.datastore.AbstractShardManagerTest;
65 import org.opendaylight.controller.cluster.datastore.ClusterWrapperImpl;
66 import org.opendaylight.controller.cluster.datastore.DataStoreVersions;
67 import org.opendaylight.controller.cluster.datastore.DatastoreContext;
68 import org.opendaylight.controller.cluster.datastore.DatastoreContextFactory;
69 import org.opendaylight.controller.cluster.datastore.DistributedDataStore;
70 import org.opendaylight.controller.cluster.datastore.Shard;
71 import org.opendaylight.controller.cluster.datastore.ShardManager.SchemaContextModules;
72 import org.opendaylight.controller.cluster.datastore.config.ConfigurationImpl;
73 import org.opendaylight.controller.cluster.datastore.config.EmptyModuleShardConfigProvider;
74 import org.opendaylight.controller.cluster.datastore.config.ModuleShardConfiguration;
75 import org.opendaylight.controller.cluster.datastore.exceptions.AlreadyExistsException;
76 import org.opendaylight.controller.cluster.datastore.exceptions.NoShardLeaderException;
77 import org.opendaylight.controller.cluster.datastore.exceptions.NotInitializedException;
78 import org.opendaylight.controller.cluster.datastore.exceptions.PrimaryNotFoundException;
79 import org.opendaylight.controller.cluster.datastore.identifiers.ShardIdentifier;
80 import org.opendaylight.controller.cluster.datastore.identifiers.ShardManagerIdentifier;
81 import org.opendaylight.controller.cluster.datastore.messages.ActorInitialized;
82 import org.opendaylight.controller.cluster.datastore.messages.AddShardReplica;
83 import org.opendaylight.controller.cluster.datastore.messages.ChangeShardMembersVotingStatus;
84 import org.opendaylight.controller.cluster.datastore.messages.CreateShard;
85 import org.opendaylight.controller.cluster.datastore.messages.FindLocalShard;
86 import org.opendaylight.controller.cluster.datastore.messages.FindPrimary;
87 import org.opendaylight.controller.cluster.datastore.messages.LocalPrimaryShardFound;
88 import org.opendaylight.controller.cluster.datastore.messages.LocalShardFound;
89 import org.opendaylight.controller.cluster.datastore.messages.LocalShardNotFound;
90 import org.opendaylight.controller.cluster.datastore.messages.PeerDown;
91 import org.opendaylight.controller.cluster.datastore.messages.PeerUp;
92 import org.opendaylight.controller.cluster.datastore.messages.PrimaryShardInfo;
93 import org.opendaylight.controller.cluster.datastore.messages.RemotePrimaryShardFound;
94 import org.opendaylight.controller.cluster.datastore.messages.RemoveShardReplica;
95 import org.opendaylight.controller.cluster.datastore.messages.ShardLeaderStateChanged;
96 import org.opendaylight.controller.cluster.datastore.messages.UpdateSchemaContext;
97 import org.opendaylight.controller.cluster.datastore.persisted.DatastoreSnapshot;
98 import org.opendaylight.controller.cluster.datastore.persisted.DatastoreSnapshot.ShardSnapshot;
99 import org.opendaylight.controller.cluster.datastore.persisted.ShardManagerSnapshot;
100 import org.opendaylight.controller.cluster.datastore.utils.ForwardingActor;
101 import org.opendaylight.controller.cluster.datastore.utils.MockClusterWrapper;
102 import org.opendaylight.controller.cluster.datastore.utils.MockConfiguration;
103 import org.opendaylight.controller.cluster.datastore.utils.PrimaryShardInfoFutureCache;
104 import org.opendaylight.controller.cluster.notifications.LeaderStateChanged;
105 import org.opendaylight.controller.cluster.notifications.RegisterRoleChangeListener;
106 import org.opendaylight.controller.cluster.notifications.RoleChangeNotification;
107 import org.opendaylight.controller.cluster.raft.RaftState;
108 import org.opendaylight.controller.cluster.raft.base.messages.FollowerInitialSyncUpStatus;
109 import org.opendaylight.controller.cluster.raft.base.messages.SwitchBehavior;
110 import org.opendaylight.controller.cluster.raft.client.messages.GetSnapshot;
111 import org.opendaylight.controller.cluster.raft.client.messages.Shutdown;
112 import org.opendaylight.controller.cluster.raft.messages.AddServer;
113 import org.opendaylight.controller.cluster.raft.messages.AddServerReply;
114 import org.opendaylight.controller.cluster.raft.messages.ChangeServersVotingStatus;
115 import org.opendaylight.controller.cluster.raft.messages.RemoveServer;
116 import org.opendaylight.controller.cluster.raft.messages.RemoveServerReply;
117 import org.opendaylight.controller.cluster.raft.messages.ServerChangeReply;
118 import org.opendaylight.controller.cluster.raft.messages.ServerChangeStatus;
119 import org.opendaylight.controller.cluster.raft.messages.ServerRemoved;
120 import org.opendaylight.controller.cluster.raft.policy.DisableElectionsRaftPolicy;
121 import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal;
122 import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore;
123 import org.opendaylight.controller.cluster.raft.utils.MessageCollectorActor;
124 import org.opendaylight.controller.md.cluster.datastore.model.TestModel;
125 import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTree;
126 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
127 import org.slf4j.Logger;
128 import org.slf4j.LoggerFactory;
129 import scala.concurrent.Await;
130 import scala.concurrent.Future;
131 import scala.concurrent.duration.FiniteDuration;
132
133 public class ShardManagerTest extends AbstractShardManagerTest {
134     private static final Logger LOG = LoggerFactory.getLogger(ShardManagerTest.class);
135     private static final MemberName MEMBER_2 = MemberName.forName("member-2");
136     private static final MemberName MEMBER_3 = MemberName.forName("member-3");
137
138     private final String shardMgrID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
139
140     private ActorSystem newActorSystem(String config) {
141         return newActorSystem("cluster-test", config);
142     }
143
144     private ActorRef newMockShardActor(ActorSystem system, String shardName, String memberName) {
145         String name = ShardIdentifier.create(shardName, MemberName.forName(memberName), "config").toString();
146         if (system == getSystem()) {
147             return actorFactory.createActor(Props.create(MessageCollectorActor.class), name);
148         }
149
150         return system.actorOf(Props.create(MessageCollectorActor.class), name);
151     }
152
153     private Props newShardMgrProps() {
154         return newShardMgrProps(new MockConfiguration());
155     }
156
157     private static DatastoreContextFactory newDatastoreContextFactory(DatastoreContext datastoreContext) {
158         DatastoreContextFactory mockFactory = mock(DatastoreContextFactory.class);
159         Mockito.doReturn(datastoreContext).when(mockFactory).getBaseDatastoreContext();
160         Mockito.doReturn(datastoreContext).when(mockFactory).getShardDatastoreContext(Mockito.anyString());
161         return mockFactory;
162     }
163
164     private TestShardManager.Builder newTestShardMgrBuilderWithMockShardActor() {
165         return newTestShardMgrBuilderWithMockShardActor(mockShardActor);
166     }
167
168     private TestShardManager.Builder newTestShardMgrBuilderWithMockShardActor(ActorRef shardActor) {
169         return TestShardManager.builder(datastoreContextBuilder).shardActor(shardActor)
170                 .distributedDataStore(mock(DistributedDataStore.class));
171     }
172
173
174     private Props newPropsShardMgrWithMockShardActor() {
175         return newTestShardMgrBuilderWithMockShardActor().props().withDispatcher(
176                 Dispatchers.DefaultDispatcherId());
177     }
178
179     private Props newPropsShardMgrWithMockShardActor(ActorRef shardActor) {
180         return newTestShardMgrBuilderWithMockShardActor(shardActor).props()
181                 .withDispatcher(Dispatchers.DefaultDispatcherId());
182     }
183
184
185     private TestShardManager newTestShardManager() {
186         return newTestShardManager(newShardMgrProps());
187     }
188
189     private TestShardManager newTestShardManager(Props props) {
190         TestActorRef<TestShardManager> shardManagerActor = actorFactory.createTestActor(props);
191         TestShardManager shardManager = shardManagerActor.underlyingActor();
192         shardManager.waitForRecoveryComplete();
193         return shardManager;
194     }
195
196     private static void waitForShardInitialized(ActorRef shardManager, String shardName, JavaTestKit kit) {
197         AssertionError last = null;
198         Stopwatch sw = Stopwatch.createStarted();
199         while (sw.elapsed(TimeUnit.SECONDS) <= 5) {
200             try {
201                 shardManager.tell(new FindLocalShard(shardName, true), kit.getRef());
202                 kit.expectMsgClass(LocalShardFound.class);
203                 return;
204             } catch (AssertionError e) {
205                 last = e;
206             }
207
208             Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
209         }
210
211         throw last;
212     }
213
214     @SuppressWarnings("unchecked")
215     private static <T> T expectMsgClassOrFailure(Class<T> msgClass, JavaTestKit kit, String msg) {
216         Object reply = kit.expectMsgAnyClassOf(JavaTestKit.duration("5 sec"), msgClass, Failure.class);
217         if (reply instanceof Failure) {
218             throw new AssertionError(msg + " failed", ((Failure)reply).cause());
219         }
220
221         return (T)reply;
222     }
223
224     @Test
225     public void testPerShardDatastoreContext() throws Exception {
226         LOG.info("testPerShardDatastoreContext starting");
227         final DatastoreContextFactory mockFactory = newDatastoreContextFactory(
228                 datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
229
230         Mockito.doReturn(
231                 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(6).build())
232                 .when(mockFactory).getShardDatastoreContext("default");
233
234         Mockito.doReturn(
235                 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(7).build())
236                 .when(mockFactory).getShardDatastoreContext("topology");
237
238         final MockConfiguration mockConfig = new MockConfiguration() {
239             @Override
240             public Collection<String> getMemberShardNames(MemberName memberName) {
241                 return Arrays.asList("default", "topology");
242             }
243
244             @Override
245             public Collection<MemberName> getMembersFromShardName(String shardName) {
246                 return members("member-1");
247             }
248         };
249
250         final ActorRef defaultShardActor = actorFactory.createActor(
251                 Props.create(MessageCollectorActor.class), actorFactory.generateActorId("default"));
252         final ActorRef topologyShardActor = actorFactory.createActor(
253                 Props.create(MessageCollectorActor.class), actorFactory.generateActorId("topology"));
254
255         final Map<String, Entry<ActorRef, DatastoreContext>> shardInfoMap = Collections.synchronizedMap(
256                 new HashMap<String, Entry<ActorRef, DatastoreContext>>());
257         shardInfoMap.put("default", new AbstractMap.SimpleEntry<>(defaultShardActor, null));
258         shardInfoMap.put("topology", new AbstractMap.SimpleEntry<>(topologyShardActor, null));
259
260         final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
261         final CountDownLatch newShardActorLatch = new CountDownLatch(2);
262         class LocalShardManager extends ShardManager {
263             LocalShardManager(AbstractShardManagerCreator<?> creator) {
264                 super(creator);
265             }
266
267             @Override
268             protected ActorRef newShardActor(SchemaContext schemaContext, ShardInformation info) {
269                 Entry<ActorRef, DatastoreContext> entry = shardInfoMap.get(info.getShardName());
270                 ActorRef ref = null;
271                 if (entry != null) {
272                     ref = entry.getKey();
273                     entry.setValue(info.getDatastoreContext());
274                 }
275
276                 newShardActorLatch.countDown();
277                 return ref;
278             }
279         }
280
281         final Creator<ShardManager> creator = new Creator<ShardManager>() {
282             private static final long serialVersionUID = 1L;
283             @Override
284             public ShardManager create() throws Exception {
285                 return new LocalShardManager(
286                         new GenericCreator<>(LocalShardManager.class).datastoreContextFactory(mockFactory)
287                                 .primaryShardInfoCache(primaryShardInfoCache).configuration(mockConfig));
288             }
289         };
290
291         JavaTestKit kit = new JavaTestKit(getSystem());
292
293         final ActorRef shardManager = actorFactory.createActor(Props.create(
294                 new DelegatingShardManagerCreator(creator)).withDispatcher(Dispatchers.DefaultDispatcherId()));
295
296         shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), kit.getRef());
297
298         assertEquals("Shard actors created", true, newShardActorLatch.await(5, TimeUnit.SECONDS));
299         assertEquals("getShardElectionTimeoutFactor", 6,
300                 shardInfoMap.get("default").getValue().getShardElectionTimeoutFactor());
301         assertEquals("getShardElectionTimeoutFactor", 7,
302                 shardInfoMap.get("topology").getValue().getShardElectionTimeoutFactor());
303
304         DatastoreContextFactory newMockFactory = newDatastoreContextFactory(
305                 datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
306         Mockito.doReturn(
307                 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(66).build())
308                 .when(newMockFactory).getShardDatastoreContext("default");
309
310         Mockito.doReturn(
311                 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(77).build())
312                 .when(newMockFactory).getShardDatastoreContext("topology");
313
314         shardManager.tell(newMockFactory, kit.getRef());
315
316         DatastoreContext newContext = MessageCollectorActor.expectFirstMatching(defaultShardActor,
317                 DatastoreContext.class);
318         assertEquals("getShardElectionTimeoutFactor", 66, newContext.getShardElectionTimeoutFactor());
319
320         newContext = MessageCollectorActor.expectFirstMatching(topologyShardActor, DatastoreContext.class);
321         assertEquals("getShardElectionTimeoutFactor", 77, newContext.getShardElectionTimeoutFactor());
322
323         LOG.info("testPerShardDatastoreContext ending");
324     }
325
326     @Test
327     public void testOnReceiveFindPrimaryForNonExistentShard() throws Exception {
328         new JavaTestKit(getSystem()) {
329             {
330                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
331
332                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
333
334                 shardManager.tell(new FindPrimary("non-existent", false), getRef());
335
336                 expectMsgClass(duration("5 seconds"), PrimaryNotFoundException.class);
337             }
338         };
339     }
340
341     @Test
342     public void testOnReceiveFindPrimaryForLocalLeaderShard() throws Exception {
343         LOG.info("testOnReceiveFindPrimaryForLocalLeaderShard starting");
344         new JavaTestKit(getSystem()) {
345             {
346                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
347
348                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
349
350                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
351                 shardManager.tell(new ActorInitialized(), mockShardActor);
352
353                 DataTree mockDataTree = mock(DataTree.class);
354                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
355                         DataStoreVersions.CURRENT_VERSION), getRef());
356
357                 MessageCollectorActor.expectFirstMatching(mockShardActor, RegisterRoleChangeListener.class);
358                 shardManager.tell(
359                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
360                         mockShardActor);
361
362                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
363
364                 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
365                         LocalPrimaryShardFound.class);
366                 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
367                         primaryFound.getPrimaryPath().contains("member-1-shard-default"));
368                 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
369             }
370         };
371
372         LOG.info("testOnReceiveFindPrimaryForLocalLeaderShard ending");
373     }
374
375     @Test
376     public void testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp() throws Exception {
377         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp starting");
378         new JavaTestKit(getSystem()) {
379             {
380                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
381
382                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
383                 shardManager.tell(new ActorInitialized(), mockShardActor);
384
385                 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
386                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
387                 shardManager.tell(
388                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
389                         mockShardActor);
390                 shardManager.tell(new LeaderStateChanged(memberId1, memberId2, DataStoreVersions.CURRENT_VERSION),
391                         mockShardActor);
392
393                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
394
395                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
396             }
397         };
398
399         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp ending");
400     }
401
402     @Test
403     public void testOnReceiveFindPrimaryForNonLocalLeaderShard() throws Exception {
404         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShard starting");
405         new JavaTestKit(getSystem()) {
406             {
407                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
408
409                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
410                 shardManager.tell(new ActorInitialized(), mockShardActor);
411
412                 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
413                 MockClusterWrapper.sendMemberUp(shardManager, "member-2", getRef().path().toString());
414
415                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
416                 shardManager.tell(
417                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
418                         mockShardActor);
419                 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
420                 shardManager.tell(new ShardLeaderStateChanged(memberId1, memberId2, leaderVersion), mockShardActor);
421
422                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
423
424                 RemotePrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
425                         RemotePrimaryShardFound.class);
426                 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
427                         primaryFound.getPrimaryPath().contains("member-2-shard-default"));
428                 assertEquals("getPrimaryVersion", leaderVersion, primaryFound.getPrimaryVersion());
429             }
430         };
431
432         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShard ending");
433     }
434
435     @Test
436     public void testOnReceiveFindPrimaryForUninitializedShard() throws Exception {
437         new JavaTestKit(getSystem()) {
438             {
439                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
440
441                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
442
443                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
444             }
445         };
446     }
447
448     @Test
449     public void testOnReceiveFindPrimaryForInitializedShardWithNoRole() throws Exception {
450         new JavaTestKit(getSystem()) {
451             {
452                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
453
454                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
455                 shardManager.tell(new ActorInitialized(), mockShardActor);
456
457                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
458
459                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
460             }
461         };
462     }
463
464     @Test
465     public void testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId() throws Exception {
466         LOG.info("testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId starting");
467         new JavaTestKit(getSystem()) {
468             {
469                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
470
471                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
472                 shardManager.tell(new ActorInitialized(), mockShardActor);
473
474                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
475                 shardManager.tell(
476                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Follower.name()),
477                         mockShardActor);
478
479                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
480
481                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
482
483                 DataTree mockDataTree = mock(DataTree.class);
484                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
485                         DataStoreVersions.CURRENT_VERSION), mockShardActor);
486
487                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
488
489                 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
490                         LocalPrimaryShardFound.class);
491                 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
492                         primaryFound.getPrimaryPath().contains("member-1-shard-default"));
493                 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
494             }
495         };
496
497         LOG.info("testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId starting");
498     }
499
500     @Test
501     public void testOnReceiveFindPrimaryWaitForShardLeader() throws Exception {
502         LOG.info("testOnReceiveFindPrimaryWaitForShardLeader starting");
503         datastoreContextBuilder.shardInitializationTimeout(10, TimeUnit.SECONDS);
504         new JavaTestKit(getSystem()) {
505             {
506                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
507
508                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
509
510                 // We're passing waitUntilInitialized = true to FindPrimary so
511                 // the response should be
512                 // delayed until we send ActorInitialized and
513                 // RoleChangeNotification.
514                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
515
516                 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
517
518                 shardManager.tell(new ActorInitialized(), mockShardActor);
519
520                 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
521
522                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
523                 shardManager.tell(
524                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
525                         mockShardActor);
526
527                 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
528
529                 DataTree mockDataTree = mock(DataTree.class);
530                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
531                         DataStoreVersions.CURRENT_VERSION), mockShardActor);
532
533                 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
534                         LocalPrimaryShardFound.class);
535                 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
536                         primaryFound.getPrimaryPath().contains("member-1-shard-default"));
537                 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
538
539                 expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
540             }
541         };
542
543         LOG.info("testOnReceiveFindPrimaryWaitForShardLeader ending");
544     }
545
546     @Test
547     public void testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard() throws Exception {
548         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard starting");
549         new JavaTestKit(getSystem()) {
550             {
551                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
552
553                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
554
555                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
556
557                 expectMsgClass(duration("2 seconds"), NotInitializedException.class);
558
559                 shardManager.tell(new ActorInitialized(), mockShardActor);
560
561                 expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
562             }
563         };
564
565         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard ending");
566     }
567
568     @Test
569     public void testOnReceiveFindPrimaryWaitForReadyWithCandidateShard() throws Exception {
570         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithCandidateShard starting");
571         new JavaTestKit(getSystem()) {
572             {
573                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
574
575                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
576                 shardManager.tell(new ActorInitialized(), mockShardActor);
577                 shardManager.tell(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix, null,
578                         RaftState.Candidate.name()), mockShardActor);
579
580                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
581
582                 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
583             }
584         };
585
586         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithCandidateShard ending");
587     }
588
589     @Test
590     public void testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard() throws Exception {
591         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard starting");
592         new JavaTestKit(getSystem()) {
593             {
594                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
595
596                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
597                 shardManager.tell(new ActorInitialized(), mockShardActor);
598                 shardManager.tell(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix, null,
599                         RaftState.IsolatedLeader.name()), mockShardActor);
600
601                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
602
603                 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
604             }
605         };
606
607         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard ending");
608     }
609
610     @Test
611     public void testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard() throws Exception {
612         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard starting");
613         new JavaTestKit(getSystem()) {
614             {
615                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
616
617                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
618                 shardManager.tell(new ActorInitialized(), mockShardActor);
619
620                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
621
622                 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
623             }
624         };
625
626         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard ending");
627     }
628
629     @Test
630     public void testOnReceiveFindPrimaryForRemoteShard() throws Exception {
631         LOG.info("testOnReceiveFindPrimaryForRemoteShard starting");
632         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
633
634         // Create an ActorSystem ShardManager actor for member-1.
635
636         final ActorSystem system1 = newActorSystem("Member1");
637         Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
638
639         final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
640                 newTestShardMgrBuilderWithMockShardActor().cluster(
641                         new ClusterWrapperImpl(system1)).props().withDispatcher(
642                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
643
644         // Create an ActorSystem ShardManager actor for member-2.
645
646         final ActorSystem system2 = newActorSystem("Member2");
647
648         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
649
650         final ActorRef mockShardActor2 = newMockShardActor(system2, "astronauts", "member-2");
651
652         MockConfiguration mockConfig2 = new MockConfiguration(
653                 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
654                         .put("astronauts", Arrays.asList("member-2")).build());
655
656         final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
657                 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
658                         new ClusterWrapperImpl(system2)).props().withDispatcher(
659                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
660
661         new JavaTestKit(system1) {
662             {
663                 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
664                 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
665
666                 shardManager2.tell(new ActorInitialized(), mockShardActor2);
667
668                 String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
669                 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
670                 shardManager2.tell(
671                         new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
672                         mockShardActor2);
673                 shardManager2.tell(
674                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
675                         mockShardActor2);
676
677                 shardManager1.underlyingActor().waitForMemberUp();
678                 shardManager1.tell(new FindPrimary("astronauts", false), getRef());
679
680                 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
681                 String path = found.getPrimaryPath();
682                 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-astronauts-config"));
683                 assertEquals("getPrimaryVersion", leaderVersion, found.getPrimaryVersion());
684
685                 shardManager2.underlyingActor().verifyFindPrimary();
686
687                 Cluster.get(system2).down(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
688
689                 shardManager1.underlyingActor().waitForMemberRemoved();
690
691                 shardManager1.tell(new FindPrimary("astronauts", false), getRef());
692
693                 expectMsgClass(duration("5 seconds"), PrimaryNotFoundException.class);
694             }
695         };
696
697         LOG.info("testOnReceiveFindPrimaryForRemoteShard ending");
698     }
699
700     @Test
701     public void testShardAvailabilityOnChangeOfMemberReachability() throws Exception {
702         LOG.info("testShardAvailabilityOnChangeOfMemberReachability starting");
703         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
704
705         // Create an ActorSystem ShardManager actor for member-1.
706
707         final ActorSystem system1 = newActorSystem("Member1");
708         Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
709
710         final ActorRef mockShardActor1 = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
711
712         final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
713                 newTestShardMgrBuilder().shardActor(mockShardActor1).cluster(
714                         new ClusterWrapperImpl(system1)).props().withDispatcher(
715                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
716
717         // Create an ActorSystem ShardManager actor for member-2.
718
719         final ActorSystem system2 = newActorSystem("Member2");
720
721         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
722
723         final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
724
725         MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
726                 .put("default", Arrays.asList("member-1", "member-2")).build());
727
728         final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
729                 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
730                         new ClusterWrapperImpl(system2)).props().withDispatcher(
731                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
732
733         new JavaTestKit(system1) {
734             {
735                 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
736                 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
737                 shardManager1.tell(new ActorInitialized(), mockShardActor1);
738                 shardManager2.tell(new ActorInitialized(), mockShardActor2);
739
740                 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
741                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
742                 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class),
743                         DataStoreVersions.CURRENT_VERSION), mockShardActor1);
744                 shardManager1.tell(
745                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
746                         mockShardActor1);
747                 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class),
748                         DataStoreVersions.CURRENT_VERSION), mockShardActor2);
749                 shardManager2.tell(
750                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
751                         mockShardActor2);
752                 shardManager1.underlyingActor().waitForMemberUp();
753
754                 shardManager1.tell(new FindPrimary("default", true), getRef());
755
756                 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
757                 String path = found.getPrimaryPath();
758                 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
759
760                 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
761                         "akka://cluster-test@127.0.0.1:2558"), getRef());
762
763                 shardManager1.underlyingActor().waitForUnreachableMember();
764
765                 PeerDown peerDown = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
766                 assertEquals("getMemberName", MEMBER_2, peerDown.getMemberName());
767                 MessageCollectorActor.clearMessages(mockShardActor1);
768
769                 shardManager1.tell(
770                         MockClusterWrapper.createMemberRemoved("member-2", "akka://cluster-test@127.0.0.1:2558"),
771                         getRef());
772
773                 MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
774
775                 shardManager1.tell(new FindPrimary("default", true), getRef());
776
777                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
778
779                 shardManager1.tell(
780                         MockClusterWrapper.createReachableMember("member-2", "akka://cluster-test@127.0.0.1:2558"),
781                         getRef());
782
783                 shardManager1.underlyingActor().waitForReachableMember();
784
785                 PeerUp peerUp = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
786                 assertEquals("getMemberName", MEMBER_2, peerUp.getMemberName());
787                 MessageCollectorActor.clearMessages(mockShardActor1);
788
789                 shardManager1.tell(new FindPrimary("default", true), getRef());
790
791                 RemotePrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
792                 String path1 = found1.getPrimaryPath();
793                 assertTrue("Unexpected primary path " + path1, path1.contains("member-2-shard-default-config"));
794
795                 shardManager1.tell(
796                         MockClusterWrapper.createMemberUp("member-2", "akka://cluster-test@127.0.0.1:2558"),
797                         getRef());
798
799                 MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
800
801                 // Test FindPrimary wait succeeds after reachable member event.
802
803                 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
804                         "akka://cluster-test@127.0.0.1:2558"), getRef());
805                 shardManager1.underlyingActor().waitForUnreachableMember();
806
807                 shardManager1.tell(new FindPrimary("default", true), getRef());
808
809                 shardManager1.tell(
810                         MockClusterWrapper.createReachableMember("member-2", "akka://cluster-test@127.0.0.1:2558"),
811                         getRef());
812
813                 RemotePrimaryShardFound found2 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
814                 String path2 = found2.getPrimaryPath();
815                 assertTrue("Unexpected primary path " + path2, path2.contains("member-2-shard-default-config"));
816             }
817         };
818
819         LOG.info("testShardAvailabilityOnChangeOfMemberReachability ending");
820     }
821
822     @Test
823     public void testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange() throws Exception {
824         LOG.info("testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange starting");
825         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
826
827         // Create an ActorSystem ShardManager actor for member-1.
828
829         final ActorSystem system1 = newActorSystem("Member1");
830         Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
831
832         final ActorRef mockShardActor1 = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
833
834         final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
835         final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
836                 newTestShardMgrBuilder().shardActor(mockShardActor1).cluster(new ClusterWrapperImpl(system1))
837                         .primaryShardInfoCache(primaryShardInfoCache).props()
838                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
839                 shardManagerID);
840
841         // Create an ActorSystem ShardManager actor for member-2.
842
843         final ActorSystem system2 = newActorSystem("Member2");
844
845         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
846
847         final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
848
849         MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
850                 .put("default", Arrays.asList("member-1", "member-2")).build());
851
852         final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
853                 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
854                         new ClusterWrapperImpl(system2)).props().withDispatcher(
855                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
856
857         new JavaTestKit(system1) {
858             {
859                 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
860                 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
861                 shardManager1.tell(new ActorInitialized(), mockShardActor1);
862                 shardManager2.tell(new ActorInitialized(), mockShardActor2);
863
864                 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
865                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
866                 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class),
867                         DataStoreVersions.CURRENT_VERSION), mockShardActor1);
868                 shardManager1.tell(
869                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
870                         mockShardActor1);
871                 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class),
872                         DataStoreVersions.CURRENT_VERSION), mockShardActor2);
873                 shardManager2.tell(
874                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
875                         mockShardActor2);
876                 shardManager1.underlyingActor().waitForMemberUp();
877
878                 shardManager1.tell(new FindPrimary("default", true), getRef());
879
880                 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
881                 String path = found.getPrimaryPath();
882                 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
883
884                 primaryShardInfoCache.putSuccessful("default", new PrimaryShardInfo(
885                         system1.actorSelection(mockShardActor1.path()), DataStoreVersions.CURRENT_VERSION));
886
887                 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
888                         "akka://cluster-test@127.0.0.1:2558"), getRef());
889
890                 shardManager1.underlyingActor().waitForUnreachableMember();
891
892                 shardManager1.tell(new FindPrimary("default", true), getRef());
893
894                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
895
896                 assertNull("Expected primaryShardInfoCache entry removed",
897                         primaryShardInfoCache.getIfPresent("default"));
898
899                 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId1, mock(DataTree.class),
900                         DataStoreVersions.CURRENT_VERSION), mockShardActor1);
901                 shardManager1.tell(
902                         new RoleChangeNotification(memberId1, RaftState.Follower.name(), RaftState.Leader.name()),
903                         mockShardActor1);
904
905                 shardManager1.tell(new FindPrimary("default", true), getRef());
906
907                 LocalPrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
908                 String path1 = found1.getPrimaryPath();
909                 assertTrue("Unexpected primary path " + path1, path1.contains("member-1-shard-default-config"));
910
911             }
912         };
913
914         LOG.info("testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange ending");
915     }
916
917
918     @Test
919     public void testOnReceiveFindLocalShardForNonExistentShard() throws Exception {
920         new JavaTestKit(getSystem()) {
921             {
922                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
923
924                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
925
926                 shardManager.tell(new FindLocalShard("non-existent", false), getRef());
927
928                 LocalShardNotFound notFound = expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
929
930                 assertEquals("getShardName", "non-existent", notFound.getShardName());
931             }
932         };
933     }
934
935     @Test
936     public void testOnReceiveFindLocalShardForExistentShard() throws Exception {
937         new JavaTestKit(getSystem()) {
938             {
939                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
940
941                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
942                 shardManager.tell(new ActorInitialized(), mockShardActor);
943
944                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
945
946                 LocalShardFound found = expectMsgClass(duration("5 seconds"), LocalShardFound.class);
947
948                 assertTrue("Found path contains " + found.getPath().path().toString(),
949                         found.getPath().path().toString().contains("member-1-shard-default-config"));
950             }
951         };
952     }
953
954     @Test
955     public void testOnReceiveFindLocalShardForNotInitializedShard() throws Exception {
956         new JavaTestKit(getSystem()) {
957             {
958                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
959
960                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
961
962                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
963             }
964         };
965     }
966
967     @Test
968     public void testOnReceiveFindLocalShardWaitForShardInitialized() throws Exception {
969         LOG.info("testOnReceiveFindLocalShardWaitForShardInitialized starting");
970         new JavaTestKit(getSystem()) {
971             {
972                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
973
974                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
975
976                 // We're passing waitUntilInitialized = true to FindLocalShard
977                 // so the response should be
978                 // delayed until we send ActorInitialized.
979                 Future<Object> future = Patterns.ask(shardManager, new FindLocalShard(Shard.DEFAULT_NAME, true),
980                         new Timeout(5, TimeUnit.SECONDS));
981
982                 shardManager.tell(new ActorInitialized(), mockShardActor);
983
984                 Object resp = Await.result(future, duration("5 seconds"));
985                 assertTrue("Expected: LocalShardFound, Actual: " + resp, resp instanceof LocalShardFound);
986             }
987         };
988
989         LOG.info("testOnReceiveFindLocalShardWaitForShardInitialized starting");
990     }
991
992     @Test
993     public void testOnRecoveryJournalIsCleaned() {
994         String persistenceID = "shard-manager-" + shardMrgIDSuffix;
995         InMemoryJournal.addEntry(persistenceID, 1L, new SchemaContextModules(ImmutableSet.of("foo")));
996         InMemoryJournal.addEntry(persistenceID, 2L, new SchemaContextModules(ImmutableSet.of("bar")));
997         InMemoryJournal.addDeleteMessagesCompleteLatch(persistenceID);
998
999         newTestShardManager();
1000
1001         InMemoryJournal.waitForDeleteMessagesComplete(persistenceID);
1002
1003         // Journal entries up to the last one should've been deleted
1004         Map<Long, Object> journal = InMemoryJournal.get(persistenceID);
1005         synchronized (journal) {
1006             assertEquals("Journal size", 0, journal.size());
1007         }
1008     }
1009
1010     @Test
1011     public void testRoleChangeNotificationAndShardLeaderStateChangedReleaseReady() throws Exception {
1012         TestShardManager shardManager = newTestShardManager();
1013
1014         String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1015         shardManager.onReceiveCommand(new RoleChangeNotification(
1016                 memberId, RaftState.Candidate.name(), RaftState.Leader.name()));
1017
1018         verify(ready, never()).countDown();
1019
1020         shardManager.onReceiveCommand(new ShardLeaderStateChanged(memberId, memberId,
1021                 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1022
1023         verify(ready, times(1)).countDown();
1024     }
1025
1026     @Test
1027     public void testRoleChangeNotificationToFollowerWithShardLeaderStateChangedReleaseReady() throws Exception {
1028         new JavaTestKit(getSystem()) {
1029             {
1030                 TestShardManager shardManager = newTestShardManager();
1031
1032                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1033                 shardManager.onReceiveCommand(new RoleChangeNotification(memberId, null, RaftState.Follower.name()));
1034
1035                 verify(ready, never()).countDown();
1036
1037                 shardManager
1038                         .onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
1039
1040                 shardManager.onReceiveCommand(
1041                         new ShardLeaderStateChanged(memberId, "member-2-shard-default-" + shardMrgIDSuffix,
1042                                 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1043
1044                 verify(ready, times(1)).countDown();
1045             }
1046         };
1047     }
1048
1049     @Test
1050     public void testReadyCountDownForMemberUpAfterLeaderStateChanged() throws Exception {
1051         new JavaTestKit(getSystem()) {
1052             {
1053                 TestShardManager shardManager = newTestShardManager();
1054
1055                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1056                 shardManager.onReceiveCommand(new RoleChangeNotification(memberId, null, RaftState.Follower.name()));
1057
1058                 verify(ready, never()).countDown();
1059
1060                 shardManager.onReceiveCommand(
1061                         new ShardLeaderStateChanged(memberId, "member-2-shard-default-" + shardMrgIDSuffix,
1062                                 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1063
1064                 shardManager
1065                         .onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
1066
1067                 verify(ready, times(1)).countDown();
1068             }
1069         };
1070     }
1071
1072     @Test
1073     public void testRoleChangeNotificationDoNothingForUnknownShard() throws Exception {
1074         TestShardManager shardManager = newTestShardManager();
1075
1076         shardManager.onReceiveCommand(new RoleChangeNotification(
1077                 "unknown", RaftState.Candidate.name(), RaftState.Leader.name()));
1078
1079         verify(ready, never()).countDown();
1080     }
1081
1082     @Test
1083     public void testByDefaultSyncStatusIsFalse() throws Exception {
1084         TestShardManager shardManager = newTestShardManager();
1085
1086         assertEquals(false, shardManager.getMBean().getSyncStatus());
1087     }
1088
1089     @Test
1090     public void testWhenShardIsLeaderSyncStatusIsTrue() throws Exception {
1091         TestShardManager shardManager = newTestShardManager();
1092
1093         shardManager.onReceiveCommand(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix,
1094                 RaftState.Follower.name(), RaftState.Leader.name()));
1095
1096         assertEquals(true, shardManager.getMBean().getSyncStatus());
1097     }
1098
1099     @Test
1100     public void testWhenShardIsCandidateSyncStatusIsFalse() throws Exception {
1101         TestShardManager shardManager = newTestShardManager();
1102
1103         String shardId = "member-1-shard-default-" + shardMrgIDSuffix;
1104         shardManager.onReceiveCommand(new RoleChangeNotification(shardId,
1105                 RaftState.Follower.name(), RaftState.Candidate.name()));
1106
1107         assertEquals(false, shardManager.getMBean().getSyncStatus());
1108
1109         // Send a FollowerInitialSyncStatus with status = true for the replica whose current state is candidate
1110         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(
1111                 true, shardId));
1112
1113         assertEquals(false, shardManager.getMBean().getSyncStatus());
1114     }
1115
1116     @Test
1117     public void testWhenShardIsFollowerSyncStatusDependsOnFollowerInitialSyncStatus() throws Exception {
1118         TestShardManager shardManager = newTestShardManager();
1119
1120         String shardId = "member-1-shard-default-" + shardMrgIDSuffix;
1121         shardManager.onReceiveCommand(new RoleChangeNotification(shardId,
1122                 RaftState.Candidate.name(), RaftState.Follower.name()));
1123
1124         // Initially will be false
1125         assertEquals(false, shardManager.getMBean().getSyncStatus());
1126
1127         // Send status true will make sync status true
1128         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(true, shardId));
1129
1130         assertEquals(true, shardManager.getMBean().getSyncStatus());
1131
1132         // Send status false will make sync status false
1133         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(false, shardId));
1134
1135         assertEquals(false, shardManager.getMBean().getSyncStatus());
1136
1137     }
1138
1139     @Test
1140     public void testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards() throws Exception {
1141         LOG.info("testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards starting");
1142         TestShardManager shardManager = newTestShardManager(newShardMgrProps(new MockConfiguration() {
1143             @Override
1144             public List<String> getMemberShardNames(MemberName memberName) {
1145                 return Arrays.asList("default", "astronauts");
1146             }
1147         }));
1148
1149         // Initially will be false
1150         assertEquals(false, shardManager.getMBean().getSyncStatus());
1151
1152         // Make default shard leader
1153         String defaultShardId = "member-1-shard-default-" + shardMrgIDSuffix;
1154         shardManager.onReceiveCommand(new RoleChangeNotification(defaultShardId,
1155                 RaftState.Follower.name(), RaftState.Leader.name()));
1156
1157         // default = Leader, astronauts is unknown so sync status remains false
1158         assertEquals(false, shardManager.getMBean().getSyncStatus());
1159
1160         // Make astronauts shard leader as well
1161         String astronautsShardId = "member-1-shard-astronauts-" + shardMrgIDSuffix;
1162         shardManager.onReceiveCommand(new RoleChangeNotification(astronautsShardId,
1163                 RaftState.Follower.name(), RaftState.Leader.name()));
1164
1165         // Now sync status should be true
1166         assertEquals(true, shardManager.getMBean().getSyncStatus());
1167
1168         // Make astronauts a Follower
1169         shardManager.onReceiveCommand(new RoleChangeNotification(astronautsShardId,
1170                 RaftState.Leader.name(), RaftState.Follower.name()));
1171
1172         // Sync status is not true
1173         assertEquals(false, shardManager.getMBean().getSyncStatus());
1174
1175         // Make the astronauts follower sync status true
1176         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(true, astronautsShardId));
1177
1178         // Sync status is now true
1179         assertEquals(true, shardManager.getMBean().getSyncStatus());
1180
1181         LOG.info("testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards ending");
1182     }
1183
1184     @Test
1185     public void testOnReceiveSwitchShardBehavior() throws Exception {
1186         new JavaTestKit(getSystem()) {
1187             {
1188                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1189
1190                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1191                 shardManager.tell(new ActorInitialized(), mockShardActor);
1192
1193                 shardManager.tell(new SwitchShardBehavior(mockShardName, RaftState.Leader, 1000), getRef());
1194
1195                 SwitchBehavior switchBehavior = MessageCollectorActor.expectFirstMatching(mockShardActor,
1196                         SwitchBehavior.class);
1197
1198                 assertEquals(RaftState.Leader, switchBehavior.getNewState());
1199                 assertEquals(1000, switchBehavior.getNewTerm());
1200             }
1201         };
1202     }
1203
1204     private static List<MemberName> members(String... names) {
1205         return Arrays.asList(names).stream().map(MemberName::forName).collect(Collectors.toList());
1206     }
1207
1208     @Test
1209     public void testOnCreateShard() {
1210         LOG.info("testOnCreateShard starting");
1211         new JavaTestKit(getSystem()) {
1212             {
1213                 datastoreContextBuilder.shardInitializationTimeout(1, TimeUnit.MINUTES).persistent(true);
1214
1215                 ActorRef shardManager = actorFactory
1216                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1217                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1218
1219                 SchemaContext schemaContext = TestModel.createTestContext();
1220                 shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
1221
1222                 DatastoreContext datastoreContext = DatastoreContext.newBuilder().shardElectionTimeoutFactor(100)
1223                         .persistent(false).build();
1224                 Shard.Builder shardBuilder = Shard.builder();
1225
1226                 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1227                         "foo", null, members("member-1", "member-5", "member-6"));
1228                 shardManager.tell(new CreateShard(config, shardBuilder, datastoreContext), getRef());
1229
1230                 expectMsgClass(duration("5 seconds"), Success.class);
1231
1232                 shardManager.tell(new FindLocalShard("foo", true), getRef());
1233
1234                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1235
1236                 assertEquals("isRecoveryApplicable", false, shardBuilder.getDatastoreContext().isPersistent());
1237                 assertTrue("Epxected ShardPeerAddressResolver", shardBuilder.getDatastoreContext().getShardRaftConfig()
1238                         .getPeerAddressResolver() instanceof ShardPeerAddressResolver);
1239                 assertEquals("peerMembers", Sets.newHashSet(
1240                         ShardIdentifier.create("foo", MemberName.forName("member-5"), shardMrgIDSuffix).toString(),
1241                         ShardIdentifier.create("foo", MemberName.forName("member-6"), shardMrgIDSuffix).toString()),
1242                         shardBuilder.getPeerAddresses().keySet());
1243                 assertEquals("ShardIdentifier", ShardIdentifier.create("foo", MEMBER_1, shardMrgIDSuffix),
1244                         shardBuilder.getId());
1245                 assertSame("schemaContext", schemaContext, shardBuilder.getSchemaContext());
1246
1247                 // Send CreateShard with same name - should return Success with
1248                 // a message.
1249
1250                 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1251
1252                 Success success = expectMsgClass(duration("5 seconds"), Success.class);
1253                 assertNotNull("Success status is null", success.status());
1254             }
1255         };
1256
1257         LOG.info("testOnCreateShard ending");
1258     }
1259
1260     @Test
1261     public void testOnCreateShardWithLocalMemberNotInShardConfig() {
1262         LOG.info("testOnCreateShardWithLocalMemberNotInShardConfig starting");
1263         new JavaTestKit(getSystem()) {
1264             {
1265                 datastoreContextBuilder.shardInitializationTimeout(1, TimeUnit.MINUTES).persistent(true);
1266
1267                 ActorRef shardManager = actorFactory
1268                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1269                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1270
1271                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1272
1273                 Shard.Builder shardBuilder = Shard.builder();
1274                 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1275                         "foo", null, members("member-5", "member-6"));
1276
1277                 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1278                 expectMsgClass(duration("5 seconds"), Success.class);
1279
1280                 shardManager.tell(new FindLocalShard("foo", true), getRef());
1281                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1282
1283                 assertEquals("peerMembers size", 0, shardBuilder.getPeerAddresses().size());
1284                 assertEquals("schemaContext", DisableElectionsRaftPolicy.class.getName(), shardBuilder
1285                         .getDatastoreContext().getShardRaftConfig().getCustomRaftPolicyImplementationClass());
1286             }
1287         };
1288
1289         LOG.info("testOnCreateShardWithLocalMemberNotInShardConfig ending");
1290     }
1291
1292     @Test
1293     public void testOnCreateShardWithNoInitialSchemaContext() {
1294         LOG.info("testOnCreateShardWithNoInitialSchemaContext starting");
1295         new JavaTestKit(getSystem()) {
1296             {
1297                 ActorRef shardManager = actorFactory
1298                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1299                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1300
1301                 Shard.Builder shardBuilder = Shard.builder();
1302
1303                 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1304                         "foo", null, members("member-1"));
1305                 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1306
1307                 expectMsgClass(duration("5 seconds"), Success.class);
1308
1309                 SchemaContext schemaContext = TestModel.createTestContext();
1310                 shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
1311
1312                 shardManager.tell(new FindLocalShard("foo", true), getRef());
1313
1314                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1315
1316                 assertSame("schemaContext", schemaContext, shardBuilder.getSchemaContext());
1317                 assertNotNull("schemaContext is null", shardBuilder.getDatastoreContext());
1318             }
1319         };
1320
1321         LOG.info("testOnCreateShardWithNoInitialSchemaContext ending");
1322     }
1323
1324     @Test
1325     public void testGetSnapshot() throws Exception {
1326         LOG.info("testGetSnapshot starting");
1327         JavaTestKit kit = new JavaTestKit(getSystem());
1328
1329         MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1330                 .put("shard1", Arrays.asList("member-1")).put("shard2", Arrays.asList("member-1"))
1331                 .put("astronauts", Collections.<String>emptyList()).build());
1332
1333         TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(newShardMgrProps(mockConfig)
1334                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1335
1336         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1337         Failure failure = kit.expectMsgClass(Failure.class);
1338         assertEquals("Failure cause type", IllegalStateException.class, failure.cause().getClass());
1339
1340         shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1341
1342         waitForShardInitialized(shardManager, "shard1", kit);
1343         waitForShardInitialized(shardManager, "shard2", kit);
1344
1345         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1346
1347         DatastoreSnapshot datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1348
1349         assertEquals("getType", shardMrgIDSuffix, datastoreSnapshot.getType());
1350         assertNull("Expected null ShardManagerSnapshot", datastoreSnapshot.getShardManagerSnapshot());
1351
1352         Function<ShardSnapshot, String> shardNameTransformer = s -> s.getName();
1353
1354         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2"), Sets.newHashSet(
1355                 Lists.transform(datastoreSnapshot.getShardSnapshots(), shardNameTransformer)));
1356
1357         // Add a new replica
1358
1359         JavaTestKit mockShardLeaderKit = new JavaTestKit(getSystem());
1360
1361         TestShardManager shardManagerInstance = shardManager.underlyingActor();
1362         shardManagerInstance.setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1363
1364         shardManager.tell(new AddShardReplica("astronauts"), kit.getRef());
1365         mockShardLeaderKit.expectMsgClass(AddServer.class);
1366         mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.OK, ""));
1367         kit.expectMsgClass(Status.Success.class);
1368         waitForShardInitialized(shardManager, "astronauts", kit);
1369
1370         // Send another GetSnapshot and verify
1371
1372         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1373         datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1374
1375         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"), Sets.newHashSet(
1376                 Lists.transform(datastoreSnapshot.getShardSnapshots(), shardNameTransformer)));
1377
1378         ShardManagerSnapshot snapshot = datastoreSnapshot.getShardManagerSnapshot();
1379         assertNotNull("Expected ShardManagerSnapshot", snapshot);
1380         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"),
1381                 Sets.newHashSet(snapshot.getShardList()));
1382
1383         LOG.info("testGetSnapshot ending");
1384     }
1385
1386     @Test
1387     public void testRestoreFromSnapshot() throws Exception {
1388         LOG.info("testRestoreFromSnapshot starting");
1389
1390         datastoreContextBuilder.shardInitializationTimeout(3, TimeUnit.SECONDS);
1391
1392         JavaTestKit kit = new JavaTestKit(getSystem());
1393
1394         MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1395                 .put("shard1", Collections.<String>emptyList()).put("shard2", Collections.<String>emptyList())
1396                 .put("astronauts", Collections.<String>emptyList()).build());
1397
1398         ShardManagerSnapshot snapshot =
1399                 new ShardManagerSnapshot(Arrays.asList("shard1", "shard2", "astronauts"), Collections.emptyMap());
1400         DatastoreSnapshot restoreFromSnapshot = new DatastoreSnapshot(shardMrgIDSuffix, snapshot,
1401                 Collections.<ShardSnapshot>emptyList());
1402         TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(newTestShardMgrBuilder(mockConfig)
1403                 .restoreFromSnapshot(restoreFromSnapshot).props().withDispatcher(Dispatchers.DefaultDispatcherId()));
1404
1405         shardManager.underlyingActor().waitForRecoveryComplete();
1406
1407         shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1408
1409         waitForShardInitialized(shardManager, "shard1", kit);
1410         waitForShardInitialized(shardManager, "shard2", kit);
1411         waitForShardInitialized(shardManager, "astronauts", kit);
1412
1413         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1414
1415         DatastoreSnapshot datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1416
1417         assertEquals("getType", shardMrgIDSuffix, datastoreSnapshot.getType());
1418
1419         assertNotNull("Expected ShardManagerSnapshot", datastoreSnapshot.getShardManagerSnapshot());
1420         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"),
1421                 Sets.newHashSet(datastoreSnapshot.getShardManagerSnapshot().getShardList()));
1422
1423         LOG.info("testRestoreFromSnapshot ending");
1424     }
1425
1426     @Test
1427     public void testAddShardReplicaForNonExistentShardConfig() throws Exception {
1428         new JavaTestKit(getSystem()) {
1429             {
1430                 ActorRef shardManager = actorFactory
1431                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1432                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1433
1434                 shardManager.tell(new AddShardReplica("model-inventory"), getRef());
1435                 Status.Failure resp = expectMsgClass(duration("2 seconds"), Status.Failure.class);
1436
1437                 assertEquals("Failure obtained", true, resp.cause() instanceof IllegalArgumentException);
1438             }
1439         };
1440     }
1441
1442     @Test
1443     public void testAddShardReplica() throws Exception {
1444         LOG.info("testAddShardReplica starting");
1445         MockConfiguration mockConfig = new MockConfiguration(
1446                 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
1447                         .put("astronauts", Arrays.asList("member-2")).build());
1448
1449         final String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
1450         datastoreContextBuilder.shardManagerPersistenceId(shardManagerID);
1451
1452         // Create an ActorSystem ShardManager actor for member-1.
1453         final ActorSystem system1 = newActorSystem("Member1");
1454         Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1455         ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
1456         final TestActorRef<TestShardManager> newReplicaShardManager = TestActorRef.create(system1,
1457                 newTestShardMgrBuilder(mockConfig).shardActor(mockDefaultShardActor)
1458                         .cluster(new ClusterWrapperImpl(system1)).props()
1459                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
1460                 shardManagerID);
1461
1462         // Create an ActorSystem ShardManager actor for member-2.
1463         final ActorSystem system2 = newActorSystem("Member2");
1464         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1465
1466         String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
1467         String name = ShardIdentifier.create("astronauts", MEMBER_2, "config").toString();
1468         final TestActorRef<MockRespondActor> mockShardLeaderActor = TestActorRef.create(system2,
1469                 Props.create(MockRespondActor.class, AddServer.class,
1470                         new AddServerReply(ServerChangeStatus.OK, memberId2))
1471                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
1472                 name);
1473         final TestActorRef<TestShardManager> leaderShardManager = TestActorRef.create(system2,
1474                 newTestShardMgrBuilder(mockConfig).shardActor(mockShardLeaderActor)
1475                         .cluster(new ClusterWrapperImpl(system2)).props()
1476                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
1477                 shardManagerID);
1478
1479         new JavaTestKit(system1) {
1480             {
1481                 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1482                 leaderShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1483
1484                 leaderShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1485
1486                 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
1487                 leaderShardManager.tell(
1488                         new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
1489                         mockShardLeaderActor);
1490                 leaderShardManager.tell(
1491                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
1492                         mockShardLeaderActor);
1493
1494                 newReplicaShardManager.underlyingActor().waitForMemberUp();
1495                 leaderShardManager.underlyingActor().waitForMemberUp();
1496
1497                 // Have a dummy snapshot to be overwritten by the new data
1498                 // persisted.
1499                 String[] restoredShards = { "default", "people" };
1500                 ShardManagerSnapshot snapshot =
1501                         new ShardManagerSnapshot(Arrays.asList(restoredShards), Collections.emptyMap());
1502                 InMemorySnapshotStore.addSnapshot(shardManagerID, snapshot);
1503                 Uninterruptibles.sleepUninterruptibly(2, TimeUnit.MILLISECONDS);
1504
1505                 InMemorySnapshotStore.addSnapshotSavedLatch(shardManagerID);
1506                 InMemorySnapshotStore.addSnapshotDeletedLatch(shardManagerID);
1507
1508                 // construct a mock response message
1509                 newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
1510                 AddServer addServerMsg = MessageCollectorActor.expectFirstMatching(mockShardLeaderActor,
1511                         AddServer.class);
1512                 String addServerId = "member-1-shard-astronauts-" + shardMrgIDSuffix;
1513                 assertEquals("AddServer serverId", addServerId, addServerMsg.getNewServerId());
1514                 expectMsgClass(duration("5 seconds"), Status.Success.class);
1515
1516                 InMemorySnapshotStore.waitForSavedSnapshot(shardManagerID, ShardManagerSnapshot.class);
1517                 InMemorySnapshotStore.waitForDeletedSnapshot(shardManagerID);
1518                 List<ShardManagerSnapshot> persistedSnapshots = InMemorySnapshotStore.getSnapshots(shardManagerID,
1519                         ShardManagerSnapshot.class);
1520                 assertEquals("Number of snapshots persisted", 1, persistedSnapshots.size());
1521                 ShardManagerSnapshot shardManagerSnapshot = persistedSnapshots.get(0);
1522                 assertEquals("Persisted local shards", Sets.newHashSet("default", "astronauts"),
1523                         Sets.newHashSet(shardManagerSnapshot.getShardList()));
1524             }
1525         };
1526         LOG.info("testAddShardReplica ending");
1527     }
1528
1529     @Test
1530     public void testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader() throws Exception {
1531         LOG.info("testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader starting");
1532         new JavaTestKit(getSystem()) {
1533             {
1534                 TestActorRef<TestShardManager> shardManager = actorFactory
1535                         .createTestActor(newPropsShardMgrWithMockShardActor(), shardMgrID);
1536
1537                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1538                 shardManager.tell(new ActorInitialized(), mockShardActor);
1539
1540                 String leaderId = "leader-member-shard-default-" + shardMrgIDSuffix;
1541                 AddServerReply addServerReply = new AddServerReply(ServerChangeStatus.ALREADY_EXISTS, null);
1542                 ActorRef leaderShardActor = shardManager.underlyingActor().getContext()
1543                         .actorOf(Props.create(MockRespondActor.class, AddServer.class, addServerReply), leaderId);
1544
1545                 MockClusterWrapper.sendMemberUp(shardManager, "leader-member", leaderShardActor.path().toString());
1546
1547                 String newReplicaId = "member-1-shard-default-" + shardMrgIDSuffix;
1548                 shardManager.tell(
1549                         new RoleChangeNotification(newReplicaId, RaftState.Candidate.name(), RaftState.Follower.name()),
1550                         mockShardActor);
1551                 shardManager.tell(
1552                         new ShardLeaderStateChanged(newReplicaId, leaderId, DataStoreVersions.CURRENT_VERSION),
1553                         mockShardActor);
1554
1555                 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1556
1557                 MessageCollectorActor.expectFirstMatching(leaderShardActor, AddServer.class);
1558
1559                 Failure resp = expectMsgClass(duration("5 seconds"), Failure.class);
1560                 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1561
1562                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1563                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1564
1565                 // Send message again to verify previous in progress state is
1566                 // cleared
1567
1568                 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1569                 resp = expectMsgClass(duration("5 seconds"), Failure.class);
1570                 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1571
1572                 // Send message again with an AddServer timeout to verify the
1573                 // pre-existing shard actor isn't terminated.
1574
1575                 shardManager.tell(
1576                         newDatastoreContextFactory(
1577                                 datastoreContextBuilder.shardLeaderElectionTimeout(100, TimeUnit.MILLISECONDS).build()),
1578                         getRef());
1579                 leaderShardActor.tell(MockRespondActor.CLEAR_RESPONSE, ActorRef.noSender());
1580                 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1581                 expectMsgClass(duration("5 seconds"), Failure.class);
1582
1583                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1584                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1585             }
1586         };
1587
1588         LOG.info("testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader ending");
1589     }
1590
1591     @Test
1592     public void testAddShardReplicaWithPreExistingLocalReplicaLeader() throws Exception {
1593         LOG.info("testAddShardReplicaWithPreExistingLocalReplicaLeader starting");
1594         new JavaTestKit(getSystem()) {
1595             {
1596                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1597                 ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1598
1599                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1600                 shardManager.tell(new ActorInitialized(), mockShardActor);
1601                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
1602                         DataStoreVersions.CURRENT_VERSION), getRef());
1603                 shardManager.tell(
1604                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
1605                         mockShardActor);
1606
1607                 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1608                 Failure resp = expectMsgClass(duration("5 seconds"), Failure.class);
1609                 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1610
1611                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1612                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1613             }
1614         };
1615
1616         LOG.info("testAddShardReplicaWithPreExistingLocalReplicaLeader ending");
1617     }
1618
1619     @Test
1620     public void testAddShardReplicaWithAddServerReplyFailure() throws Exception {
1621         LOG.info("testAddShardReplicaWithAddServerReplyFailure starting");
1622         new JavaTestKit(getSystem()) {
1623             {
1624                 JavaTestKit mockShardLeaderKit = new JavaTestKit(getSystem());
1625
1626                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1627                         .put("astronauts", Arrays.asList("member-2")).build());
1628
1629                 ActorRef mockNewReplicaShardActor = newMockShardActor(getSystem(), "astronauts", "member-1");
1630                 final TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(
1631                         newTestShardMgrBuilder(mockConfig).shardActor(mockNewReplicaShardActor).props()
1632                             .withDispatcher(Dispatchers.DefaultDispatcherId()), shardMgrID);
1633                 shardManager.underlyingActor()
1634                         .setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1635
1636                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1637
1638                 JavaTestKit terminateWatcher = new JavaTestKit(getSystem());
1639                 terminateWatcher.watch(mockNewReplicaShardActor);
1640
1641                 shardManager.tell(new AddShardReplica("astronauts"), getRef());
1642
1643                 AddServer addServerMsg = mockShardLeaderKit.expectMsgClass(AddServer.class);
1644                 assertEquals("AddServer serverId", "member-1-shard-astronauts-" + shardMrgIDSuffix,
1645                         addServerMsg.getNewServerId());
1646                 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.TIMEOUT, null));
1647
1648                 Failure failure = expectMsgClass(duration("5 seconds"), Failure.class);
1649                 assertEquals("Failure cause", TimeoutException.class, failure.cause().getClass());
1650
1651                 shardManager.tell(new FindLocalShard("astronauts", false), getRef());
1652                 expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
1653
1654                 terminateWatcher.expectTerminated(mockNewReplicaShardActor);
1655
1656                 shardManager.tell(new AddShardReplica("astronauts"), getRef());
1657                 mockShardLeaderKit.expectMsgClass(AddServer.class);
1658                 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.NO_LEADER, null));
1659                 failure = expectMsgClass(duration("5 seconds"), Failure.class);
1660                 assertEquals("Failure cause", NoShardLeaderException.class, failure.cause().getClass());
1661             }
1662         };
1663
1664         LOG.info("testAddShardReplicaWithAddServerReplyFailure ending");
1665     }
1666
1667     @Test
1668     public void testAddShardReplicaWithAlreadyInProgress() throws Exception {
1669         testServerChangeWhenAlreadyInProgress("astronauts", new AddShardReplica("astronauts"),
1670                 AddServer.class, new AddShardReplica("astronauts"));
1671     }
1672
1673     @Test
1674     public void testAddShardReplicaWithFindPrimaryTimeout() throws Exception {
1675         LOG.info("testAddShardReplicaWithFindPrimaryTimeout starting");
1676         datastoreContextBuilder.shardInitializationTimeout(100, TimeUnit.MILLISECONDS);
1677         new JavaTestKit(getSystem()) {
1678             {
1679                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1680                         .put("astronauts", Arrays.asList("member-2")).build());
1681
1682                 final ActorRef newReplicaShardManager = actorFactory
1683                         .createActor(newTestShardMgrBuilder(mockConfig).shardActor(mockShardActor).props()
1684                                 .withDispatcher(Dispatchers.DefaultDispatcherId()), shardMgrID);
1685
1686                 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1687                 MockClusterWrapper.sendMemberUp(newReplicaShardManager, "member-2",
1688                         AddressFromURIString.parse("akka://non-existent@127.0.0.1:5").toString());
1689
1690                 newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
1691                 Status.Failure resp = expectMsgClass(duration("5 seconds"), Status.Failure.class);
1692                 assertEquals("Failure obtained", true, resp.cause() instanceof RuntimeException);
1693             }
1694         };
1695
1696         LOG.info("testAddShardReplicaWithFindPrimaryTimeout ending");
1697     }
1698
1699     @Test
1700     public void testRemoveShardReplicaForNonExistentShard() throws Exception {
1701         new JavaTestKit(getSystem()) {
1702             {
1703                 ActorRef shardManager = actorFactory
1704                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1705                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1706
1707                 shardManager.tell(new RemoveShardReplica("model-inventory", MEMBER_1), getRef());
1708                 Status.Failure resp = expectMsgClass(duration("10 seconds"), Status.Failure.class);
1709                 assertEquals("Failure obtained", true, resp.cause() instanceof PrimaryNotFoundException);
1710             }
1711         };
1712     }
1713
1714     @Test
1715     /**
1716      * Primary is Local
1717      */
1718     public void testRemoveShardReplicaLocal() throws Exception {
1719         new JavaTestKit(getSystem()) {
1720             {
1721                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1722
1723                 final ActorRef respondActor = actorFactory.createActor(Props.create(MockRespondActor.class,
1724                         RemoveServer.class, new RemoveServerReply(ServerChangeStatus.OK, null)), memberId);
1725
1726                 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
1727
1728                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1729                 shardManager.tell(new ActorInitialized(), respondActor);
1730                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
1731                         DataStoreVersions.CURRENT_VERSION), getRef());
1732                 shardManager.tell(
1733                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
1734                         respondActor);
1735
1736                 shardManager.tell(new RemoveShardReplica(Shard.DEFAULT_NAME, MEMBER_1), getRef());
1737                 final RemoveServer removeServer = MessageCollectorActor.expectFirstMatching(respondActor,
1738                         RemoveServer.class);
1739                 assertEquals(ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString(),
1740                         removeServer.getServerId());
1741                 expectMsgClass(duration("5 seconds"), Success.class);
1742             }
1743         };
1744     }
1745
1746     @Test
1747     public void testRemoveShardReplicaRemote() throws Exception {
1748         MockConfiguration mockConfig = new MockConfiguration(
1749                 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
1750                         .put("astronauts", Arrays.asList("member-1")).build());
1751
1752         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
1753
1754         // Create an ActorSystem ShardManager actor for member-1.
1755         final ActorSystem system1 = newActorSystem("Member1");
1756         Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1757         ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
1758
1759         final TestActorRef<TestShardManager> newReplicaShardManager = TestActorRef.create(system1,
1760                 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockDefaultShardActor).cluster(
1761                         new ClusterWrapperImpl(system1)).props().withDispatcher(Dispatchers.DefaultDispatcherId()),
1762                 shardManagerID);
1763
1764         // Create an ActorSystem ShardManager actor for member-2.
1765         final ActorSystem system2 = newActorSystem("Member2");
1766         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1767
1768         String name = ShardIdentifier.create("default", MEMBER_2, shardMrgIDSuffix).toString();
1769         String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
1770         final TestActorRef<MockRespondActor> mockShardLeaderActor =
1771                 TestActorRef.create(system2, Props.create(MockRespondActor.class, RemoveServer.class,
1772                         new RemoveServerReply(ServerChangeStatus.OK, memberId2)), name);
1773
1774         LOG.error("Mock Shard Leader Actor : {}", mockShardLeaderActor);
1775
1776         final TestActorRef<TestShardManager> leaderShardManager = TestActorRef.create(system2,
1777                 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockShardLeaderActor).cluster(
1778                         new ClusterWrapperImpl(system2)).props().withDispatcher(Dispatchers.DefaultDispatcherId()),
1779                 shardManagerID);
1780
1781         // Because mockShardLeaderActor is created at the top level of the actor system it has an address like so,
1782         //    akka://cluster-test@127.0.0.1:2559/user/member-2-shard-default-config1
1783         // However when a shard manager has a local shard which is a follower and a leader that is remote it will
1784         // try to compute an address for the remote shard leader using the ShardPeerAddressResolver. This address will
1785         // look like so,
1786         //    akka://cluster-test@127.0.0.1:2559/user/shardmanager-config1/member-2-shard-default-config1
1787         // In this specific case if we did a FindPrimary for shard default from member-1 we would come up
1788         // with the address of an actor which does not exist, therefore any message sent to that actor would go to
1789         // dead letters.
1790         // To work around this problem we create a ForwardingActor with the right address and pass to it the
1791         // mockShardLeaderActor. The ForwardingActor simply forwards all messages to the mockShardLeaderActor and every
1792         // thing works as expected
1793         final ActorRef actorRef = leaderShardManager.underlyingActor().context()
1794                 .actorOf(Props.create(ForwardingActor.class, mockShardLeaderActor),
1795                         "member-2-shard-default-" + shardMrgIDSuffix);
1796
1797         LOG.error("Forwarding actor : {}", actorRef);
1798
1799         new JavaTestKit(system1) {
1800             {
1801                 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1802                 leaderShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1803
1804                 leaderShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1805                 newReplicaShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1806
1807                 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
1808                 leaderShardManager.tell(
1809                         new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
1810                         mockShardLeaderActor);
1811                 leaderShardManager.tell(
1812                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
1813                         mockShardLeaderActor);
1814
1815                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
1816                 newReplicaShardManager.tell(
1817                         new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class), leaderVersion),
1818                         mockShardActor);
1819                 newReplicaShardManager.tell(
1820                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
1821                         mockShardActor);
1822
1823                 newReplicaShardManager.underlyingActor().waitForMemberUp();
1824                 leaderShardManager.underlyingActor().waitForMemberUp();
1825
1826                 // construct a mock response message
1827                 newReplicaShardManager.tell(new RemoveShardReplica("default", MEMBER_1), getRef());
1828                 RemoveServer removeServer = MessageCollectorActor.expectFirstMatching(mockShardLeaderActor,
1829                         RemoveServer.class);
1830                 String removeServerId = ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString();
1831                 assertEquals("RemoveServer serverId", removeServerId, removeServer.getServerId());
1832                 expectMsgClass(duration("5 seconds"), Status.Success.class);
1833             }
1834         };
1835     }
1836
1837     @Test
1838     public void testRemoveShardReplicaWhenAnotherRemoveShardReplicaAlreadyInProgress() throws Exception {
1839         testServerChangeWhenAlreadyInProgress("astronauts", new RemoveShardReplica("astronauts", MEMBER_2),
1840                 RemoveServer.class, new RemoveShardReplica("astronauts", MEMBER_3));
1841     }
1842
1843     @Test
1844     public void testRemoveShardReplicaWhenAddShardReplicaAlreadyInProgress() throws Exception {
1845         testServerChangeWhenAlreadyInProgress("astronauts", new AddShardReplica("astronauts"),
1846                 AddServer.class, new RemoveShardReplica("astronauts", MEMBER_2));
1847     }
1848
1849
1850     public void testServerChangeWhenAlreadyInProgress(final String shardName, final Object firstServerChange,
1851                                                       final Class<?> firstForwardedServerChangeClass,
1852                                                       final Object secondServerChange) throws Exception {
1853         new JavaTestKit(getSystem()) {
1854             {
1855                 JavaTestKit mockShardLeaderKit = new JavaTestKit(getSystem());
1856                 final JavaTestKit secondRequestKit = new JavaTestKit(getSystem());
1857
1858                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1859                         .put(shardName, Arrays.asList("member-2")).build());
1860
1861                 final TestActorRef<TestShardManager> shardManager = TestActorRef.create(getSystem(),
1862                         newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockShardActor)
1863                                 .cluster(new MockClusterWrapper()).props()
1864                                 .withDispatcher(Dispatchers.DefaultDispatcherId()),
1865                         shardMgrID);
1866
1867                 shardManager.underlyingActor()
1868                         .setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1869
1870                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1871
1872                 shardManager.tell(firstServerChange, getRef());
1873
1874                 mockShardLeaderKit.expectMsgClass(firstForwardedServerChangeClass);
1875
1876                 shardManager.tell(secondServerChange, secondRequestKit.getRef());
1877
1878                 secondRequestKit.expectMsgClass(duration("5 seconds"), Failure.class);
1879             }
1880         };
1881     }
1882
1883     @Test
1884     public void testServerRemovedShardActorNotRunning() throws Exception {
1885         LOG.info("testServerRemovedShardActorNotRunning starting");
1886         new JavaTestKit(getSystem()) {
1887             {
1888                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1889                         .put("default", Arrays.asList("member-1", "member-2"))
1890                         .put("astronauts", Arrays.asList("member-2"))
1891                         .put("people", Arrays.asList("member-1", "member-2")).build());
1892
1893                 TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(
1894                         newShardMgrProps(mockConfig).withDispatcher(Dispatchers.DefaultDispatcherId()));
1895
1896                 shardManager.underlyingActor().waitForRecoveryComplete();
1897                 shardManager.tell(new FindLocalShard("people", false), getRef());
1898                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1899
1900                 shardManager.tell(new FindLocalShard("default", false), getRef());
1901                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1902
1903                 // Removed the default shard replica from member-1
1904                 ShardIdentifier.Builder builder = new ShardIdentifier.Builder();
1905                 ShardIdentifier shardId = builder.shardName("default").memberName(MEMBER_1).type(shardMrgIDSuffix)
1906                         .build();
1907                 shardManager.tell(new ServerRemoved(shardId.toString()), getRef());
1908
1909                 shardManager.underlyingActor().verifySnapshotPersisted(Sets.newHashSet("people"));
1910             }
1911         };
1912
1913         LOG.info("testServerRemovedShardActorNotRunning ending");
1914     }
1915
1916     @Test
1917     public void testServerRemovedShardActorRunning() throws Exception {
1918         LOG.info("testServerRemovedShardActorRunning starting");
1919         new JavaTestKit(getSystem()) {
1920             {
1921                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1922                         .put("default", Arrays.asList("member-1", "member-2"))
1923                         .put("astronauts", Arrays.asList("member-2"))
1924                         .put("people", Arrays.asList("member-1", "member-2")).build());
1925
1926                 String shardId = ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString();
1927                 ActorRef shard = actorFactory.createActor(MessageCollectorActor.props(), shardId);
1928
1929                 TestActorRef<TestShardManager> shardManager = actorFactory
1930                         .createTestActor(newTestShardMgrBuilder(mockConfig).addShardActor("default", shard).props()
1931                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1932
1933                 shardManager.underlyingActor().waitForRecoveryComplete();
1934
1935                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1936                 shardManager.tell(new ActorInitialized(), shard);
1937
1938                 waitForShardInitialized(shardManager, "people", this);
1939                 waitForShardInitialized(shardManager, "default", this);
1940
1941                 // Removed the default shard replica from member-1
1942                 shardManager.tell(new ServerRemoved(shardId), getRef());
1943
1944                 shardManager.underlyingActor().verifySnapshotPersisted(Sets.newHashSet("people"));
1945
1946                 MessageCollectorActor.expectFirstMatching(shard, Shutdown.class);
1947             }
1948         };
1949
1950         LOG.info("testServerRemovedShardActorRunning ending");
1951     }
1952
1953     @Test
1954     public void testShardPersistenceWithRestoredData() throws Exception {
1955         LOG.info("testShardPersistenceWithRestoredData starting");
1956         new JavaTestKit(getSystem()) {
1957             {
1958                 MockConfiguration mockConfig =
1959                     new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1960                             .put("default", Arrays.asList("member-1", "member-2"))
1961                             .put("astronauts", Arrays.asList("member-2"))
1962                             .put("people", Arrays.asList("member-1", "member-2")).build());
1963                 String[] restoredShards = {"default", "astronauts"};
1964                 ShardManagerSnapshot snapshot =
1965                         new ShardManagerSnapshot(Arrays.asList(restoredShards), Collections.emptyMap());
1966                 InMemorySnapshotStore.addSnapshot("shard-manager-" + shardMrgIDSuffix, snapshot);
1967
1968                 // create shardManager to come up with restored data
1969                 TestActorRef<TestShardManager> newRestoredShardManager = actorFactory.createTestActor(
1970                         newShardMgrProps(mockConfig).withDispatcher(Dispatchers.DefaultDispatcherId()));
1971
1972                 newRestoredShardManager.underlyingActor().waitForRecoveryComplete();
1973
1974                 newRestoredShardManager.tell(new FindLocalShard("people", false), getRef());
1975                 LocalShardNotFound notFound = expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
1976                 assertEquals("for uninitialized shard", "people", notFound.getShardName());
1977
1978                 // Verify a local shard is created for the restored shards,
1979                 // although we expect a NotInitializedException for the shards
1980                 // as the actor initialization
1981                 // message is not sent for them
1982                 newRestoredShardManager.tell(new FindLocalShard("default", false), getRef());
1983                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1984
1985                 newRestoredShardManager.tell(new FindLocalShard("astronauts", false), getRef());
1986                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1987             }
1988         };
1989
1990         LOG.info("testShardPersistenceWithRestoredData ending");
1991     }
1992
1993     @Test
1994     public void testShutDown() throws Exception {
1995         LOG.info("testShutDown starting");
1996         new JavaTestKit(getSystem()) {
1997             {
1998                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1999                         .put("shard1", Arrays.asList("member-1")).put("shard2", Arrays.asList("member-1")).build());
2000
2001                 String shardId1 = ShardIdentifier.create("shard1", MEMBER_1, shardMrgIDSuffix).toString();
2002                 ActorRef shard1 = actorFactory.createActor(MessageCollectorActor.props(), shardId1);
2003
2004                 String shardId2 = ShardIdentifier.create("shard2", MEMBER_1, shardMrgIDSuffix).toString();
2005                 ActorRef shard2 = actorFactory.createActor(MessageCollectorActor.props(), shardId2);
2006
2007                 ActorRef shardManager = actorFactory.createActor(newTestShardMgrBuilder(mockConfig)
2008                         .addShardActor("shard1", shard1).addShardActor("shard2", shard2).props());
2009
2010                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
2011                 shardManager.tell(new ActorInitialized(), shard1);
2012                 shardManager.tell(new ActorInitialized(), shard2);
2013
2014                 FiniteDuration duration = FiniteDuration.create(5, TimeUnit.SECONDS);
2015                 Future<Boolean> stopFuture = Patterns.gracefulStop(shardManager, duration, Shutdown.INSTANCE);
2016
2017                 MessageCollectorActor.expectFirstMatching(shard1, Shutdown.class);
2018                 MessageCollectorActor.expectFirstMatching(shard2, Shutdown.class);
2019
2020                 try {
2021                     Await.ready(stopFuture, FiniteDuration.create(500, TimeUnit.MILLISECONDS));
2022                     fail("ShardManager actor stopped without waiting for the Shards to be stopped");
2023                 } catch (TimeoutException e) {
2024                     // expected
2025                 }
2026
2027                 actorFactory.killActor(shard1, this);
2028                 actorFactory.killActor(shard2, this);
2029
2030                 Boolean stopped = Await.result(stopFuture, duration);
2031                 assertEquals("Stopped", Boolean.TRUE, stopped);
2032             }
2033         };
2034
2035         LOG.info("testShutDown ending");
2036     }
2037
2038     @Test
2039     public void testChangeServersVotingStatus() throws Exception {
2040         new JavaTestKit(getSystem()) {
2041             {
2042                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
2043
2044                 ActorRef respondActor = actorFactory
2045                         .createActor(Props.create(MockRespondActor.class, ChangeServersVotingStatus.class,
2046                                 new ServerChangeReply(ServerChangeStatus.OK, null)), memberId);
2047
2048                 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
2049
2050                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
2051                 shardManager.tell(new ActorInitialized(), respondActor);
2052                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
2053                         DataStoreVersions.CURRENT_VERSION), getRef());
2054                 shardManager.tell(
2055                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
2056                         respondActor);
2057
2058                 shardManager.tell(
2059                         new ChangeShardMembersVotingStatus("default", ImmutableMap.of("member-2", Boolean.TRUE)),
2060                         getRef());
2061
2062                 ChangeServersVotingStatus actualChangeStatusMsg = MessageCollectorActor
2063                         .expectFirstMatching(respondActor, ChangeServersVotingStatus.class);
2064                 assertEquals("ChangeServersVotingStatus map", actualChangeStatusMsg.getServerVotingStatusMap(),
2065                         ImmutableMap.of(ShardIdentifier
2066                                 .create("default", MemberName.forName("member-2"), shardMrgIDSuffix).toString(),
2067                                 Boolean.TRUE));
2068
2069                 expectMsgClass(duration("5 seconds"), Success.class);
2070             }
2071         };
2072     }
2073
2074     @Test
2075     public void testChangeServersVotingStatusWithNoLeader() throws Exception {
2076         new JavaTestKit(getSystem()) {
2077             {
2078                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
2079
2080                 ActorRef respondActor = actorFactory
2081                         .createActor(Props.create(MockRespondActor.class, ChangeServersVotingStatus.class,
2082                                 new ServerChangeReply(ServerChangeStatus.NO_LEADER, null)), memberId);
2083
2084                 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
2085
2086                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
2087                 shardManager.tell(new ActorInitialized(), respondActor);
2088                 shardManager.tell(new RoleChangeNotification(memberId, null, RaftState.Follower.name()), respondActor);
2089
2090                 shardManager.tell(
2091                         new ChangeShardMembersVotingStatus("default", ImmutableMap.of("member-2", Boolean.TRUE)),
2092                         getRef());
2093
2094                 MessageCollectorActor.expectFirstMatching(respondActor, ChangeServersVotingStatus.class);
2095
2096                 Status.Failure resp = expectMsgClass(duration("5 seconds"), Status.Failure.class);
2097                 assertEquals("Failure resposnse", true, resp.cause() instanceof NoShardLeaderException);
2098             }
2099         };
2100     }
2101
2102     public static class TestShardManager extends ShardManager {
2103         private final CountDownLatch recoveryComplete = new CountDownLatch(1);
2104         private final CountDownLatch snapshotPersist = new CountDownLatch(1);
2105         private ShardManagerSnapshot snapshot;
2106         private final Map<String, ActorRef> shardActors;
2107         private final ActorRef shardActor;
2108         private CountDownLatch findPrimaryMessageReceived = new CountDownLatch(1);
2109         private CountDownLatch memberUpReceived = new CountDownLatch(1);
2110         private CountDownLatch memberRemovedReceived = new CountDownLatch(1);
2111         private CountDownLatch memberUnreachableReceived = new CountDownLatch(1);
2112         private CountDownLatch memberReachableReceived = new CountDownLatch(1);
2113         private volatile MessageInterceptor messageInterceptor;
2114
2115         private TestShardManager(Builder builder) {
2116             super(builder);
2117             shardActor = builder.shardActor;
2118             shardActors = builder.shardActors;
2119         }
2120
2121         @Override
2122         protected void handleRecover(Object message) throws Exception {
2123             try {
2124                 super.handleRecover(message);
2125             } finally {
2126                 if (message instanceof RecoveryCompleted) {
2127                     recoveryComplete.countDown();
2128                 }
2129             }
2130         }
2131
2132         private void countDownIfOther(final Member member, CountDownLatch latch) {
2133             if (!getCluster().getCurrentMemberName().equals(memberToName(member))) {
2134                 latch.countDown();
2135             }
2136         }
2137
2138         @Override
2139         public void handleCommand(Object message) throws Exception {
2140             try {
2141                 if (messageInterceptor != null && messageInterceptor.canIntercept(message)) {
2142                     getSender().tell(messageInterceptor.apply(message), getSelf());
2143                 } else {
2144                     super.handleCommand(message);
2145                 }
2146             } finally {
2147                 if (message instanceof FindPrimary) {
2148                     findPrimaryMessageReceived.countDown();
2149                 } else if (message instanceof ClusterEvent.MemberUp) {
2150                     countDownIfOther(((ClusterEvent.MemberUp) message).member(), memberUpReceived);
2151                 } else if (message instanceof ClusterEvent.MemberRemoved) {
2152                     countDownIfOther(((ClusterEvent.MemberRemoved) message).member(), memberRemovedReceived);
2153                 } else if (message instanceof ClusterEvent.UnreachableMember) {
2154                     countDownIfOther(((ClusterEvent.UnreachableMember) message).member(), memberUnreachableReceived);
2155                 } else if (message instanceof ClusterEvent.ReachableMember) {
2156                     countDownIfOther(((ClusterEvent.ReachableMember) message).member(), memberReachableReceived);
2157                 }
2158             }
2159         }
2160
2161         void setMessageInterceptor(MessageInterceptor messageInterceptor) {
2162             this.messageInterceptor = messageInterceptor;
2163         }
2164
2165         void waitForRecoveryComplete() {
2166             assertEquals("Recovery complete", true,
2167                     Uninterruptibles.awaitUninterruptibly(recoveryComplete, 5, TimeUnit.SECONDS));
2168         }
2169
2170         public void waitForMemberUp() {
2171             assertEquals("MemberUp received", true,
2172                     Uninterruptibles.awaitUninterruptibly(memberUpReceived, 5, TimeUnit.SECONDS));
2173             memberUpReceived = new CountDownLatch(1);
2174         }
2175
2176         void waitForMemberRemoved() {
2177             assertEquals("MemberRemoved received", true,
2178                     Uninterruptibles.awaitUninterruptibly(memberRemovedReceived, 5, TimeUnit.SECONDS));
2179             memberRemovedReceived = new CountDownLatch(1);
2180         }
2181
2182         void waitForUnreachableMember() {
2183             assertEquals("UnreachableMember received", true,
2184                 Uninterruptibles.awaitUninterruptibly(memberUnreachableReceived, 5, TimeUnit.SECONDS
2185                 ));
2186             memberUnreachableReceived = new CountDownLatch(1);
2187         }
2188
2189         void waitForReachableMember() {
2190             assertEquals("ReachableMember received", true,
2191                 Uninterruptibles.awaitUninterruptibly(memberReachableReceived, 5, TimeUnit.SECONDS));
2192             memberReachableReceived = new CountDownLatch(1);
2193         }
2194
2195         void verifyFindPrimary() {
2196             assertEquals("FindPrimary received", true,
2197                     Uninterruptibles.awaitUninterruptibly(findPrimaryMessageReceived, 5, TimeUnit.SECONDS));
2198             findPrimaryMessageReceived = new CountDownLatch(1);
2199         }
2200
2201         public static Builder builder(DatastoreContext.Builder datastoreContextBuilder) {
2202             return new Builder(datastoreContextBuilder);
2203         }
2204
2205         public static class Builder extends AbstractGenericCreator<Builder, TestShardManager> {
2206             private ActorRef shardActor;
2207             private final Map<String, ActorRef> shardActors = new HashMap<>();
2208
2209             Builder(DatastoreContext.Builder datastoreContextBuilder) {
2210                 super(TestShardManager.class);
2211                 datastoreContextFactory(newDatastoreContextFactory(datastoreContextBuilder.build()));
2212             }
2213
2214             Builder shardActor(ActorRef newShardActor) {
2215                 this.shardActor = newShardActor;
2216                 return this;
2217             }
2218
2219             Builder addShardActor(String shardName, ActorRef actorRef) {
2220                 shardActors.put(shardName, actorRef);
2221                 return this;
2222             }
2223         }
2224
2225         @Override
2226         public void saveSnapshot(Object obj) {
2227             snapshot = (ShardManagerSnapshot) obj;
2228             snapshotPersist.countDown();
2229             super.saveSnapshot(obj);
2230         }
2231
2232         void verifySnapshotPersisted(Set<String> shardList) {
2233             assertEquals("saveSnapshot invoked", true,
2234                     Uninterruptibles.awaitUninterruptibly(snapshotPersist, 5, TimeUnit.SECONDS));
2235             assertEquals("Shard Persisted", shardList, Sets.newHashSet(snapshot.getShardList()));
2236         }
2237
2238         @Override
2239         protected ActorRef newShardActor(SchemaContext schemaContext, ShardInformation info) {
2240             if (shardActors.get(info.getShardName()) != null) {
2241                 return shardActors.get(info.getShardName());
2242             }
2243
2244             if (shardActor != null) {
2245                 return shardActor;
2246             }
2247
2248             return super.newShardActor(schemaContext, info);
2249         }
2250     }
2251
2252     private abstract static class AbstractGenericCreator<T extends AbstractGenericCreator<T, ?>, C extends ShardManager>
2253                                                      extends AbstractShardManagerCreator<T> {
2254         private final Class<C> shardManagerClass;
2255
2256         AbstractGenericCreator(Class<C> shardManagerClass) {
2257             this.shardManagerClass = shardManagerClass;
2258             cluster(new MockClusterWrapper()).configuration(new MockConfiguration()).waitTillReadyCountDownLatch(ready)
2259                     .primaryShardInfoCache(new PrimaryShardInfoFutureCache());
2260         }
2261
2262         @Override
2263         public Props props() {
2264             verify();
2265             return Props.create(shardManagerClass, this);
2266         }
2267     }
2268
2269     private static class GenericCreator<C extends ShardManager> extends AbstractGenericCreator<GenericCreator<C>, C> {
2270         GenericCreator(Class<C> shardManagerClass) {
2271             super(shardManagerClass);
2272         }
2273     }
2274
2275     private static class DelegatingShardManagerCreator implements Creator<ShardManager> {
2276         private static final long serialVersionUID = 1L;
2277         private final Creator<ShardManager> delegate;
2278
2279         DelegatingShardManagerCreator(Creator<ShardManager> delegate) {
2280             this.delegate = delegate;
2281         }
2282
2283         @Override
2284         public ShardManager create() throws Exception {
2285             return delegate.create();
2286         }
2287     }
2288
2289     interface MessageInterceptor extends Function<Object, Object> {
2290         boolean canIntercept(Object message);
2291     }
2292
2293     private static MessageInterceptor newFindPrimaryInterceptor(final ActorRef primaryActor) {
2294         return new MessageInterceptor() {
2295             @Override
2296             public Object apply(Object message) {
2297                 return new RemotePrimaryShardFound(Serialization.serializedActorPath(primaryActor), (short) 1);
2298             }
2299
2300             @Override
2301             public boolean canIntercept(Object message) {
2302                 return message instanceof FindPrimary;
2303             }
2304         };
2305     }
2306
2307     private static class MockRespondActor extends MessageCollectorActor {
2308         static final String CLEAR_RESPONSE = "clear-response";
2309
2310         private Object responseMsg;
2311         private final Class<?> requestClass;
2312
2313         @SuppressWarnings("unused")
2314         MockRespondActor(Class<?> requestClass, Object responseMsg) {
2315             this.requestClass = requestClass;
2316             this.responseMsg = responseMsg;
2317         }
2318
2319         @Override
2320         public void onReceive(Object message) throws Exception {
2321             if (message.equals(CLEAR_RESPONSE)) {
2322                 responseMsg = null;
2323             } else {
2324                 super.onReceive(message);
2325                 if (message.getClass().equals(requestClass) && responseMsg != null) {
2326                     getSender().tell(responseMsg, getSelf());
2327                 }
2328             }
2329         }
2330     }
2331 }

©2013 OpenDaylight, A Linux Foundation Collaborative Project. All Rights Reserved.
OpenDaylight is a registered trademark of The OpenDaylight Project, Inc.
Linux Foundation and OpenDaylight are registered trademarks of the Linux Foundation.
Linux is a registered trademark of Linus Torvalds.