2 * Copyright (c) 2014, 2015 Cisco Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
9 package org.opendaylight.controller.cluster.datastore.shardmanager;
11 import static org.junit.Assert.assertEquals;
12 import static org.junit.Assert.assertNotNull;
13 import static org.junit.Assert.assertNull;
14 import static org.junit.Assert.assertSame;
15 import static org.junit.Assert.assertTrue;
16 import static org.junit.Assert.fail;
17 import static org.mockito.Mockito.mock;
18 import static org.mockito.Mockito.never;
19 import static org.mockito.Mockito.times;
20 import static org.mockito.Mockito.verify;
21 import akka.actor.ActorRef;
22 import akka.actor.ActorSystem;
23 import akka.actor.AddressFromURIString;
24 import akka.actor.Props;
25 import akka.actor.Status;
26 import akka.actor.Status.Failure;
27 import akka.actor.Status.Success;
28 import akka.cluster.Cluster;
29 import akka.cluster.ClusterEvent;
30 import akka.cluster.Member;
31 import akka.dispatch.Dispatchers;
32 import akka.japi.Creator;
33 import akka.pattern.Patterns;
34 import akka.persistence.RecoveryCompleted;
35 import akka.serialization.Serialization;
36 import akka.testkit.JavaTestKit;
37 import akka.testkit.TestActorRef;
38 import akka.util.Timeout;
39 import com.google.common.base.Function;
40 import com.google.common.base.Stopwatch;
41 import com.google.common.collect.ImmutableMap;
42 import com.google.common.collect.ImmutableSet;
43 import com.google.common.collect.Lists;
44 import com.google.common.collect.Sets;
45 import com.google.common.util.concurrent.Uninterruptibles;
46 import com.typesafe.config.ConfigFactory;
48 import java.util.AbstractMap;
49 import java.util.ArrayList;
50 import java.util.Arrays;
51 import java.util.Collection;
52 import java.util.Collections;
53 import java.util.HashMap;
54 import java.util.List;
56 import java.util.Map.Entry;
58 import java.util.concurrent.CountDownLatch;
59 import java.util.concurrent.TimeUnit;
60 import java.util.concurrent.TimeoutException;
61 import java.util.stream.Collectors;
62 import org.apache.commons.lang3.SerializationUtils;
63 import org.junit.After;
64 import org.junit.Before;
65 import org.junit.Test;
66 import org.mockito.Mock;
67 import org.mockito.Mockito;
68 import org.mockito.MockitoAnnotations;
69 import org.opendaylight.controller.cluster.access.concepts.MemberName;
70 import org.opendaylight.controller.cluster.datastore.AbstractActorTest;
71 import org.opendaylight.controller.cluster.datastore.ClusterWrapperImpl;
72 import org.opendaylight.controller.cluster.datastore.DataStoreVersions;
73 import org.opendaylight.controller.cluster.datastore.DatastoreContext;
74 import org.opendaylight.controller.cluster.datastore.DatastoreContextFactory;
75 import org.opendaylight.controller.cluster.datastore.Shard;
76 import org.opendaylight.controller.cluster.datastore.ShardManager.SchemaContextModules;
77 import org.opendaylight.controller.cluster.datastore.config.Configuration;
78 import org.opendaylight.controller.cluster.datastore.config.ConfigurationImpl;
79 import org.opendaylight.controller.cluster.datastore.config.EmptyModuleShardConfigProvider;
80 import org.opendaylight.controller.cluster.datastore.config.ModuleShardConfiguration;
81 import org.opendaylight.controller.cluster.datastore.exceptions.AlreadyExistsException;
82 import org.opendaylight.controller.cluster.datastore.exceptions.NoShardLeaderException;
83 import org.opendaylight.controller.cluster.datastore.exceptions.NotInitializedException;
84 import org.opendaylight.controller.cluster.datastore.exceptions.PrimaryNotFoundException;
85 import org.opendaylight.controller.cluster.datastore.identifiers.ShardIdentifier;
86 import org.opendaylight.controller.cluster.datastore.identifiers.ShardManagerIdentifier;
87 import org.opendaylight.controller.cluster.datastore.messages.ActorInitialized;
88 import org.opendaylight.controller.cluster.datastore.messages.AddShardReplica;
89 import org.opendaylight.controller.cluster.datastore.messages.CreateShard;
90 import org.opendaylight.controller.cluster.datastore.messages.DatastoreSnapshot;
91 import org.opendaylight.controller.cluster.datastore.messages.DatastoreSnapshot.ShardSnapshot;
92 import org.opendaylight.controller.cluster.datastore.messages.FindLocalShard;
93 import org.opendaylight.controller.cluster.datastore.messages.FindPrimary;
94 import org.opendaylight.controller.cluster.datastore.messages.LocalPrimaryShardFound;
95 import org.opendaylight.controller.cluster.datastore.messages.LocalShardFound;
96 import org.opendaylight.controller.cluster.datastore.messages.LocalShardNotFound;
97 import org.opendaylight.controller.cluster.datastore.messages.PeerDown;
98 import org.opendaylight.controller.cluster.datastore.messages.PeerUp;
99 import org.opendaylight.controller.cluster.datastore.messages.PrimaryShardInfo;
100 import org.opendaylight.controller.cluster.datastore.messages.RemotePrimaryShardFound;
101 import org.opendaylight.controller.cluster.datastore.messages.RemoveShardReplica;
102 import org.opendaylight.controller.cluster.datastore.messages.ShardLeaderStateChanged;
103 import org.opendaylight.controller.cluster.datastore.messages.UpdateSchemaContext;
104 import org.opendaylight.controller.cluster.datastore.utils.ForwardingActor;
105 import org.opendaylight.controller.cluster.datastore.utils.MockClusterWrapper;
106 import org.opendaylight.controller.cluster.datastore.utils.MockConfiguration;
107 import org.opendaylight.controller.cluster.datastore.utils.PrimaryShardInfoFutureCache;
108 import org.opendaylight.controller.cluster.notifications.LeaderStateChanged;
109 import org.opendaylight.controller.cluster.notifications.RegisterRoleChangeListener;
110 import org.opendaylight.controller.cluster.notifications.RoleChangeNotification;
111 import org.opendaylight.controller.cluster.raft.RaftState;
112 import org.opendaylight.controller.cluster.raft.TestActorFactory;
113 import org.opendaylight.controller.cluster.raft.base.messages.FollowerInitialSyncUpStatus;
114 import org.opendaylight.controller.cluster.raft.base.messages.SwitchBehavior;
115 import org.opendaylight.controller.cluster.raft.client.messages.GetSnapshot;
116 import org.opendaylight.controller.cluster.raft.client.messages.Shutdown;
117 import org.opendaylight.controller.cluster.raft.messages.AddServer;
118 import org.opendaylight.controller.cluster.raft.messages.AddServerReply;
119 import org.opendaylight.controller.cluster.raft.messages.RemoveServer;
120 import org.opendaylight.controller.cluster.raft.messages.RemoveServerReply;
121 import org.opendaylight.controller.cluster.raft.messages.ServerChangeStatus;
122 import org.opendaylight.controller.cluster.raft.messages.ServerRemoved;
123 import org.opendaylight.controller.cluster.raft.policy.DisableElectionsRaftPolicy;
124 import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal;
125 import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore;
126 import org.opendaylight.controller.cluster.raft.utils.MessageCollectorActor;
127 import org.opendaylight.controller.md.cluster.datastore.model.TestModel;
128 import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTree;
129 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
130 import org.slf4j.Logger;
131 import org.slf4j.LoggerFactory;
132 import scala.concurrent.Await;
133 import scala.concurrent.Future;
134 import scala.concurrent.duration.FiniteDuration;
136 public class ShardManagerTest extends AbstractActorTest {
137 private static final Logger LOG = LoggerFactory.getLogger(ShardManagerTest.class);
138 private static final MemberName MEMBER_1 = MemberName.forName("member-1");
139 private static final MemberName MEMBER_2 = MemberName.forName("member-2");
140 private static final MemberName MEMBER_3 = MemberName.forName("member-3");
142 private static int ID_COUNTER = 1;
144 private final String shardMrgIDSuffix = "config" + ID_COUNTER++;
145 private final String shardMgrID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
148 private static CountDownLatch ready;
150 private static TestActorRef<MessageCollectorActor> mockShardActor;
152 private static ShardIdentifier mockShardName;
154 private final DatastoreContext.Builder datastoreContextBuilder = DatastoreContext.newBuilder().
155 dataStoreName(shardMrgIDSuffix).shardInitializationTimeout(600, TimeUnit.MILLISECONDS)
156 .shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(6);
158 private final Collection<ActorSystem> actorSystems = new ArrayList<>();
160 private final TestActorFactory actorFactory = new TestActorFactory(getSystem());
163 public void setUp() {
164 MockitoAnnotations.initMocks(this);
166 InMemoryJournal.clear();
167 InMemorySnapshotStore.clear();
169 if(mockShardActor == null) {
170 mockShardName = new ShardIdentifier(Shard.DEFAULT_NAME, MEMBER_1, "config");
171 mockShardActor = TestActorRef.create(getSystem(), Props.create(MessageCollectorActor.class),
172 mockShardName.toString());
175 mockShardActor.underlyingActor().clear();
179 public void tearDown() {
180 InMemoryJournal.clear();
181 InMemorySnapshotStore.clear();
183 for(ActorSystem system: actorSystems) {
184 JavaTestKit.shutdownActorSystem(system, null, Boolean.TRUE);
187 actorFactory.close();
190 private ActorSystem newActorSystem(String config) {
191 ActorSystem system = ActorSystem.create("cluster-test", ConfigFactory.load().getConfig(config));
192 actorSystems.add(system);
196 private ActorRef newMockShardActor(ActorSystem system, String shardName, String memberName) {
197 String name = new ShardIdentifier(shardName, MemberName.forName(memberName), "config").toString();
198 if(system == getSystem()) {
199 return actorFactory.createTestActor(Props.create(MessageCollectorActor.class), name);
202 return TestActorRef.create(system, Props.create(MessageCollectorActor.class), name);
205 private Props newShardMgrProps() {
206 return newShardMgrProps(new MockConfiguration());
209 private static DatastoreContextFactory newDatastoreContextFactory(DatastoreContext datastoreContext) {
210 DatastoreContextFactory mockFactory = mock(DatastoreContextFactory.class);
211 Mockito.doReturn(datastoreContext).when(mockFactory).getBaseDatastoreContext();
212 Mockito.doReturn(datastoreContext).when(mockFactory).getShardDatastoreContext(Mockito.anyString());
216 private TestShardManager.Builder newTestShardMgrBuilder() {
217 return TestShardManager.builder(datastoreContextBuilder);
220 private TestShardManager.Builder newTestShardMgrBuilder(Configuration config) {
221 return TestShardManager.builder(datastoreContextBuilder).configuration(config);
224 private Props newShardMgrProps(Configuration config) {
225 return newTestShardMgrBuilder(config).props();
228 private TestShardManager.Builder newTestShardMgrBuilderWithMockShardActor() {
229 return newTestShardMgrBuilderWithMockShardActor(mockShardActor);
232 private TestShardManager.Builder newTestShardMgrBuilderWithMockShardActor(ActorRef shardActor) {
233 return TestShardManager.builder(datastoreContextBuilder).shardActor(shardActor);
237 private Props newPropsShardMgrWithMockShardActor() {
238 return newTestShardMgrBuilderWithMockShardActor().props();
241 private Props newPropsShardMgrWithMockShardActor(ActorRef shardActor) {
242 return newTestShardMgrBuilderWithMockShardActor(shardActor).props();
246 private TestShardManager newTestShardManager() {
247 return newTestShardManager(newShardMgrProps());
250 private TestShardManager newTestShardManager(Props props) {
251 TestActorRef<TestShardManager> shardManagerActor = actorFactory.createTestActor(props);
252 TestShardManager shardManager = shardManagerActor.underlyingActor();
253 shardManager.waitForRecoveryComplete();
257 private static void waitForShardInitialized(ActorRef shardManager, String shardName, JavaTestKit kit) {
258 AssertionError last = null;
259 Stopwatch sw = Stopwatch.createStarted();
260 while(sw.elapsed(TimeUnit.SECONDS) <= 5) {
262 shardManager.tell(new FindLocalShard(shardName, true), kit.getRef());
263 kit.expectMsgClass(LocalShardFound.class);
265 } catch(AssertionError e) {
269 Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
275 private static <T> T expectMsgClassOrFailure(Class<T> msgClass, JavaTestKit kit, String msg) {
276 Object reply = kit.expectMsgAnyClassOf(JavaTestKit.duration("5 sec"), msgClass, Failure.class);
277 if(reply instanceof Failure) {
278 throw new AssertionError(msg + " failed", ((Failure)reply).cause());
285 public void testPerShardDatastoreContext() throws Exception {
286 LOG.info("testPerShardDatastoreContext starting");
287 final DatastoreContextFactory mockFactory = newDatastoreContextFactory(
288 datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
290 Mockito.doReturn(DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).
291 shardElectionTimeoutFactor(6).build()).when(mockFactory).getShardDatastoreContext("default");
293 Mockito.doReturn(DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).
294 shardElectionTimeoutFactor(7).build()).when(mockFactory).getShardDatastoreContext("topology");
296 final MockConfiguration mockConfig = new MockConfiguration() {
298 public Collection<String> getMemberShardNames(MemberName memberName) {
299 return Arrays.asList("default", "topology");
303 public Collection<MemberName> getMembersFromShardName(String shardName) {
304 return members("member-1");
308 final TestActorRef<MessageCollectorActor> defaultShardActor = actorFactory.createTestActor(
309 Props.create(MessageCollectorActor.class), actorFactory.generateActorId("default"));
310 final TestActorRef<MessageCollectorActor> topologyShardActor = actorFactory.createTestActor(
311 Props.create(MessageCollectorActor.class), actorFactory.generateActorId("topology"));
313 final Map<String, Entry<ActorRef, DatastoreContext>> shardInfoMap = Collections.synchronizedMap(
314 new HashMap<String, Entry<ActorRef, DatastoreContext>>());
315 shardInfoMap.put("default", new AbstractMap.SimpleEntry<ActorRef, DatastoreContext>(defaultShardActor, null));
316 shardInfoMap.put("topology", new AbstractMap.SimpleEntry<ActorRef, DatastoreContext>(topologyShardActor, null));
318 final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
319 final CountDownLatch newShardActorLatch = new CountDownLatch(2);
320 class LocalShardManager extends ShardManager {
321 public LocalShardManager(AbstractShardManagerCreator<?> creator) {
326 protected ActorRef newShardActor(SchemaContext schemaContext, ShardInformation info) {
327 Entry<ActorRef, DatastoreContext> entry = shardInfoMap.get(info.getShardName());
330 ref = entry.getKey();
331 entry.setValue(info.getDatastoreContext());
334 newShardActorLatch.countDown();
339 final Creator<ShardManager> creator = new Creator<ShardManager>() {
340 private static final long serialVersionUID = 1L;
342 public ShardManager create() throws Exception {
343 return new LocalShardManager(new GenericCreator<LocalShardManager>(LocalShardManager.class).
344 datastoreContextFactory(mockFactory).primaryShardInfoCache(primaryShardInfoCache).
345 configuration(mockConfig));
349 JavaTestKit kit = new JavaTestKit(getSystem());
351 final ActorRef shardManager = actorFactory.createActor(Props.create(
352 new DelegatingShardManagerCreator(creator)).withDispatcher(Dispatchers.DefaultDispatcherId()));
354 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), kit.getRef());
356 assertEquals("Shard actors created", true, newShardActorLatch.await(5, TimeUnit.SECONDS));
357 assertEquals("getShardElectionTimeoutFactor", 6, shardInfoMap.get("default").getValue().
358 getShardElectionTimeoutFactor());
359 assertEquals("getShardElectionTimeoutFactor", 7, shardInfoMap.get("topology").getValue().
360 getShardElectionTimeoutFactor());
362 DatastoreContextFactory newMockFactory = newDatastoreContextFactory(
363 datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
364 Mockito.doReturn(DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).
365 shardElectionTimeoutFactor(66).build()).when(newMockFactory).getShardDatastoreContext("default");
367 Mockito.doReturn(DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).
368 shardElectionTimeoutFactor(77).build()).when(newMockFactory).getShardDatastoreContext("topology");
370 shardManager.tell(newMockFactory, kit.getRef());
372 DatastoreContext newContext = MessageCollectorActor.expectFirstMatching(defaultShardActor, DatastoreContext.class);
373 assertEquals("getShardElectionTimeoutFactor", 66, newContext.getShardElectionTimeoutFactor());
375 newContext = MessageCollectorActor.expectFirstMatching(topologyShardActor, DatastoreContext.class);
376 assertEquals("getShardElectionTimeoutFactor", 77, newContext.getShardElectionTimeoutFactor());
378 LOG.info("testPerShardDatastoreContext ending");
382 public void testOnReceiveFindPrimaryForNonExistentShard() throws Exception {
383 new JavaTestKit(getSystem()) {{
384 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
386 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
388 shardManager.tell(new FindPrimary("non-existent", false), getRef());
390 expectMsgClass(duration("5 seconds"), PrimaryNotFoundException.class);
395 public void testOnReceiveFindPrimaryForLocalLeaderShard() throws Exception {
396 LOG.info("testOnReceiveFindPrimaryForLocalLeaderShard starting");
397 new JavaTestKit(getSystem()) {{
398 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
400 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
402 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
403 shardManager.tell(new ActorInitialized(), mockShardActor);
405 DataTree mockDataTree = mock(DataTree.class);
406 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
407 DataStoreVersions.CURRENT_VERSION), getRef());
409 MessageCollectorActor.expectFirstMatching(mockShardActor, RegisterRoleChangeListener.class);
410 shardManager.tell((new RoleChangeNotification(memberId, RaftState.Candidate.name(),
411 RaftState.Leader.name())), mockShardActor);
413 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
415 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
416 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
417 primaryFound.getPrimaryPath().contains("member-1-shard-default"));
418 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
421 LOG.info("testOnReceiveFindPrimaryForLocalLeaderShard ending");
425 public void testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp() throws Exception {
426 LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp starting");
427 new JavaTestKit(getSystem()) {{
428 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
430 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
431 shardManager.tell(new ActorInitialized(), mockShardActor);
433 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
434 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
435 shardManager.tell(new RoleChangeNotification(memberId1,
436 RaftState.Candidate.name(), RaftState.Follower.name()), mockShardActor);
437 shardManager.tell(new LeaderStateChanged(memberId1, memberId2, DataStoreVersions.CURRENT_VERSION), mockShardActor);
439 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
441 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
444 LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp ending");
448 public void testOnReceiveFindPrimaryForNonLocalLeaderShard() throws Exception {
449 LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShard starting");
450 new JavaTestKit(getSystem()) {{
451 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
453 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
454 shardManager.tell(new ActorInitialized(), mockShardActor);
456 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
457 MockClusterWrapper.sendMemberUp(shardManager, "member-2", getRef().path().toString());
459 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
460 shardManager.tell(new RoleChangeNotification(memberId1,
461 RaftState.Candidate.name(), RaftState.Follower.name()), mockShardActor);
462 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
463 shardManager.tell(new ShardLeaderStateChanged(memberId1, memberId2, leaderVersion), mockShardActor);
465 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
467 RemotePrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
468 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
469 primaryFound.getPrimaryPath().contains("member-2-shard-default"));
470 assertEquals("getPrimaryVersion", leaderVersion, primaryFound.getPrimaryVersion());
473 LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShard ending");
477 public void testOnReceiveFindPrimaryForUninitializedShard() throws Exception {
478 new JavaTestKit(getSystem()) {{
479 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
481 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
483 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
488 public void testOnReceiveFindPrimaryForInitializedShardWithNoRole() throws Exception {
489 new JavaTestKit(getSystem()) {{
490 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
492 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
493 shardManager.tell(new ActorInitialized(), mockShardActor);
495 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
497 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
502 public void testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId() throws Exception {
503 LOG.info("testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId starting");
504 new JavaTestKit(getSystem()) {{
505 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
507 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
508 shardManager.tell(new ActorInitialized(), mockShardActor);
510 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
511 shardManager.tell(new RoleChangeNotification(memberId,
512 RaftState.Candidate.name(), RaftState.Follower.name()), mockShardActor);
514 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
516 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
518 DataTree mockDataTree = mock(DataTree.class);
519 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
520 DataStoreVersions.CURRENT_VERSION), mockShardActor);
522 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
524 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
525 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
526 primaryFound.getPrimaryPath().contains("member-1-shard-default"));
527 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
530 LOG.info("testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId starting");
534 public void testOnReceiveFindPrimaryWaitForShardLeader() throws Exception {
535 LOG.info("testOnReceiveFindPrimaryWaitForShardLeader starting");
536 datastoreContextBuilder.shardInitializationTimeout(10, TimeUnit.SECONDS);
537 new JavaTestKit(getSystem()) {{
538 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
540 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
542 // We're passing waitUntilInitialized = true to FindPrimary so the response should be
543 // delayed until we send ActorInitialized and RoleChangeNotification.
544 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
546 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
548 shardManager.tell(new ActorInitialized(), mockShardActor);
550 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
552 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
553 shardManager.tell(new RoleChangeNotification(memberId,
554 RaftState.Candidate.name(), RaftState.Leader.name()), mockShardActor);
556 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
558 DataTree mockDataTree = mock(DataTree.class);
559 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
560 DataStoreVersions.CURRENT_VERSION), mockShardActor);
562 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
563 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
564 primaryFound.getPrimaryPath().contains("member-1-shard-default"));
565 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree() );
567 expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
570 LOG.info("testOnReceiveFindPrimaryWaitForShardLeader ending");
574 public void testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard() throws Exception {
575 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard starting");
576 new JavaTestKit(getSystem()) {{
577 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
579 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
581 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
583 expectMsgClass(duration("2 seconds"), NotInitializedException.class);
585 shardManager.tell(new ActorInitialized(), mockShardActor);
587 expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
590 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard ending");
594 public void testOnReceiveFindPrimaryWaitForReadyWithCandidateShard() throws Exception {
595 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithCandidateShard starting");
596 new JavaTestKit(getSystem()) {{
597 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
599 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
600 shardManager.tell(new ActorInitialized(), mockShardActor);
601 shardManager.tell(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix,
602 null, RaftState.Candidate.name()), mockShardActor);
604 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
606 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
609 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithCandidateShard ending");
613 public void testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard() throws Exception {
614 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard starting");
615 new JavaTestKit(getSystem()) {{
616 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
618 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
619 shardManager.tell(new ActorInitialized(), mockShardActor);
620 shardManager.tell(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix,
621 null, RaftState.IsolatedLeader.name()), mockShardActor);
623 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
625 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
628 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard ending");
632 public void testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard() throws Exception {
633 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard starting");
634 new JavaTestKit(getSystem()) {{
635 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
637 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
638 shardManager.tell(new ActorInitialized(), mockShardActor);
640 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
642 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
645 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard ending");
649 public void testOnReceiveFindPrimaryForRemoteShard() throws Exception {
650 LOG.info("testOnReceiveFindPrimaryForRemoteShard starting");
651 String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
653 // Create an ActorSystem ShardManager actor for member-1.
655 final ActorSystem system1 = newActorSystem("Member1");
656 Cluster.get(system1).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
658 final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
659 newTestShardMgrBuilderWithMockShardActor().cluster(
660 new ClusterWrapperImpl(system1)).props().withDispatcher(
661 Dispatchers.DefaultDispatcherId()), shardManagerID);
663 // Create an ActorSystem ShardManager actor for member-2.
665 final ActorSystem system2 = newActorSystem("Member2");
667 Cluster.get(system2).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
669 final ActorRef mockShardActor2 = newMockShardActor(system2, "astronauts", "member-2");
671 MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder().
672 put("default", Arrays.asList("member-1", "member-2")).
673 put("astronauts", Arrays.asList("member-2")).build());
675 final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
676 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
677 new ClusterWrapperImpl(system2)).props().withDispatcher(
678 Dispatchers.DefaultDispatcherId()), shardManagerID);
680 new JavaTestKit(system1) {{
682 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
683 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
685 shardManager2.tell(new ActorInitialized(), mockShardActor2);
687 String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
688 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
689 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2,
690 mock(DataTree.class), leaderVersion), mockShardActor2);
691 shardManager2.tell(new RoleChangeNotification(memberId2,
692 RaftState.Candidate.name(), RaftState.Leader.name()), mockShardActor2);
694 shardManager1.underlyingActor().waitForMemberUp();
695 shardManager1.tell(new FindPrimary("astronauts", false), getRef());
697 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
698 String path = found.getPrimaryPath();
699 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-astronauts-config"));
700 assertEquals("getPrimaryVersion", leaderVersion, found.getPrimaryVersion());
702 shardManager2.underlyingActor().verifyFindPrimary();
704 Cluster.get(system2).down(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
706 shardManager1.underlyingActor().waitForMemberRemoved();
708 shardManager1.tell(new FindPrimary("astronauts", false), getRef());
710 expectMsgClass(duration("5 seconds"), PrimaryNotFoundException.class);
713 LOG.info("testOnReceiveFindPrimaryForRemoteShard ending");
717 public void testShardAvailabilityOnChangeOfMemberReachability() throws Exception {
718 LOG.info("testShardAvailabilityOnChangeOfMemberReachability starting");
719 String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
721 // Create an ActorSystem ShardManager actor for member-1.
723 final ActorSystem system1 = newActorSystem("Member1");
724 Cluster.get(system1).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
726 final ActorRef mockShardActor1 = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
728 final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
729 newTestShardMgrBuilder().shardActor(mockShardActor1).cluster(
730 new ClusterWrapperImpl(system1)).props().withDispatcher(
731 Dispatchers.DefaultDispatcherId()), shardManagerID);
733 // Create an ActorSystem ShardManager actor for member-2.
735 final ActorSystem system2 = newActorSystem("Member2");
737 Cluster.get(system2).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
739 final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
741 MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder().
742 put("default", Arrays.asList("member-1", "member-2")).build());
744 final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
745 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
746 new ClusterWrapperImpl(system2)).props().withDispatcher(
747 Dispatchers.DefaultDispatcherId()), shardManagerID);
749 new JavaTestKit(system1) {{
751 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
752 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
753 shardManager1.tell(new ActorInitialized(), mockShardActor1);
754 shardManager2.tell(new ActorInitialized(), mockShardActor2);
756 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
757 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
758 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2,
759 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION), mockShardActor1);
760 shardManager1.tell(new RoleChangeNotification(memberId1,
761 RaftState.Candidate.name(), RaftState.Follower.name()), mockShardActor1);
762 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class),
763 DataStoreVersions.CURRENT_VERSION),
765 shardManager2.tell(new RoleChangeNotification(memberId2,
766 RaftState.Candidate.name(), RaftState.Leader.name()), mockShardActor2);
767 shardManager1.underlyingActor().waitForMemberUp();
769 shardManager1.tell(new FindPrimary("default", true), getRef());
771 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
772 String path = found.getPrimaryPath();
773 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
775 shardManager1.tell(MockClusterWrapper.
776 createUnreachableMember("member-2", "akka.tcp://cluster-test@127.0.0.1:2558"), getRef());
778 shardManager1.underlyingActor().waitForUnreachableMember();
780 PeerDown peerDown = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
781 assertEquals("getMemberName", MEMBER_2, peerDown.getMemberName());
782 MessageCollectorActor.clearMessages(mockShardActor1);
784 shardManager1.tell(MockClusterWrapper.
785 createMemberRemoved("member-2", "akka.tcp://cluster-test@127.0.0.1:2558"), getRef());
787 MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
789 shardManager1.tell(new FindPrimary("default", true), getRef());
791 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
793 shardManager1.tell(MockClusterWrapper.
794 createReachableMember("member-2", "akka.tcp://cluster-test@127.0.0.1:2558"), getRef());
796 shardManager1.underlyingActor().waitForReachableMember();
798 PeerUp peerUp = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
799 assertEquals("getMemberName", MEMBER_2, peerUp.getMemberName());
800 MessageCollectorActor.clearMessages(mockShardActor1);
802 shardManager1.tell(new FindPrimary("default", true), getRef());
804 RemotePrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
805 String path1 = found1.getPrimaryPath();
806 assertTrue("Unexpected primary path " + path1, path1.contains("member-2-shard-default-config"));
808 shardManager1.tell(MockClusterWrapper.
809 createMemberUp("member-2", "akka.tcp://cluster-test@127.0.0.1:2558"), getRef());
811 MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
813 // Test FindPrimary wait succeeds after reachable member event.
815 shardManager1.tell(MockClusterWrapper.
816 createUnreachableMember("member-2", "akka.tcp://cluster-test@127.0.0.1:2558"), getRef());
817 shardManager1.underlyingActor().waitForUnreachableMember();
819 shardManager1.tell(new FindPrimary("default", true), getRef());
821 shardManager1.tell(MockClusterWrapper.
822 createReachableMember("member-2", "akka.tcp://cluster-test@127.0.0.1:2558"), getRef());
824 RemotePrimaryShardFound found2 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
825 String path2 = found2.getPrimaryPath();
826 assertTrue("Unexpected primary path " + path2, path2.contains("member-2-shard-default-config"));
829 LOG.info("testShardAvailabilityOnChangeOfMemberReachability ending");
833 public void testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange() throws Exception {
834 LOG.info("testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange starting");
835 String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
837 // Create an ActorSystem ShardManager actor for member-1.
839 final ActorSystem system1 = newActorSystem("Member1");
840 Cluster.get(system1).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
842 final ActorRef mockShardActor1 = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
844 final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
845 final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
846 newTestShardMgrBuilder().shardActor(mockShardActor1).cluster(
847 new ClusterWrapperImpl(system1)).primaryShardInfoCache(primaryShardInfoCache).props().
848 withDispatcher(Dispatchers.DefaultDispatcherId()), shardManagerID);
850 // Create an ActorSystem ShardManager actor for member-2.
852 final ActorSystem system2 = newActorSystem("Member2");
854 Cluster.get(system2).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
856 final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
858 MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder().
859 put("default", Arrays.asList("member-1", "member-2")).build());
861 final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
862 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
863 new ClusterWrapperImpl(system2)).props().withDispatcher(
864 Dispatchers.DefaultDispatcherId()), shardManagerID);
866 new JavaTestKit(system1) {{
867 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
868 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
869 shardManager1.tell(new ActorInitialized(), mockShardActor1);
870 shardManager2.tell(new ActorInitialized(), mockShardActor2);
872 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
873 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
874 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2,
875 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION), mockShardActor1);
876 shardManager1.tell(new RoleChangeNotification(memberId1,
877 RaftState.Candidate.name(), RaftState.Follower.name()), mockShardActor1);
878 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class),
879 DataStoreVersions.CURRENT_VERSION),
881 shardManager2.tell(new RoleChangeNotification(memberId2,
882 RaftState.Candidate.name(), RaftState.Leader.name()), mockShardActor2);
883 shardManager1.underlyingActor().waitForMemberUp();
885 shardManager1.tell(new FindPrimary("default", true), getRef());
887 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
888 String path = found.getPrimaryPath();
889 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
891 primaryShardInfoCache.putSuccessful("default", new PrimaryShardInfo(system1.actorSelection(
892 mockShardActor1.path()), DataStoreVersions.CURRENT_VERSION));
894 shardManager1.tell(MockClusterWrapper.
895 createUnreachableMember("member-2", "akka.tcp://cluster-test@127.0.0.1:2558"), getRef());
897 shardManager1.underlyingActor().waitForUnreachableMember();
899 shardManager1.tell(new FindPrimary("default", true), getRef());
901 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
903 assertNull("Expected primaryShardInfoCache entry removed", primaryShardInfoCache.getIfPresent("default"));
905 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId1, mock(DataTree.class),
906 DataStoreVersions.CURRENT_VERSION), mockShardActor1);
907 shardManager1.tell(new RoleChangeNotification(memberId1,
908 RaftState.Follower.name(), RaftState.Leader.name()), mockShardActor1);
910 shardManager1.tell(new FindPrimary("default", true), getRef());
912 LocalPrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
913 String path1 = found1.getPrimaryPath();
914 assertTrue("Unexpected primary path " + path1, path1.contains("member-1-shard-default-config"));
918 LOG.info("testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange ending");
923 public void testOnReceiveFindLocalShardForNonExistentShard() throws Exception {
924 new JavaTestKit(getSystem()) {{
925 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
927 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
929 shardManager.tell(new FindLocalShard("non-existent", false), getRef());
931 LocalShardNotFound notFound = expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
933 assertEquals("getShardName", "non-existent", notFound.getShardName());
938 public void testOnReceiveFindLocalShardForExistentShard() throws Exception {
939 new JavaTestKit(getSystem()) {{
940 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
942 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
943 shardManager.tell(new ActorInitialized(), mockShardActor);
945 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
947 LocalShardFound found = expectMsgClass(duration("5 seconds"), LocalShardFound.class);
949 assertTrue("Found path contains " + found.getPath().path().toString(),
950 found.getPath().path().toString().contains("member-1-shard-default-config"));
955 public void testOnReceiveFindLocalShardForNotInitializedShard() throws Exception {
956 new JavaTestKit(getSystem()) {{
957 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
959 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
961 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
966 public void testOnReceiveFindLocalShardWaitForShardInitialized() throws Exception {
967 LOG.info("testOnReceiveFindLocalShardWaitForShardInitialized starting");
968 new JavaTestKit(getSystem()) {{
969 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
971 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
973 // We're passing waitUntilInitialized = true to FindLocalShard so the response should be
974 // delayed until we send ActorInitialized.
975 Future<Object> future = Patterns.ask(shardManager, new FindLocalShard(Shard.DEFAULT_NAME, true),
976 new Timeout(5, TimeUnit.SECONDS));
978 shardManager.tell(new ActorInitialized(), mockShardActor);
980 Object resp = Await.result(future, duration("5 seconds"));
981 assertTrue("Expected: LocalShardFound, Actual: " + resp, resp instanceof LocalShardFound);
984 LOG.info("testOnReceiveFindLocalShardWaitForShardInitialized starting");
988 public void testOnRecoveryJournalIsCleaned() {
989 String persistenceID = "shard-manager-" + shardMrgIDSuffix;
990 InMemoryJournal.addEntry(persistenceID, 1L, new SchemaContextModules(ImmutableSet.of("foo")));
991 InMemoryJournal.addEntry(persistenceID, 2L, new SchemaContextModules(ImmutableSet.of("bar")));
992 InMemoryJournal.addDeleteMessagesCompleteLatch(persistenceID);
994 newTestShardManager();
996 InMemoryJournal.waitForDeleteMessagesComplete(persistenceID);
998 // Journal entries up to the last one should've been deleted
999 Map<Long, Object> journal = InMemoryJournal.get(persistenceID);
1000 synchronized (journal) {
1001 assertEquals("Journal size", 0, journal.size());
1006 public void testRoleChangeNotificationAndShardLeaderStateChangedReleaseReady() throws Exception {
1007 TestShardManager shardManager = newTestShardManager();
1009 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1010 shardManager.onReceiveCommand(new RoleChangeNotification(
1011 memberId, RaftState.Candidate.name(), RaftState.Leader.name()));
1013 verify(ready, never()).countDown();
1015 shardManager.onReceiveCommand(new ShardLeaderStateChanged(memberId, memberId,
1016 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1018 verify(ready, times(1)).countDown();
1022 public void testRoleChangeNotificationToFollowerWithShardLeaderStateChangedReleaseReady() throws Exception {
1023 new JavaTestKit(getSystem()) {{
1024 TestShardManager shardManager = newTestShardManager();
1026 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1027 shardManager.onReceiveCommand(new RoleChangeNotification(
1028 memberId, null, RaftState.Follower.name()));
1030 verify(ready, never()).countDown();
1032 shardManager.onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
1034 shardManager.onReceiveCommand(new ShardLeaderStateChanged(memberId,
1035 "member-2-shard-default-" + shardMrgIDSuffix, mock(DataTree.class),
1036 DataStoreVersions.CURRENT_VERSION));
1038 verify(ready, times(1)).countDown();
1043 public void testReadyCountDownForMemberUpAfterLeaderStateChanged() throws Exception {
1044 new JavaTestKit(getSystem()) {{
1045 TestShardManager shardManager = newTestShardManager();
1047 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1048 shardManager.onReceiveCommand(new RoleChangeNotification(memberId, null, RaftState.Follower.name()));
1050 verify(ready, never()).countDown();
1052 shardManager.onReceiveCommand(new ShardLeaderStateChanged(memberId,
1053 "member-2-shard-default-" + shardMrgIDSuffix, mock(DataTree.class),
1054 DataStoreVersions.CURRENT_VERSION));
1056 shardManager.onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
1058 verify(ready, times(1)).countDown();
1063 public void testRoleChangeNotificationDoNothingForUnknownShard() throws Exception {
1064 TestShardManager shardManager = newTestShardManager();
1066 shardManager.onReceiveCommand(new RoleChangeNotification(
1067 "unknown", RaftState.Candidate.name(), RaftState.Leader.name()));
1069 verify(ready, never()).countDown();
1073 public void testByDefaultSyncStatusIsFalse() throws Exception{
1074 TestShardManager shardManager = newTestShardManager();
1076 assertEquals(false, shardManager.getMBean().getSyncStatus());
1080 public void testWhenShardIsLeaderSyncStatusIsTrue() throws Exception{
1081 TestShardManager shardManager = newTestShardManager();
1083 shardManager.onReceiveCommand(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix,
1084 RaftState.Follower.name(), RaftState.Leader.name()));
1086 assertEquals(true, shardManager.getMBean().getSyncStatus());
1090 public void testWhenShardIsCandidateSyncStatusIsFalse() throws Exception{
1091 TestShardManager shardManager = newTestShardManager();
1093 String shardId = "member-1-shard-default-" + shardMrgIDSuffix;
1094 shardManager.onReceiveCommand(new RoleChangeNotification(shardId,
1095 RaftState.Follower.name(), RaftState.Candidate.name()));
1097 assertEquals(false, shardManager.getMBean().getSyncStatus());
1099 // Send a FollowerInitialSyncStatus with status = true for the replica whose current state is candidate
1100 shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(
1103 assertEquals(false, shardManager.getMBean().getSyncStatus());
1107 public void testWhenShardIsFollowerSyncStatusDependsOnFollowerInitialSyncStatus() throws Exception{
1108 TestShardManager shardManager = newTestShardManager();
1110 String shardId = "member-1-shard-default-" + shardMrgIDSuffix;
1111 shardManager.onReceiveCommand(new RoleChangeNotification(shardId,
1112 RaftState.Candidate.name(), RaftState.Follower.name()));
1114 // Initially will be false
1115 assertEquals(false, shardManager.getMBean().getSyncStatus());
1117 // Send status true will make sync status true
1118 shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(true, shardId));
1120 assertEquals(true, shardManager.getMBean().getSyncStatus());
1122 // Send status false will make sync status false
1123 shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(false, shardId));
1125 assertEquals(false, shardManager.getMBean().getSyncStatus());
1130 public void testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards() throws Exception{
1131 LOG.info("testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards starting");
1132 TestShardManager shardManager = newTestShardManager(newShardMgrProps(new MockConfiguration() {
1134 public List<String> getMemberShardNames(MemberName memberName) {
1135 return Arrays.asList("default", "astronauts");
1139 // Initially will be false
1140 assertEquals(false, shardManager.getMBean().getSyncStatus());
1142 // Make default shard leader
1143 String defaultShardId = "member-1-shard-default-" + shardMrgIDSuffix;
1144 shardManager.onReceiveCommand(new RoleChangeNotification(defaultShardId,
1145 RaftState.Follower.name(), RaftState.Leader.name()));
1147 // default = Leader, astronauts is unknown so sync status remains false
1148 assertEquals(false, shardManager.getMBean().getSyncStatus());
1150 // Make astronauts shard leader as well
1151 String astronautsShardId = "member-1-shard-astronauts-" + shardMrgIDSuffix;
1152 shardManager.onReceiveCommand(new RoleChangeNotification(astronautsShardId,
1153 RaftState.Follower.name(), RaftState.Leader.name()));
1155 // Now sync status should be true
1156 assertEquals(true, shardManager.getMBean().getSyncStatus());
1158 // Make astronauts a Follower
1159 shardManager.onReceiveCommand(new RoleChangeNotification(astronautsShardId,
1160 RaftState.Leader.name(), RaftState.Follower.name()));
1162 // Sync status is not true
1163 assertEquals(false, shardManager.getMBean().getSyncStatus());
1165 // Make the astronauts follower sync status true
1166 shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(true, astronautsShardId));
1168 // Sync status is now true
1169 assertEquals(true, shardManager.getMBean().getSyncStatus());
1171 LOG.info("testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards ending");
1175 public void testOnReceiveSwitchShardBehavior() throws Exception {
1176 new JavaTestKit(getSystem()) {{
1177 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1179 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1180 shardManager.tell(new ActorInitialized(), mockShardActor);
1182 shardManager.tell(new SwitchShardBehavior(mockShardName, RaftState.Leader, 1000), getRef());
1184 SwitchBehavior switchBehavior = MessageCollectorActor.expectFirstMatching(mockShardActor, SwitchBehavior.class);
1186 assertEquals(RaftState.Leader, switchBehavior.getNewState());
1187 assertEquals(1000, switchBehavior.getNewTerm());
1191 private static List<MemberName> members(String... names) {
1192 return Arrays.asList(names).stream().map(MemberName::forName).collect(Collectors.toList());
1196 public void testOnCreateShard() {
1197 LOG.info("testOnCreateShard starting");
1198 new JavaTestKit(getSystem()) {{
1199 datastoreContextBuilder.shardInitializationTimeout(1, TimeUnit.MINUTES).persistent(true);
1201 ActorRef shardManager = actorFactory.createActor(newShardMgrProps(
1202 new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1204 SchemaContext schemaContext = TestModel.createTestContext();
1205 shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
1207 DatastoreContext datastoreContext = DatastoreContext.newBuilder().shardElectionTimeoutFactor(100).
1208 persistent(false).build();
1209 Shard.Builder shardBuilder = Shard.builder();
1211 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1212 "foo", null, members("member-1", "member-5", "member-6"));
1213 shardManager.tell(new CreateShard(config, shardBuilder, datastoreContext), getRef());
1215 expectMsgClass(duration("5 seconds"), Success.class);
1217 shardManager.tell(new FindLocalShard("foo", true), getRef());
1219 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1221 assertEquals("isRecoveryApplicable", false, shardBuilder.getDatastoreContext().isPersistent());
1222 assertTrue("Epxected ShardPeerAddressResolver", shardBuilder.getDatastoreContext().getShardRaftConfig().
1223 getPeerAddressResolver() instanceof ShardPeerAddressResolver);
1224 assertEquals("peerMembers", Sets.newHashSet(
1225 new ShardIdentifier("foo", MemberName.forName("member-5"), shardMrgIDSuffix).toString(),
1226 new ShardIdentifier("foo", MemberName.forName("member-6"), shardMrgIDSuffix).toString()),
1227 shardBuilder.getPeerAddresses().keySet());
1228 assertEquals("ShardIdentifier", new ShardIdentifier("foo", MEMBER_1, shardMrgIDSuffix),
1229 shardBuilder.getId());
1230 assertSame("schemaContext", schemaContext, shardBuilder.getSchemaContext());
1232 // Send CreateShard with same name - should return Success with a message.
1234 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1236 Success success = expectMsgClass(duration("5 seconds"), Success.class);
1237 assertNotNull("Success status is null", success.status());
1240 LOG.info("testOnCreateShard ending");
1244 public void testOnCreateShardWithLocalMemberNotInShardConfig() {
1245 LOG.info("testOnCreateShardWithLocalMemberNotInShardConfig starting");
1246 new JavaTestKit(getSystem()) {{
1247 datastoreContextBuilder.shardInitializationTimeout(1, TimeUnit.MINUTES).persistent(true);
1249 ActorRef shardManager = actorFactory.createActor(newShardMgrProps(
1250 new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1252 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1254 Shard.Builder shardBuilder = Shard.builder();
1255 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1256 "foo", null, members("member-5", "member-6"));
1258 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1259 expectMsgClass(duration("5 seconds"), Success.class);
1261 shardManager.tell(new FindLocalShard("foo", true), getRef());
1262 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1264 assertEquals("peerMembers size", 0, shardBuilder.getPeerAddresses().size());
1265 assertEquals("schemaContext", DisableElectionsRaftPolicy.class.getName(),
1266 shardBuilder.getDatastoreContext().getShardRaftConfig().getCustomRaftPolicyImplementationClass());
1269 LOG.info("testOnCreateShardWithLocalMemberNotInShardConfig ending");
1273 public void testOnCreateShardWithNoInitialSchemaContext() {
1274 LOG.info("testOnCreateShardWithNoInitialSchemaContext starting");
1275 new JavaTestKit(getSystem()) {{
1276 ActorRef shardManager = actorFactory.createActor(newShardMgrProps(
1277 new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1279 Shard.Builder shardBuilder = Shard.builder();
1281 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1282 "foo", null, members("member-1"));
1283 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1285 expectMsgClass(duration("5 seconds"), Success.class);
1287 SchemaContext schemaContext = TestModel.createTestContext();
1288 shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
1290 shardManager.tell(new FindLocalShard("foo", true), getRef());
1292 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1294 assertSame("schemaContext", schemaContext, shardBuilder.getSchemaContext());
1295 assertNotNull("schemaContext is null", shardBuilder.getDatastoreContext());
1298 LOG.info("testOnCreateShardWithNoInitialSchemaContext ending");
1302 public void testGetSnapshot() throws Throwable {
1303 LOG.info("testGetSnapshot starting");
1304 JavaTestKit kit = new JavaTestKit(getSystem());
1306 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder().
1307 put("shard1", Arrays.asList("member-1")).
1308 put("shard2", Arrays.asList("member-1")).
1309 put("astronauts", Collections.<String>emptyList()).build());
1311 TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(newShardMgrProps(mockConfig).
1312 withDispatcher(Dispatchers.DefaultDispatcherId()));
1314 shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1315 Failure failure = kit.expectMsgClass(Failure.class);
1316 assertEquals("Failure cause type", IllegalStateException.class, failure.cause().getClass());
1318 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1320 waitForShardInitialized(shardManager, "shard1", kit);
1321 waitForShardInitialized(shardManager, "shard2", kit);
1323 shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1325 DatastoreSnapshot datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1327 assertEquals("getType", shardMrgIDSuffix, datastoreSnapshot.getType());
1328 assertNull("Expected null ShardManagerSnapshot", datastoreSnapshot.getShardManagerSnapshot());
1330 Function<ShardSnapshot, String> shardNameTransformer = s -> s.getName();
1332 assertEquals("Shard names", Sets.newHashSet("shard1", "shard2"), Sets.newHashSet(
1333 Lists.transform(datastoreSnapshot.getShardSnapshots(), shardNameTransformer)));
1335 // Add a new replica
1337 JavaTestKit mockShardLeaderKit = new JavaTestKit(getSystem());
1339 TestShardManager shardManagerInstance = shardManager.underlyingActor();
1340 shardManagerInstance.setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1342 shardManager.tell(new AddShardReplica("astronauts"), kit.getRef());
1343 mockShardLeaderKit.expectMsgClass(AddServer.class);
1344 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.OK, ""));
1345 kit.expectMsgClass(Status.Success.class);
1346 waitForShardInitialized(shardManager, "astronauts", kit);
1348 // Send another GetSnapshot and verify
1350 shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1351 datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1353 assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"), Sets.newHashSet(
1354 Lists.transform(datastoreSnapshot.getShardSnapshots(), shardNameTransformer)));
1356 byte[] snapshotBytes = datastoreSnapshot.getShardManagerSnapshot();
1357 assertNotNull("Expected ShardManagerSnapshot", snapshotBytes);
1358 ShardManagerSnapshot snapshot = SerializationUtils.deserialize(snapshotBytes);
1359 assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"),
1360 Sets.newHashSet(snapshot.getShardList()));
1362 LOG.info("testGetSnapshot ending");
1366 public void testRestoreFromSnapshot() throws Throwable {
1367 LOG.info("testRestoreFromSnapshot starting");
1369 datastoreContextBuilder.shardInitializationTimeout(3, TimeUnit.SECONDS);
1371 JavaTestKit kit = new JavaTestKit(getSystem());
1373 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder().
1374 put("shard1", Collections.<String>emptyList()).
1375 put("shard2", Collections.<String>emptyList()).
1376 put("astronauts", Collections.<String>emptyList()).build());
1379 ShardManagerSnapshot snapshot = new ShardManagerSnapshot(Arrays.asList("shard1", "shard2", "astronauts"));
1380 DatastoreSnapshot restoreFromSnapshot = new DatastoreSnapshot(shardMrgIDSuffix,
1381 SerializationUtils.serialize(snapshot), Collections.<ShardSnapshot>emptyList());
1382 TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(newTestShardMgrBuilder(mockConfig).
1383 restoreFromSnapshot(restoreFromSnapshot).props().withDispatcher(Dispatchers.DefaultDispatcherId()));
1385 shardManager.underlyingActor().waitForRecoveryComplete();
1387 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1389 waitForShardInitialized(shardManager, "shard1", kit);
1390 waitForShardInitialized(shardManager, "shard2", kit);
1391 waitForShardInitialized(shardManager, "astronauts", kit);
1393 shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1395 DatastoreSnapshot datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1397 assertEquals("getType", shardMrgIDSuffix, datastoreSnapshot.getType());
1399 byte[] snapshotBytes = datastoreSnapshot.getShardManagerSnapshot();
1400 assertNotNull("Expected ShardManagerSnapshot", snapshotBytes);
1401 snapshot = SerializationUtils.deserialize(snapshotBytes);
1402 assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"),
1403 Sets.newHashSet(snapshot.getShardList()));
1405 LOG.info("testRestoreFromSnapshot ending");
1409 public void testAddShardReplicaForNonExistentShardConfig() throws Exception {
1410 new JavaTestKit(getSystem()) {{
1411 ActorRef shardManager = actorFactory.createActor(newShardMgrProps(
1412 new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1414 shardManager.tell(new AddShardReplica("model-inventory"), getRef());
1415 Status.Failure resp = expectMsgClass(duration("2 seconds"), Status.Failure.class);
1417 assertEquals("Failure obtained", true,
1418 (resp.cause() instanceof IllegalArgumentException));
1423 public void testAddShardReplica() throws Exception {
1424 LOG.info("testAddShardReplica starting");
1425 MockConfiguration mockConfig =
1426 new MockConfiguration(ImmutableMap.<String, List<String>>builder().
1427 put("default", Arrays.asList("member-1", "member-2")).
1428 put("astronauts", Arrays.asList("member-2")).build());
1430 final String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
1431 datastoreContextBuilder.shardManagerPersistenceId(shardManagerID);
1433 // Create an ActorSystem ShardManager actor for member-1.
1434 final ActorSystem system1 = newActorSystem("Member1");
1435 Cluster.get(system1).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
1436 ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
1437 final TestActorRef<TestShardManager> newReplicaShardManager = TestActorRef.create(system1,
1438 newTestShardMgrBuilder(mockConfig).shardActor(mockDefaultShardActor).cluster(
1439 new ClusterWrapperImpl(system1)).props().withDispatcher(Dispatchers.DefaultDispatcherId()), shardManagerID);
1441 // Create an ActorSystem ShardManager actor for member-2.
1442 final ActorSystem system2 = newActorSystem("Member2");
1443 Cluster.get(system2).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
1445 String name = new ShardIdentifier("astronauts", MEMBER_2, "config").toString();
1446 final TestActorRef<MockRespondActor> mockShardLeaderActor =
1447 TestActorRef.create(system2, Props.create(MockRespondActor.class).
1448 withDispatcher(Dispatchers.DefaultDispatcherId()), name);
1449 final TestActorRef<TestShardManager> leaderShardManager = TestActorRef.create(system2,
1450 newTestShardMgrBuilder(mockConfig).shardActor(mockShardLeaderActor).cluster(
1451 new ClusterWrapperImpl(system2)).props().
1452 withDispatcher(Dispatchers.DefaultDispatcherId()), shardManagerID);
1454 new JavaTestKit(system1) {{
1456 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1457 leaderShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1459 leaderShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1461 String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
1462 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
1463 leaderShardManager.tell(new ShardLeaderStateChanged(memberId2, memberId2,
1464 mock(DataTree.class), leaderVersion), mockShardLeaderActor);
1465 leaderShardManager.tell(new RoleChangeNotification(memberId2,
1466 RaftState.Candidate.name(), RaftState.Leader.name()), mockShardLeaderActor);
1468 newReplicaShardManager.underlyingActor().waitForMemberUp();
1469 leaderShardManager.underlyingActor().waitForMemberUp();
1471 //Have a dummy snapshot to be overwritten by the new data persisted.
1472 String[] restoredShards = {"default", "people"};
1473 ShardManagerSnapshot snapshot = new ShardManagerSnapshot(Arrays.asList(restoredShards));
1474 InMemorySnapshotStore.addSnapshot(shardManagerID, snapshot);
1475 Uninterruptibles.sleepUninterruptibly(2, TimeUnit.MILLISECONDS);
1477 InMemorySnapshotStore.addSnapshotSavedLatch(shardManagerID);
1478 InMemorySnapshotStore.addSnapshotDeletedLatch(shardManagerID);
1480 //construct a mock response message
1481 AddServerReply response = new AddServerReply(ServerChangeStatus.OK, memberId2);
1482 mockShardLeaderActor.underlyingActor().updateResponse(response);
1483 newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
1484 AddServer addServerMsg = MessageCollectorActor.expectFirstMatching(mockShardLeaderActor,
1486 String addServerId = "member-1-shard-astronauts-" + shardMrgIDSuffix;
1487 assertEquals("AddServer serverId", addServerId, addServerMsg.getNewServerId());
1488 expectMsgClass(duration("5 seconds"), Status.Success.class);
1490 InMemorySnapshotStore.waitForSavedSnapshot(shardManagerID, ShardManagerSnapshot.class);
1491 InMemorySnapshotStore.waitForDeletedSnapshot(shardManagerID);
1492 List<ShardManagerSnapshot> persistedSnapshots =
1493 InMemorySnapshotStore.getSnapshots(shardManagerID, ShardManagerSnapshot.class);
1494 assertEquals("Number of snapshots persisted", 1, persistedSnapshots.size());
1495 ShardManagerSnapshot shardManagerSnapshot = persistedSnapshots.get(0);
1496 assertEquals("Persisted local shards", Sets.newHashSet("default", "astronauts"),
1497 Sets.newHashSet(shardManagerSnapshot.getShardList()));
1499 LOG.info("testAddShardReplica ending");
1503 public void testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader() throws Exception {
1504 LOG.info("testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader starting");
1505 new JavaTestKit(getSystem()) {{
1506 TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(
1507 newPropsShardMgrWithMockShardActor(), shardMgrID);
1509 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1510 shardManager.tell(new ActorInitialized(), mockShardActor);
1512 String leaderId = "leader-member-shard-default-" + shardMrgIDSuffix;
1513 AddServerReply addServerReply = new AddServerReply(ServerChangeStatus.ALREADY_EXISTS, null);
1514 ActorRef leaderShardActor = shardManager.underlyingActor().getContext().actorOf(
1515 Props.create(MockRespondActor.class, addServerReply), leaderId);
1517 MockClusterWrapper.sendMemberUp(shardManager, "leader-member", leaderShardActor.path().toString());
1519 String newReplicaId = "member-1-shard-default-" + shardMrgIDSuffix;
1520 shardManager.tell(new RoleChangeNotification(newReplicaId,
1521 RaftState.Candidate.name(), RaftState.Follower.name()), mockShardActor);
1522 shardManager.tell(new ShardLeaderStateChanged(newReplicaId, leaderId,
1523 DataStoreVersions.CURRENT_VERSION), mockShardActor);
1525 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1527 MessageCollectorActor.expectFirstMatching(leaderShardActor, AddServer.class);
1529 Failure resp = expectMsgClass(duration("5 seconds"), Failure.class);
1530 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1532 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1533 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1535 // Send message again to verify previous in progress state is cleared
1537 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1538 resp = expectMsgClass(duration("5 seconds"), Failure.class);
1539 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1541 // Send message again with an AddServer timeout to verify the pre-existing shard actor isn't terminated.
1543 shardManager.tell(newDatastoreContextFactory(datastoreContextBuilder.
1544 shardLeaderElectionTimeout(100, TimeUnit.MILLISECONDS).build()), getRef());
1545 leaderShardActor.tell(MockRespondActor.CLEAR_RESPONSE, ActorRef.noSender());
1546 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1547 expectMsgClass(duration("5 seconds"), Failure.class);
1549 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1550 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1553 LOG.info("testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader ending");
1557 public void testAddShardReplicaWithPreExistingLocalReplicaLeader() throws Exception {
1558 LOG.info("testAddShardReplicaWithPreExistingLocalReplicaLeader starting");
1559 new JavaTestKit(getSystem()) {{
1560 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1561 ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1563 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1564 shardManager.tell(new ActorInitialized(), mockShardActor);
1565 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
1566 DataStoreVersions.CURRENT_VERSION), getRef());
1567 shardManager.tell((new RoleChangeNotification(memberId, RaftState.Candidate.name(),
1568 RaftState.Leader.name())), mockShardActor);
1570 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1571 Failure resp = expectMsgClass(duration("5 seconds"), Failure.class);
1572 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1574 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1575 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1578 LOG.info("testAddShardReplicaWithPreExistingLocalReplicaLeader ending");
1582 public void testAddShardReplicaWithAddServerReplyFailure() throws Exception {
1583 LOG.info("testAddShardReplicaWithAddServerReplyFailure starting");
1584 new JavaTestKit(getSystem()) {{
1585 JavaTestKit mockShardLeaderKit = new JavaTestKit(getSystem());
1587 MockConfiguration mockConfig =
1588 new MockConfiguration(ImmutableMap.<String, List<String>>builder().
1589 put("astronauts", Arrays.asList("member-2")).build());
1591 ActorRef mockNewReplicaShardActor = newMockShardActor(getSystem(), "astronauts", "member-1");
1592 final TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(
1593 newTestShardMgrBuilder(mockConfig).shardActor(mockNewReplicaShardActor).props(), shardMgrID);
1594 shardManager.underlyingActor().setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1596 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1598 JavaTestKit terminateWatcher = new JavaTestKit(getSystem());
1599 terminateWatcher.watch(mockNewReplicaShardActor);
1601 shardManager.tell(new AddShardReplica("astronauts"), getRef());
1603 AddServer addServerMsg = mockShardLeaderKit.expectMsgClass(AddServer.class);
1604 assertEquals("AddServer serverId", "member-1-shard-astronauts-" + shardMrgIDSuffix,
1605 addServerMsg.getNewServerId());
1606 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.TIMEOUT, null));
1608 Failure failure = expectMsgClass(duration("5 seconds"), Failure.class);
1609 assertEquals("Failure cause", TimeoutException.class, failure.cause().getClass());
1611 shardManager.tell(new FindLocalShard("astronauts", false), getRef());
1612 expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
1614 terminateWatcher.expectTerminated(mockNewReplicaShardActor);
1616 shardManager.tell(new AddShardReplica("astronauts"), getRef());
1617 mockShardLeaderKit.expectMsgClass(AddServer.class);
1618 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.NO_LEADER, null));
1619 failure = expectMsgClass(duration("5 seconds"), Failure.class);
1620 assertEquals("Failure cause", NoShardLeaderException.class, failure.cause().getClass());
1623 LOG.info("testAddShardReplicaWithAddServerReplyFailure ending");
1627 public void testAddShardReplicaWithAlreadyInProgress() throws Exception {
1628 testServerChangeWhenAlreadyInProgress("astronauts", new AddShardReplica("astronauts"),
1629 AddServer.class, new AddShardReplica("astronauts"));
1633 public void testAddShardReplicaWithFindPrimaryTimeout() throws Exception {
1634 LOG.info("testAddShardReplicaWithFindPrimaryTimeout starting");
1635 datastoreContextBuilder.shardInitializationTimeout(100, TimeUnit.MILLISECONDS);
1636 new JavaTestKit(getSystem()) {{
1637 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder().
1638 put("astronauts", Arrays.asList("member-2")).build());
1640 final ActorRef newReplicaShardManager = actorFactory.createActor(newTestShardMgrBuilder(mockConfig).
1641 shardActor(mockShardActor).props(), shardMgrID);
1643 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1644 MockClusterWrapper.sendMemberUp(newReplicaShardManager, "member-2",
1645 AddressFromURIString.parse("akka.tcp://non-existent@127.0.0.1:5").toString());
1647 newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
1648 Status.Failure resp = expectMsgClass(duration("5 seconds"), Status.Failure.class);
1649 assertEquals("Failure obtained", true,
1650 (resp.cause() instanceof RuntimeException));
1653 LOG.info("testAddShardReplicaWithFindPrimaryTimeout ending");
1657 public void testRemoveShardReplicaForNonExistentShard() throws Exception {
1658 new JavaTestKit(getSystem()) {{
1659 ActorRef shardManager = actorFactory.createActor(newShardMgrProps(
1660 new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1662 shardManager.tell(new RemoveShardReplica("model-inventory", MEMBER_1), getRef());
1663 Status.Failure resp = expectMsgClass(duration("10 seconds"), Status.Failure.class);
1664 assertEquals("Failure obtained", true,
1665 (resp.cause() instanceof PrimaryNotFoundException));
1673 public void testRemoveShardReplicaLocal() throws Exception {
1674 new JavaTestKit(getSystem()) {{
1675 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1677 final TestActorRef<MockRespondActor> respondActor =
1678 TestActorRef.create(getSystem(), Props.create(MockRespondActor.class), memberId);
1680 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
1682 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1683 shardManager.tell(new ActorInitialized(), respondActor);
1684 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
1685 DataStoreVersions.CURRENT_VERSION), getRef());
1686 shardManager.tell((new RoleChangeNotification(memberId, RaftState.Candidate.name(),
1687 RaftState.Leader.name())), respondActor);
1689 respondActor.underlyingActor().updateResponse(new RemoveServerReply(ServerChangeStatus.OK, null));
1690 shardManager.tell(new RemoveShardReplica(Shard.DEFAULT_NAME, MEMBER_1), getRef());
1691 final RemoveServer removeServer = MessageCollectorActor.expectFirstMatching(respondActor, RemoveServer.class);
1692 assertEquals(new ShardIdentifier("default", MEMBER_1, shardMrgIDSuffix).toString(),
1693 removeServer.getServerId());
1694 expectMsgClass(duration("5 seconds"), Success.class);
1699 public void testRemoveShardReplicaRemote() throws Exception {
1700 MockConfiguration mockConfig =
1701 new MockConfiguration(ImmutableMap.<String, List<String>>builder().
1702 put("default", Arrays.asList("member-1", "member-2")).
1703 put("astronauts", Arrays.asList("member-1")).build());
1705 String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
1707 // Create an ActorSystem ShardManager actor for member-1.
1708 final ActorSystem system1 = newActorSystem("Member1");
1709 Cluster.get(system1).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
1710 ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
1712 final TestActorRef<TestShardManager> newReplicaShardManager = TestActorRef.create(system1,
1713 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockDefaultShardActor).cluster(
1714 new ClusterWrapperImpl(system1)).props().withDispatcher(Dispatchers.DefaultDispatcherId()),
1717 // Create an ActorSystem ShardManager actor for member-2.
1718 final ActorSystem system2 = newActorSystem("Member2");
1719 Cluster.get(system2).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
1721 String name = new ShardIdentifier("default", MEMBER_2, shardMrgIDSuffix).toString();
1722 final TestActorRef<MockRespondActor> mockShardLeaderActor =
1723 TestActorRef.create(system2, Props.create(MockRespondActor.class), name);
1725 LOG.error("Mock Shard Leader Actor : {}", mockShardLeaderActor);
1727 final TestActorRef<TestShardManager> leaderShardManager = TestActorRef.create(system2,
1728 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockShardLeaderActor).cluster(
1729 new ClusterWrapperImpl(system2)).props().withDispatcher(Dispatchers.DefaultDispatcherId()),
1732 // Because mockShardLeaderActor is created at the top level of the actor system it has an address like so,
1733 // akka.tcp://cluster-test@127.0.0.1:2559/user/member-2-shard-default-config1
1734 // However when a shard manager has a local shard which is a follower and a leader that is remote it will
1735 // try to compute an address for the remote shard leader using the ShardPeerAddressResolver. This address will
1737 // akka.tcp://cluster-test@127.0.0.1:2559/user/shardmanager-config1/member-2-shard-default-config1
1738 // In this specific case if we did a FindPrimary for shard default from member-1 we would come up
1739 // with the address of an actor which does not exist, therefore any message sent to that actor would go to
1741 // To work around this problem we create a ForwardingActor with the right address and pass to it the
1742 // mockShardLeaderActor. The ForwardingActor simply forwards all messages to the mockShardLeaderActor and every
1743 // thing works as expected
1744 final ActorRef actorRef = leaderShardManager.underlyingActor().context()
1745 .actorOf(Props.create(ForwardingActor.class, mockShardLeaderActor), "member-2-shard-default-" + shardMrgIDSuffix);
1747 LOG.error("Forwarding actor : {}", actorRef);
1749 new JavaTestKit(system1) {{
1751 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1752 leaderShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1754 leaderShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1755 newReplicaShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1757 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
1758 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
1759 leaderShardManager.tell(new ShardLeaderStateChanged(memberId2, memberId2,
1760 mock(DataTree.class), leaderVersion), mockShardLeaderActor);
1761 leaderShardManager.tell(new RoleChangeNotification(memberId2,
1762 RaftState.Candidate.name(), RaftState.Leader.name()), mockShardLeaderActor);
1764 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
1765 newReplicaShardManager.tell(new ShardLeaderStateChanged(memberId1, memberId2,
1766 mock(DataTree.class), leaderVersion), mockShardActor);
1767 newReplicaShardManager.tell(new RoleChangeNotification(memberId1,
1768 RaftState.Candidate.name(), RaftState.Follower.name()), mockShardActor);
1770 newReplicaShardManager.underlyingActor().waitForMemberUp();
1771 leaderShardManager.underlyingActor().waitForMemberUp();
1773 //construct a mock response message
1774 RemoveServerReply response = new RemoveServerReply(ServerChangeStatus.OK, memberId2);
1775 mockShardLeaderActor.underlyingActor().updateResponse(response);
1776 newReplicaShardManager.tell(new RemoveShardReplica("default", MEMBER_1), getRef());
1777 RemoveServer removeServer = MessageCollectorActor.expectFirstMatching(mockShardLeaderActor,
1778 RemoveServer.class);
1779 String removeServerId = new ShardIdentifier("default", MEMBER_1, shardMrgIDSuffix).toString();
1780 assertEquals("RemoveServer serverId", removeServerId, removeServer.getServerId());
1781 expectMsgClass(duration("5 seconds"), Status.Success.class);
1787 public void testRemoveShardReplicaWhenAnotherRemoveShardReplicaAlreadyInProgress() throws Exception {
1788 testServerChangeWhenAlreadyInProgress("astronauts", new RemoveShardReplica("astronauts", MEMBER_2),
1789 RemoveServer.class, new RemoveShardReplica("astronauts", MEMBER_3));
1793 public void testRemoveShardReplicaWhenAddShardReplicaAlreadyInProgress() throws Exception {
1794 testServerChangeWhenAlreadyInProgress("astronauts", new AddShardReplica("astronauts"),
1795 AddServer.class, new RemoveShardReplica("astronauts", MEMBER_2));
1799 public void testServerChangeWhenAlreadyInProgress(final String shardName, final Object firstServerChange,
1800 final Class<?> firstForwardedServerChangeClass,
1801 final Object secondServerChange) throws Exception {
1802 new JavaTestKit(getSystem()) {{
1803 JavaTestKit mockShardLeaderKit = new JavaTestKit(getSystem());
1804 JavaTestKit secondRequestKit = new JavaTestKit(getSystem());
1806 MockConfiguration mockConfig =
1807 new MockConfiguration(ImmutableMap.<String, List<String>>builder().
1808 put(shardName, Arrays.asList("member-2")).build());
1810 final TestActorRef<TestShardManager> shardManager = TestActorRef.create(getSystem(),
1811 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockShardActor).cluster(
1812 new MockClusterWrapper()).props().withDispatcher(Dispatchers.DefaultDispatcherId()),
1815 shardManager.underlyingActor().setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1817 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1819 shardManager.tell(firstServerChange, getRef());
1821 mockShardLeaderKit.expectMsgClass(firstForwardedServerChangeClass);
1823 shardManager.tell(secondServerChange, secondRequestKit.getRef());
1825 secondRequestKit.expectMsgClass(duration("5 seconds"), Failure.class);
1830 public void testServerRemovedShardActorNotRunning() throws Exception {
1831 LOG.info("testServerRemovedShardActorNotRunning starting");
1832 new JavaTestKit(getSystem()) {{
1833 MockConfiguration mockConfig =
1834 new MockConfiguration(ImmutableMap.<String, List<String>>builder().
1835 put("default", Arrays.asList("member-1", "member-2")).
1836 put("astronauts", Arrays.asList("member-2")).
1837 put("people", Arrays.asList("member-1", "member-2")).build());
1839 TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(newShardMgrProps(mockConfig));
1841 shardManager.underlyingActor().waitForRecoveryComplete();
1842 shardManager.tell(new FindLocalShard("people", false), getRef());
1843 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1845 shardManager.tell(new FindLocalShard("default", false), getRef());
1846 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1848 // Removed the default shard replica from member-1
1849 ShardIdentifier.Builder builder = new ShardIdentifier.Builder();
1850 ShardIdentifier shardId = builder.shardName("default").memberName(MEMBER_1).type(shardMrgIDSuffix).build();
1851 shardManager.tell(new ServerRemoved(shardId.toString()), getRef());
1853 shardManager.underlyingActor().verifySnapshotPersisted(Sets.newHashSet("people"));
1856 LOG.info("testServerRemovedShardActorNotRunning ending");
1860 public void testServerRemovedShardActorRunning() throws Exception {
1861 LOG.info("testServerRemovedShardActorRunning starting");
1862 new JavaTestKit(getSystem()) {{
1863 MockConfiguration mockConfig =
1864 new MockConfiguration(ImmutableMap.<String, List<String>>builder().
1865 put("default", Arrays.asList("member-1", "member-2")).
1866 put("astronauts", Arrays.asList("member-2")).
1867 put("people", Arrays.asList("member-1", "member-2")).build());
1869 String shardId = ShardIdentifier.builder().shardName("default").memberName(MEMBER_1).
1870 type(shardMrgIDSuffix).build().toString();
1871 TestActorRef<MessageCollectorActor> shard = actorFactory.createTestActor(
1872 MessageCollectorActor.props(), shardId);
1874 TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(
1875 newTestShardMgrBuilder(mockConfig).addShardActor("default", shard).props());
1877 shardManager.underlyingActor().waitForRecoveryComplete();
1879 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1880 shardManager.tell(new ActorInitialized(), shard);
1882 waitForShardInitialized(shardManager, "people", this);
1883 waitForShardInitialized(shardManager, "default", this);
1885 // Removed the default shard replica from member-1
1886 shardManager.tell(new ServerRemoved(shardId), getRef());
1888 shardManager.underlyingActor().verifySnapshotPersisted(Sets.newHashSet("people"));
1890 MessageCollectorActor.expectFirstMatching(shard, Shutdown.class);
1893 LOG.info("testServerRemovedShardActorRunning ending");
1898 public void testShardPersistenceWithRestoredData() throws Exception {
1899 LOG.info("testShardPersistenceWithRestoredData starting");
1900 new JavaTestKit(getSystem()) {{
1901 MockConfiguration mockConfig =
1902 new MockConfiguration(ImmutableMap.<String, List<String>>builder().
1903 put("default", Arrays.asList("member-1", "member-2")).
1904 put("astronauts", Arrays.asList("member-2")).
1905 put("people", Arrays.asList("member-1", "member-2")).build());
1906 String[] restoredShards = {"default", "astronauts"};
1907 ShardManagerSnapshot snapshot = new ShardManagerSnapshot(Arrays.asList(restoredShards));
1908 InMemorySnapshotStore.addSnapshot("shard-manager-" + shardMrgIDSuffix, snapshot);
1910 //create shardManager to come up with restored data
1911 TestActorRef<TestShardManager> newRestoredShardManager = actorFactory.createTestActor(
1912 newShardMgrProps(mockConfig));
1914 newRestoredShardManager.underlyingActor().waitForRecoveryComplete();
1916 newRestoredShardManager.tell(new FindLocalShard("people", false), getRef());
1917 LocalShardNotFound notFound = expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
1918 assertEquals("for uninitialized shard", "people", notFound.getShardName());
1920 //Verify a local shard is created for the restored shards,
1921 //although we expect a NotInitializedException for the shards as the actor initialization
1922 //message is not sent for them
1923 newRestoredShardManager.tell(new FindLocalShard("default", false), getRef());
1924 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1926 newRestoredShardManager.tell(new FindLocalShard("astronauts", false), getRef());
1927 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1930 LOG.info("testShardPersistenceWithRestoredData ending");
1934 public void testShutDown() throws Exception {
1935 LOG.info("testShutDown starting");
1936 new JavaTestKit(getSystem()) {{
1937 MockConfiguration mockConfig =
1938 new MockConfiguration(ImmutableMap.<String, List<String>>builder().
1939 put("shard1", Arrays.asList("member-1")).
1940 put("shard2", Arrays.asList("member-1")).build());
1942 String shardId1 = ShardIdentifier.builder().shardName("shard1").memberName(MEMBER_1).
1943 type(shardMrgIDSuffix).build().toString();
1944 TestActorRef<MessageCollectorActor> shard1 = actorFactory.createTestActor(
1945 MessageCollectorActor.props().withDispatcher(Dispatchers.DefaultDispatcherId()), shardId1);
1947 String shardId2 = ShardIdentifier.builder().shardName("shard2").memberName(MEMBER_1).
1948 type(shardMrgIDSuffix).build().toString();
1949 TestActorRef<MessageCollectorActor> shard2 = actorFactory.createTestActor(
1950 MessageCollectorActor.props().withDispatcher(Dispatchers.DefaultDispatcherId()), shardId2);
1952 TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(newTestShardMgrBuilder(
1953 mockConfig).addShardActor("shard1", shard1).addShardActor("shard2", shard2).props().
1954 withDispatcher(Dispatchers.DefaultDispatcherId()));
1956 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1957 shardManager.tell(new ActorInitialized(), shard1);
1958 shardManager.tell(new ActorInitialized(), shard2);
1960 FiniteDuration duration = FiniteDuration.create(5, TimeUnit.SECONDS);
1961 Future<Boolean> stopFuture = Patterns.gracefulStop(shardManager, duration, Shutdown.INSTANCE);
1963 MessageCollectorActor.expectFirstMatching(shard1, Shutdown.class);
1964 MessageCollectorActor.expectFirstMatching(shard2, Shutdown.class);
1967 Await.ready(stopFuture, FiniteDuration.create(500, TimeUnit.MILLISECONDS));
1968 fail("ShardManager actor stopped without waiting for the Shards to be stopped");
1969 } catch(TimeoutException e) {
1973 actorFactory.killActor(shard1, this);
1974 actorFactory.killActor(shard2, this);
1976 Boolean stopped = Await.result(stopFuture, duration);
1977 assertEquals("Stopped", Boolean.TRUE, stopped);
1980 LOG.info("testShutDown ending");
1983 private static class TestShardManager extends ShardManager {
1984 private final CountDownLatch recoveryComplete = new CountDownLatch(1);
1985 private final CountDownLatch snapshotPersist = new CountDownLatch(1);
1986 private ShardManagerSnapshot snapshot;
1987 private final Map<String, ActorRef> shardActors;
1988 private final ActorRef shardActor;
1989 private CountDownLatch findPrimaryMessageReceived = new CountDownLatch(1);
1990 private CountDownLatch memberUpReceived = new CountDownLatch(1);
1991 private CountDownLatch memberRemovedReceived = new CountDownLatch(1);
1992 private CountDownLatch memberUnreachableReceived = new CountDownLatch(1);
1993 private CountDownLatch memberReachableReceived = new CountDownLatch(1);
1994 private volatile MessageInterceptor messageInterceptor;
1996 private TestShardManager(Builder builder) {
1998 shardActor = builder.shardActor;
1999 shardActors = builder.shardActors;
2003 protected void handleRecover(Object message) throws Exception {
2005 super.handleRecover(message);
2007 if(message instanceof RecoveryCompleted) {
2008 recoveryComplete.countDown();
2013 private void countDownIfOther(final Member member, CountDownLatch latch) {
2014 if (!getCluster().getCurrentMemberName().equals(memberToName(member))) {
2020 public void handleCommand(Object message) throws Exception {
2022 if(messageInterceptor != null && messageInterceptor.canIntercept(message)) {
2023 getSender().tell(messageInterceptor.apply(message), getSelf());
2025 super.handleCommand(message);
2028 if(message instanceof FindPrimary) {
2029 findPrimaryMessageReceived.countDown();
2030 } else if(message instanceof ClusterEvent.MemberUp) {
2031 countDownIfOther(((ClusterEvent.MemberUp)message).member(), memberUpReceived);
2032 } else if(message instanceof ClusterEvent.MemberRemoved) {
2033 countDownIfOther(((ClusterEvent.MemberRemoved)message).member(), memberRemovedReceived);
2034 } else if(message instanceof ClusterEvent.UnreachableMember) {
2035 countDownIfOther(((ClusterEvent.UnreachableMember)message).member(), memberUnreachableReceived);
2036 } else if(message instanceof ClusterEvent.ReachableMember) {
2037 countDownIfOther(((ClusterEvent.ReachableMember)message).member(), memberReachableReceived);
2042 void setMessageInterceptor(MessageInterceptor messageInterceptor) {
2043 this.messageInterceptor = messageInterceptor;
2046 void waitForRecoveryComplete() {
2047 assertEquals("Recovery complete", true,
2048 Uninterruptibles.awaitUninterruptibly(recoveryComplete, 5, TimeUnit.SECONDS));
2051 void waitForMemberUp() {
2052 assertEquals("MemberUp received", true,
2053 Uninterruptibles.awaitUninterruptibly(memberUpReceived, 5, TimeUnit.SECONDS));
2054 memberUpReceived = new CountDownLatch(1);
2057 void waitForMemberRemoved() {
2058 assertEquals("MemberRemoved received", true,
2059 Uninterruptibles.awaitUninterruptibly(memberRemovedReceived, 5, TimeUnit.SECONDS));
2060 memberRemovedReceived = new CountDownLatch(1);
2063 void waitForUnreachableMember() {
2064 assertEquals("UnreachableMember received", true,
2065 Uninterruptibles.awaitUninterruptibly(memberUnreachableReceived, 5, TimeUnit.SECONDS
2067 memberUnreachableReceived = new CountDownLatch(1);
2070 void waitForReachableMember() {
2071 assertEquals("ReachableMember received", true,
2072 Uninterruptibles.awaitUninterruptibly(memberReachableReceived, 5, TimeUnit.SECONDS));
2073 memberReachableReceived = new CountDownLatch(1);
2076 void verifyFindPrimary() {
2077 assertEquals("FindPrimary received", true,
2078 Uninterruptibles.awaitUninterruptibly(findPrimaryMessageReceived, 5, TimeUnit.SECONDS));
2079 findPrimaryMessageReceived = new CountDownLatch(1);
2082 public static Builder builder(DatastoreContext.Builder datastoreContextBuilder) {
2083 return new Builder(datastoreContextBuilder);
2086 private static class Builder extends AbstractGenericCreator<Builder, TestShardManager> {
2087 private ActorRef shardActor;
2088 private final Map<String, ActorRef> shardActors = new HashMap<>();
2090 Builder(DatastoreContext.Builder datastoreContextBuilder) {
2091 super(TestShardManager.class);
2092 datastoreContextFactory(newDatastoreContextFactory(datastoreContextBuilder.build()));
2095 Builder shardActor(ActorRef shardActor) {
2096 this.shardActor = shardActor;
2100 Builder addShardActor(String shardName, ActorRef actorRef){
2101 shardActors.put(shardName, actorRef);
2107 public void saveSnapshot(Object obj) {
2108 snapshot = (ShardManagerSnapshot) obj;
2109 snapshotPersist.countDown();
2110 super.saveSnapshot(obj);
2113 void verifySnapshotPersisted(Set<String> shardList) {
2114 assertEquals("saveSnapshot invoked", true,
2115 Uninterruptibles.awaitUninterruptibly(snapshotPersist, 5, TimeUnit.SECONDS));
2116 assertEquals("Shard Persisted", shardList, Sets.newHashSet(snapshot.getShardList()));
2120 protected ActorRef newShardActor(SchemaContext schemaContext, ShardInformation info) {
2121 if(shardActors.get(info.getShardName()) != null){
2122 return shardActors.get(info.getShardName());
2125 if(shardActor != null) {
2129 return super.newShardActor(schemaContext, info);
2133 private static abstract class AbstractGenericCreator<T extends AbstractGenericCreator<T, ?>, C extends ShardManager>
2134 extends AbstractShardManagerCreator<T> {
2135 private final Class<C> shardManagerClass;
2137 AbstractGenericCreator(Class<C> shardManagerClass) {
2138 this.shardManagerClass = shardManagerClass;
2139 cluster(new MockClusterWrapper()).configuration(new MockConfiguration()).
2140 waitTillReadyCountdownLatch(ready).primaryShardInfoCache(new PrimaryShardInfoFutureCache());
2144 public Props props() {
2146 return Props.create(shardManagerClass, this);
2150 private static class GenericCreator<C extends ShardManager> extends AbstractGenericCreator<GenericCreator<C>, C> {
2151 GenericCreator(Class<C> shardManagerClass) {
2152 super(shardManagerClass);
2156 private static class DelegatingShardManagerCreator implements Creator<ShardManager> {
2157 private static final long serialVersionUID = 1L;
2158 private final Creator<ShardManager> delegate;
2160 public DelegatingShardManagerCreator(Creator<ShardManager> delegate) {
2161 this.delegate = delegate;
2165 public ShardManager create() throws Exception {
2166 return delegate.create();
2170 interface MessageInterceptor extends Function<Object, Object> {
2171 boolean canIntercept(Object message);
2174 private static MessageInterceptor newFindPrimaryInterceptor(final ActorRef primaryActor) {
2175 return new MessageInterceptor(){
2177 public Object apply(Object message) {
2178 return new RemotePrimaryShardFound(Serialization.serializedActorPath(primaryActor), (short) 1);
2182 public boolean canIntercept(Object message) {
2183 return message instanceof FindPrimary;
2188 private static class MockRespondActor extends MessageCollectorActor {
2189 static final String CLEAR_RESPONSE = "clear-response";
2190 static final org.slf4j.Logger LOG = LoggerFactory.getLogger(MockRespondActor.class);
2192 private volatile Object responseMsg;
2194 @SuppressWarnings("unused")
2195 public MockRespondActor() {
2198 @SuppressWarnings("unused")
2199 public MockRespondActor(Object responseMsg) {
2200 this.responseMsg = responseMsg;
2203 public void updateResponse(Object response) {
2204 responseMsg = response;
2208 public void onReceive(Object message) throws Exception {
2209 if(!"get-all-messages".equals(message)) {
2210 LOG.debug("Received message : {}", message);
2212 super.onReceive(message);
2213 if (message instanceof AddServer && responseMsg != null) {
2214 getSender().tell(responseMsg, getSelf());
2215 } else if(message instanceof RemoveServer && responseMsg != null){
2216 getSender().tell(responseMsg, getSelf());
2217 } else if(message.equals(CLEAR_RESPONSE)) {