2 * Copyright (c) 2014, 2015 Cisco Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
9 package org.opendaylight.controller.cluster.datastore.shardmanager;
11 import static org.junit.Assert.assertEquals;
12 import static org.junit.Assert.assertNotNull;
13 import static org.junit.Assert.assertNull;
14 import static org.junit.Assert.assertSame;
15 import static org.junit.Assert.assertTrue;
16 import static org.junit.Assert.fail;
17 import static org.mockito.Mockito.mock;
18 import static org.mockito.Mockito.never;
19 import static org.mockito.Mockito.times;
20 import static org.mockito.Mockito.verify;
22 import akka.actor.ActorRef;
23 import akka.actor.ActorSystem;
24 import akka.actor.AddressFromURIString;
25 import akka.actor.Props;
26 import akka.actor.Status;
27 import akka.actor.Status.Failure;
28 import akka.actor.Status.Success;
29 import akka.cluster.Cluster;
30 import akka.cluster.ClusterEvent;
31 import akka.cluster.Member;
32 import akka.dispatch.Dispatchers;
33 import akka.japi.Creator;
34 import akka.pattern.Patterns;
35 import akka.persistence.RecoveryCompleted;
36 import akka.serialization.Serialization;
37 import akka.testkit.JavaTestKit;
38 import akka.testkit.TestActorRef;
39 import akka.util.Timeout;
40 import com.google.common.base.Function;
41 import com.google.common.base.Stopwatch;
42 import com.google.common.collect.ImmutableMap;
43 import com.google.common.collect.ImmutableSet;
44 import com.google.common.collect.Lists;
45 import com.google.common.collect.Sets;
46 import com.google.common.util.concurrent.Uninterruptibles;
48 import java.util.AbstractMap;
49 import java.util.Arrays;
50 import java.util.Collection;
51 import java.util.Collections;
52 import java.util.HashMap;
53 import java.util.List;
55 import java.util.Map.Entry;
57 import java.util.concurrent.CountDownLatch;
58 import java.util.concurrent.TimeUnit;
59 import java.util.concurrent.TimeoutException;
60 import java.util.stream.Collectors;
61 import org.junit.Test;
62 import org.mockito.Mockito;
63 import org.opendaylight.controller.cluster.access.concepts.MemberName;
64 import org.opendaylight.controller.cluster.datastore.AbstractShardManagerTest;
65 import org.opendaylight.controller.cluster.datastore.ClusterWrapperImpl;
66 import org.opendaylight.controller.cluster.datastore.DataStoreVersions;
67 import org.opendaylight.controller.cluster.datastore.DatastoreContext;
68 import org.opendaylight.controller.cluster.datastore.DatastoreContextFactory;
69 import org.opendaylight.controller.cluster.datastore.DistributedDataStore;
70 import org.opendaylight.controller.cluster.datastore.Shard;
71 import org.opendaylight.controller.cluster.datastore.ShardManager.SchemaContextModules;
72 import org.opendaylight.controller.cluster.datastore.config.ConfigurationImpl;
73 import org.opendaylight.controller.cluster.datastore.config.EmptyModuleShardConfigProvider;
74 import org.opendaylight.controller.cluster.datastore.config.ModuleShardConfiguration;
75 import org.opendaylight.controller.cluster.datastore.exceptions.AlreadyExistsException;
76 import org.opendaylight.controller.cluster.datastore.exceptions.NoShardLeaderException;
77 import org.opendaylight.controller.cluster.datastore.exceptions.NotInitializedException;
78 import org.opendaylight.controller.cluster.datastore.exceptions.PrimaryNotFoundException;
79 import org.opendaylight.controller.cluster.datastore.identifiers.ShardIdentifier;
80 import org.opendaylight.controller.cluster.datastore.identifiers.ShardManagerIdentifier;
81 import org.opendaylight.controller.cluster.datastore.messages.ActorInitialized;
82 import org.opendaylight.controller.cluster.datastore.messages.AddShardReplica;
83 import org.opendaylight.controller.cluster.datastore.messages.ChangeShardMembersVotingStatus;
84 import org.opendaylight.controller.cluster.datastore.messages.CreateShard;
85 import org.opendaylight.controller.cluster.datastore.messages.FindLocalShard;
86 import org.opendaylight.controller.cluster.datastore.messages.FindPrimary;
87 import org.opendaylight.controller.cluster.datastore.messages.LocalPrimaryShardFound;
88 import org.opendaylight.controller.cluster.datastore.messages.LocalShardFound;
89 import org.opendaylight.controller.cluster.datastore.messages.LocalShardNotFound;
90 import org.opendaylight.controller.cluster.datastore.messages.PeerDown;
91 import org.opendaylight.controller.cluster.datastore.messages.PeerUp;
92 import org.opendaylight.controller.cluster.datastore.messages.PrimaryShardInfo;
93 import org.opendaylight.controller.cluster.datastore.messages.RemotePrimaryShardFound;
94 import org.opendaylight.controller.cluster.datastore.messages.RemoveShardReplica;
95 import org.opendaylight.controller.cluster.datastore.messages.ShardLeaderStateChanged;
96 import org.opendaylight.controller.cluster.datastore.messages.UpdateSchemaContext;
97 import org.opendaylight.controller.cluster.datastore.persisted.DatastoreSnapshot;
98 import org.opendaylight.controller.cluster.datastore.persisted.DatastoreSnapshot.ShardSnapshot;
99 import org.opendaylight.controller.cluster.datastore.persisted.ShardManagerSnapshot;
100 import org.opendaylight.controller.cluster.datastore.utils.ForwardingActor;
101 import org.opendaylight.controller.cluster.datastore.utils.MockClusterWrapper;
102 import org.opendaylight.controller.cluster.datastore.utils.MockConfiguration;
103 import org.opendaylight.controller.cluster.datastore.utils.PrimaryShardInfoFutureCache;
104 import org.opendaylight.controller.cluster.notifications.LeaderStateChanged;
105 import org.opendaylight.controller.cluster.notifications.RegisterRoleChangeListener;
106 import org.opendaylight.controller.cluster.notifications.RoleChangeNotification;
107 import org.opendaylight.controller.cluster.raft.RaftState;
108 import org.opendaylight.controller.cluster.raft.base.messages.FollowerInitialSyncUpStatus;
109 import org.opendaylight.controller.cluster.raft.base.messages.SwitchBehavior;
110 import org.opendaylight.controller.cluster.raft.client.messages.GetSnapshot;
111 import org.opendaylight.controller.cluster.raft.client.messages.Shutdown;
112 import org.opendaylight.controller.cluster.raft.messages.AddServer;
113 import org.opendaylight.controller.cluster.raft.messages.AddServerReply;
114 import org.opendaylight.controller.cluster.raft.messages.ChangeServersVotingStatus;
115 import org.opendaylight.controller.cluster.raft.messages.RemoveServer;
116 import org.opendaylight.controller.cluster.raft.messages.RemoveServerReply;
117 import org.opendaylight.controller.cluster.raft.messages.ServerChangeReply;
118 import org.opendaylight.controller.cluster.raft.messages.ServerChangeStatus;
119 import org.opendaylight.controller.cluster.raft.messages.ServerRemoved;
120 import org.opendaylight.controller.cluster.raft.policy.DisableElectionsRaftPolicy;
121 import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal;
122 import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore;
123 import org.opendaylight.controller.cluster.raft.utils.MessageCollectorActor;
124 import org.opendaylight.controller.md.cluster.datastore.model.TestModel;
125 import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTree;
126 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
127 import org.slf4j.Logger;
128 import org.slf4j.LoggerFactory;
129 import scala.concurrent.Await;
130 import scala.concurrent.Future;
131 import scala.concurrent.duration.FiniteDuration;
133 public class ShardManagerTest extends AbstractShardManagerTest {
134 private static final Logger LOG = LoggerFactory.getLogger(ShardManagerTest.class);
135 private static final MemberName MEMBER_2 = MemberName.forName("member-2");
136 private static final MemberName MEMBER_3 = MemberName.forName("member-3");
138 private final String shardMgrID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
140 private ActorSystem newActorSystem(String config) {
141 return newActorSystem("cluster-test", config);
144 private ActorRef newMockShardActor(ActorSystem system, String shardName, String memberName) {
145 String name = ShardIdentifier.create(shardName, MemberName.forName(memberName), "config").toString();
146 if (system == getSystem()) {
147 return actorFactory.createTestActor(Props.create(MessageCollectorActor.class), name);
150 return TestActorRef.create(system, Props.create(MessageCollectorActor.class), name);
153 private Props newShardMgrProps() {
154 return newShardMgrProps(new MockConfiguration());
157 private static DatastoreContextFactory newDatastoreContextFactory(DatastoreContext datastoreContext) {
158 DatastoreContextFactory mockFactory = mock(DatastoreContextFactory.class);
159 Mockito.doReturn(datastoreContext).when(mockFactory).getBaseDatastoreContext();
160 Mockito.doReturn(datastoreContext).when(mockFactory).getShardDatastoreContext(Mockito.anyString());
164 private TestShardManager.Builder newTestShardMgrBuilderWithMockShardActor() {
165 return newTestShardMgrBuilderWithMockShardActor(mockShardActor);
168 private TestShardManager.Builder newTestShardMgrBuilderWithMockShardActor(ActorRef shardActor) {
169 return TestShardManager.builder(datastoreContextBuilder).shardActor(shardActor)
170 .distributedDataStore(mock(DistributedDataStore.class));
174 private Props newPropsShardMgrWithMockShardActor() {
175 return newTestShardMgrBuilderWithMockShardActor().props().withDispatcher(
176 Dispatchers.DefaultDispatcherId());
179 private Props newPropsShardMgrWithMockShardActor(ActorRef shardActor) {
180 return newTestShardMgrBuilderWithMockShardActor(shardActor).props();
184 private TestShardManager newTestShardManager() {
185 return newTestShardManager(newShardMgrProps());
188 private TestShardManager newTestShardManager(Props props) {
189 TestActorRef<TestShardManager> shardManagerActor = actorFactory.createTestActor(props);
190 TestShardManager shardManager = shardManagerActor.underlyingActor();
191 shardManager.waitForRecoveryComplete();
195 private static void waitForShardInitialized(ActorRef shardManager, String shardName, JavaTestKit kit) {
196 AssertionError last = null;
197 Stopwatch sw = Stopwatch.createStarted();
198 while (sw.elapsed(TimeUnit.SECONDS) <= 5) {
200 shardManager.tell(new FindLocalShard(shardName, true), kit.getRef());
201 kit.expectMsgClass(LocalShardFound.class);
203 } catch (AssertionError e) {
207 Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
213 @SuppressWarnings("unchecked")
214 private static <T> T expectMsgClassOrFailure(Class<T> msgClass, JavaTestKit kit, String msg) {
215 Object reply = kit.expectMsgAnyClassOf(JavaTestKit.duration("5 sec"), msgClass, Failure.class);
216 if (reply instanceof Failure) {
217 throw new AssertionError(msg + " failed", ((Failure)reply).cause());
224 public void testPerShardDatastoreContext() throws Exception {
225 LOG.info("testPerShardDatastoreContext starting");
226 final DatastoreContextFactory mockFactory = newDatastoreContextFactory(
227 datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
230 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(6).build())
231 .when(mockFactory).getShardDatastoreContext("default");
234 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(7).build())
235 .when(mockFactory).getShardDatastoreContext("topology");
237 final MockConfiguration mockConfig = new MockConfiguration() {
239 public Collection<String> getMemberShardNames(MemberName memberName) {
240 return Arrays.asList("default", "topology");
244 public Collection<MemberName> getMembersFromShardName(String shardName) {
245 return members("member-1");
249 final TestActorRef<MessageCollectorActor> defaultShardActor = actorFactory.createTestActor(
250 Props.create(MessageCollectorActor.class), actorFactory.generateActorId("default"));
251 final TestActorRef<MessageCollectorActor> topologyShardActor = actorFactory.createTestActor(
252 Props.create(MessageCollectorActor.class), actorFactory.generateActorId("topology"));
254 final Map<String, Entry<ActorRef, DatastoreContext>> shardInfoMap = Collections.synchronizedMap(
255 new HashMap<String, Entry<ActorRef, DatastoreContext>>());
256 shardInfoMap.put("default", new AbstractMap.SimpleEntry<>(defaultShardActor, null));
257 shardInfoMap.put("topology", new AbstractMap.SimpleEntry<>(topologyShardActor, null));
259 final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
260 final CountDownLatch newShardActorLatch = new CountDownLatch(2);
261 class LocalShardManager extends ShardManager {
262 LocalShardManager(AbstractShardManagerCreator<?> creator) {
267 protected ActorRef newShardActor(SchemaContext schemaContext, ShardInformation info) {
268 Entry<ActorRef, DatastoreContext> entry = shardInfoMap.get(info.getShardName());
271 ref = entry.getKey();
272 entry.setValue(info.getDatastoreContext());
275 newShardActorLatch.countDown();
280 final Creator<ShardManager> creator = new Creator<ShardManager>() {
281 private static final long serialVersionUID = 1L;
283 public ShardManager create() throws Exception {
284 return new LocalShardManager(
285 new GenericCreator<>(LocalShardManager.class).datastoreContextFactory(mockFactory)
286 .primaryShardInfoCache(primaryShardInfoCache).configuration(mockConfig));
290 JavaTestKit kit = new JavaTestKit(getSystem());
292 final ActorRef shardManager = actorFactory.createActor(Props.create(
293 new DelegatingShardManagerCreator(creator)).withDispatcher(Dispatchers.DefaultDispatcherId()));
295 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), kit.getRef());
297 assertEquals("Shard actors created", true, newShardActorLatch.await(5, TimeUnit.SECONDS));
298 assertEquals("getShardElectionTimeoutFactor", 6,
299 shardInfoMap.get("default").getValue().getShardElectionTimeoutFactor());
300 assertEquals("getShardElectionTimeoutFactor", 7,
301 shardInfoMap.get("topology").getValue().getShardElectionTimeoutFactor());
303 DatastoreContextFactory newMockFactory = newDatastoreContextFactory(
304 datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
306 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(66).build())
307 .when(newMockFactory).getShardDatastoreContext("default");
310 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(77).build())
311 .when(newMockFactory).getShardDatastoreContext("topology");
313 shardManager.tell(newMockFactory, kit.getRef());
315 DatastoreContext newContext = MessageCollectorActor.expectFirstMatching(defaultShardActor,
316 DatastoreContext.class);
317 assertEquals("getShardElectionTimeoutFactor", 66, newContext.getShardElectionTimeoutFactor());
319 newContext = MessageCollectorActor.expectFirstMatching(topologyShardActor, DatastoreContext.class);
320 assertEquals("getShardElectionTimeoutFactor", 77, newContext.getShardElectionTimeoutFactor());
322 LOG.info("testPerShardDatastoreContext ending");
326 public void testOnReceiveFindPrimaryForNonExistentShard() throws Exception {
327 new JavaTestKit(getSystem()) {
329 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
331 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
333 shardManager.tell(new FindPrimary("non-existent", false), getRef());
335 expectMsgClass(duration("5 seconds"), PrimaryNotFoundException.class);
341 public void testOnReceiveFindPrimaryForLocalLeaderShard() throws Exception {
342 LOG.info("testOnReceiveFindPrimaryForLocalLeaderShard starting");
343 new JavaTestKit(getSystem()) {
345 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
347 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
349 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
350 shardManager.tell(new ActorInitialized(), mockShardActor);
352 DataTree mockDataTree = mock(DataTree.class);
353 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
354 DataStoreVersions.CURRENT_VERSION), getRef());
356 MessageCollectorActor.expectFirstMatching(mockShardActor, RegisterRoleChangeListener.class);
358 new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
361 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
363 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
364 LocalPrimaryShardFound.class);
365 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
366 primaryFound.getPrimaryPath().contains("member-1-shard-default"));
367 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
371 LOG.info("testOnReceiveFindPrimaryForLocalLeaderShard ending");
375 public void testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp() throws Exception {
376 LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp starting");
377 new JavaTestKit(getSystem()) {
379 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
381 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
382 shardManager.tell(new ActorInitialized(), mockShardActor);
384 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
385 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
387 new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
389 shardManager.tell(new LeaderStateChanged(memberId1, memberId2, DataStoreVersions.CURRENT_VERSION),
392 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
394 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
398 LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp ending");
402 public void testOnReceiveFindPrimaryForNonLocalLeaderShard() throws Exception {
403 LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShard starting");
404 new JavaTestKit(getSystem()) {
406 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
408 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
409 shardManager.tell(new ActorInitialized(), mockShardActor);
411 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
412 MockClusterWrapper.sendMemberUp(shardManager, "member-2", getRef().path().toString());
414 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
416 new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
418 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
419 shardManager.tell(new ShardLeaderStateChanged(memberId1, memberId2, leaderVersion), mockShardActor);
421 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
423 RemotePrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
424 RemotePrimaryShardFound.class);
425 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
426 primaryFound.getPrimaryPath().contains("member-2-shard-default"));
427 assertEquals("getPrimaryVersion", leaderVersion, primaryFound.getPrimaryVersion());
431 LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShard ending");
435 public void testOnReceiveFindPrimaryForUninitializedShard() throws Exception {
436 new JavaTestKit(getSystem()) {
438 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
440 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
442 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
448 public void testOnReceiveFindPrimaryForInitializedShardWithNoRole() throws Exception {
449 new JavaTestKit(getSystem()) {
451 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
453 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
454 shardManager.tell(new ActorInitialized(), mockShardActor);
456 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
458 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
464 public void testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId() throws Exception {
465 LOG.info("testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId starting");
466 new JavaTestKit(getSystem()) {
468 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
470 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
471 shardManager.tell(new ActorInitialized(), mockShardActor);
473 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
475 new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Follower.name()),
478 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
480 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
482 DataTree mockDataTree = mock(DataTree.class);
483 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
484 DataStoreVersions.CURRENT_VERSION), mockShardActor);
486 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
488 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
489 LocalPrimaryShardFound.class);
490 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
491 primaryFound.getPrimaryPath().contains("member-1-shard-default"));
492 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
496 LOG.info("testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId starting");
500 public void testOnReceiveFindPrimaryWaitForShardLeader() throws Exception {
501 LOG.info("testOnReceiveFindPrimaryWaitForShardLeader starting");
502 datastoreContextBuilder.shardInitializationTimeout(10, TimeUnit.SECONDS);
503 new JavaTestKit(getSystem()) {
505 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
507 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
509 // We're passing waitUntilInitialized = true to FindPrimary so
510 // the response should be
511 // delayed until we send ActorInitialized and
512 // RoleChangeNotification.
513 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
515 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
517 shardManager.tell(new ActorInitialized(), mockShardActor);
519 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
521 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
523 new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
526 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
528 DataTree mockDataTree = mock(DataTree.class);
529 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
530 DataStoreVersions.CURRENT_VERSION), mockShardActor);
532 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
533 LocalPrimaryShardFound.class);
534 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
535 primaryFound.getPrimaryPath().contains("member-1-shard-default"));
536 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
538 expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
542 LOG.info("testOnReceiveFindPrimaryWaitForShardLeader ending");
546 public void testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard() throws Exception {
547 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard starting");
548 new JavaTestKit(getSystem()) {
550 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
552 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
554 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
556 expectMsgClass(duration("2 seconds"), NotInitializedException.class);
558 shardManager.tell(new ActorInitialized(), mockShardActor);
560 expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
564 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard ending");
568 public void testOnReceiveFindPrimaryWaitForReadyWithCandidateShard() throws Exception {
569 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithCandidateShard starting");
570 new JavaTestKit(getSystem()) {
572 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
574 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
575 shardManager.tell(new ActorInitialized(), mockShardActor);
576 shardManager.tell(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix, null,
577 RaftState.Candidate.name()), mockShardActor);
579 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
581 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
585 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithCandidateShard ending");
589 public void testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard() throws Exception {
590 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard starting");
591 new JavaTestKit(getSystem()) {
593 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
595 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
596 shardManager.tell(new ActorInitialized(), mockShardActor);
597 shardManager.tell(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix, null,
598 RaftState.IsolatedLeader.name()), mockShardActor);
600 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
602 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
606 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard ending");
610 public void testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard() throws Exception {
611 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard starting");
612 new JavaTestKit(getSystem()) {
614 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
616 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
617 shardManager.tell(new ActorInitialized(), mockShardActor);
619 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
621 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
625 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard ending");
629 public void testOnReceiveFindPrimaryForRemoteShard() throws Exception {
630 LOG.info("testOnReceiveFindPrimaryForRemoteShard starting");
631 String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
633 // Create an ActorSystem ShardManager actor for member-1.
635 final ActorSystem system1 = newActorSystem("Member1");
636 Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
638 final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
639 newTestShardMgrBuilderWithMockShardActor().cluster(
640 new ClusterWrapperImpl(system1)).props().withDispatcher(
641 Dispatchers.DefaultDispatcherId()), shardManagerID);
643 // Create an ActorSystem ShardManager actor for member-2.
645 final ActorSystem system2 = newActorSystem("Member2");
647 Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
649 final ActorRef mockShardActor2 = newMockShardActor(system2, "astronauts", "member-2");
651 MockConfiguration mockConfig2 = new MockConfiguration(
652 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
653 .put("astronauts", Arrays.asList("member-2")).build());
655 final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
656 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
657 new ClusterWrapperImpl(system2)).props().withDispatcher(
658 Dispatchers.DefaultDispatcherId()), shardManagerID);
660 new JavaTestKit(system1) {
662 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
663 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
665 shardManager2.tell(new ActorInitialized(), mockShardActor2);
667 String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
668 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
670 new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
673 new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
676 shardManager1.underlyingActor().waitForMemberUp();
677 shardManager1.tell(new FindPrimary("astronauts", false), getRef());
679 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
680 String path = found.getPrimaryPath();
681 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-astronauts-config"));
682 assertEquals("getPrimaryVersion", leaderVersion, found.getPrimaryVersion());
684 shardManager2.underlyingActor().verifyFindPrimary();
686 Cluster.get(system2).down(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
688 shardManager1.underlyingActor().waitForMemberRemoved();
690 shardManager1.tell(new FindPrimary("astronauts", false), getRef());
692 expectMsgClass(duration("5 seconds"), PrimaryNotFoundException.class);
696 LOG.info("testOnReceiveFindPrimaryForRemoteShard ending");
700 public void testShardAvailabilityOnChangeOfMemberReachability() throws Exception {
701 LOG.info("testShardAvailabilityOnChangeOfMemberReachability starting");
702 String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
704 // Create an ActorSystem ShardManager actor for member-1.
706 final ActorSystem system1 = newActorSystem("Member1");
707 Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
709 final ActorRef mockShardActor1 = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
711 final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
712 newTestShardMgrBuilder().shardActor(mockShardActor1).cluster(
713 new ClusterWrapperImpl(system1)).props().withDispatcher(
714 Dispatchers.DefaultDispatcherId()), shardManagerID);
716 // Create an ActorSystem ShardManager actor for member-2.
718 final ActorSystem system2 = newActorSystem("Member2");
720 Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
722 final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
724 MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
725 .put("default", Arrays.asList("member-1", "member-2")).build());
727 final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
728 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
729 new ClusterWrapperImpl(system2)).props().withDispatcher(
730 Dispatchers.DefaultDispatcherId()), shardManagerID);
732 new JavaTestKit(system1) {
734 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
735 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
736 shardManager1.tell(new ActorInitialized(), mockShardActor1);
737 shardManager2.tell(new ActorInitialized(), mockShardActor2);
739 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
740 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
741 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class),
742 DataStoreVersions.CURRENT_VERSION), mockShardActor1);
744 new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
746 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class),
747 DataStoreVersions.CURRENT_VERSION), mockShardActor2);
749 new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
751 shardManager1.underlyingActor().waitForMemberUp();
753 shardManager1.tell(new FindPrimary("default", true), getRef());
755 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
756 String path = found.getPrimaryPath();
757 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
759 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
760 "akka://cluster-test@127.0.0.1:2558"), getRef());
762 shardManager1.underlyingActor().waitForUnreachableMember();
764 PeerDown peerDown = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
765 assertEquals("getMemberName", MEMBER_2, peerDown.getMemberName());
766 MessageCollectorActor.clearMessages(mockShardActor1);
769 MockClusterWrapper.createMemberRemoved("member-2", "akka://cluster-test@127.0.0.1:2558"),
772 MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
774 shardManager1.tell(new FindPrimary("default", true), getRef());
776 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
779 MockClusterWrapper.createReachableMember("member-2", "akka://cluster-test@127.0.0.1:2558"),
782 shardManager1.underlyingActor().waitForReachableMember();
784 PeerUp peerUp = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
785 assertEquals("getMemberName", MEMBER_2, peerUp.getMemberName());
786 MessageCollectorActor.clearMessages(mockShardActor1);
788 shardManager1.tell(new FindPrimary("default", true), getRef());
790 RemotePrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
791 String path1 = found1.getPrimaryPath();
792 assertTrue("Unexpected primary path " + path1, path1.contains("member-2-shard-default-config"));
795 MockClusterWrapper.createMemberUp("member-2", "akka://cluster-test@127.0.0.1:2558"),
798 MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
800 // Test FindPrimary wait succeeds after reachable member event.
802 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
803 "akka://cluster-test@127.0.0.1:2558"), getRef());
804 shardManager1.underlyingActor().waitForUnreachableMember();
806 shardManager1.tell(new FindPrimary("default", true), getRef());
809 MockClusterWrapper.createReachableMember("member-2", "akka://cluster-test@127.0.0.1:2558"),
812 RemotePrimaryShardFound found2 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
813 String path2 = found2.getPrimaryPath();
814 assertTrue("Unexpected primary path " + path2, path2.contains("member-2-shard-default-config"));
818 LOG.info("testShardAvailabilityOnChangeOfMemberReachability ending");
822 public void testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange() throws Exception {
823 LOG.info("testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange starting");
824 String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
826 // Create an ActorSystem ShardManager actor for member-1.
828 final ActorSystem system1 = newActorSystem("Member1");
829 Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
831 final ActorRef mockShardActor1 = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
833 final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
834 final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
835 newTestShardMgrBuilder().shardActor(mockShardActor1).cluster(new ClusterWrapperImpl(system1))
836 .primaryShardInfoCache(primaryShardInfoCache).props()
837 .withDispatcher(Dispatchers.DefaultDispatcherId()),
840 // Create an ActorSystem ShardManager actor for member-2.
842 final ActorSystem system2 = newActorSystem("Member2");
844 Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
846 final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
848 MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
849 .put("default", Arrays.asList("member-1", "member-2")).build());
851 final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
852 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
853 new ClusterWrapperImpl(system2)).props().withDispatcher(
854 Dispatchers.DefaultDispatcherId()), shardManagerID);
856 new JavaTestKit(system1) {
858 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
859 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
860 shardManager1.tell(new ActorInitialized(), mockShardActor1);
861 shardManager2.tell(new ActorInitialized(), mockShardActor2);
863 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
864 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
865 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class),
866 DataStoreVersions.CURRENT_VERSION), mockShardActor1);
868 new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
870 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class),
871 DataStoreVersions.CURRENT_VERSION), mockShardActor2);
873 new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
875 shardManager1.underlyingActor().waitForMemberUp();
877 shardManager1.tell(new FindPrimary("default", true), getRef());
879 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
880 String path = found.getPrimaryPath();
881 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
883 primaryShardInfoCache.putSuccessful("default", new PrimaryShardInfo(
884 system1.actorSelection(mockShardActor1.path()), DataStoreVersions.CURRENT_VERSION));
886 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
887 "akka://cluster-test@127.0.0.1:2558"), getRef());
889 shardManager1.underlyingActor().waitForUnreachableMember();
891 shardManager1.tell(new FindPrimary("default", true), getRef());
893 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
895 assertNull("Expected primaryShardInfoCache entry removed",
896 primaryShardInfoCache.getIfPresent("default"));
898 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId1, mock(DataTree.class),
899 DataStoreVersions.CURRENT_VERSION), mockShardActor1);
901 new RoleChangeNotification(memberId1, RaftState.Follower.name(), RaftState.Leader.name()),
904 shardManager1.tell(new FindPrimary("default", true), getRef());
906 LocalPrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
907 String path1 = found1.getPrimaryPath();
908 assertTrue("Unexpected primary path " + path1, path1.contains("member-1-shard-default-config"));
913 LOG.info("testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange ending");
918 public void testOnReceiveFindLocalShardForNonExistentShard() throws Exception {
919 new JavaTestKit(getSystem()) {
921 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
923 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
925 shardManager.tell(new FindLocalShard("non-existent", false), getRef());
927 LocalShardNotFound notFound = expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
929 assertEquals("getShardName", "non-existent", notFound.getShardName());
935 public void testOnReceiveFindLocalShardForExistentShard() throws Exception {
936 new JavaTestKit(getSystem()) {
938 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
940 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
941 shardManager.tell(new ActorInitialized(), mockShardActor);
943 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
945 LocalShardFound found = expectMsgClass(duration("5 seconds"), LocalShardFound.class);
947 assertTrue("Found path contains " + found.getPath().path().toString(),
948 found.getPath().path().toString().contains("member-1-shard-default-config"));
954 public void testOnReceiveFindLocalShardForNotInitializedShard() throws Exception {
955 new JavaTestKit(getSystem()) {
957 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
959 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
961 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
967 public void testOnReceiveFindLocalShardWaitForShardInitialized() throws Exception {
968 LOG.info("testOnReceiveFindLocalShardWaitForShardInitialized starting");
969 new JavaTestKit(getSystem()) {
971 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
973 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
975 // We're passing waitUntilInitialized = true to FindLocalShard
976 // so the response should be
977 // delayed until we send ActorInitialized.
978 Future<Object> future = Patterns.ask(shardManager, new FindLocalShard(Shard.DEFAULT_NAME, true),
979 new Timeout(5, TimeUnit.SECONDS));
981 shardManager.tell(new ActorInitialized(), mockShardActor);
983 Object resp = Await.result(future, duration("5 seconds"));
984 assertTrue("Expected: LocalShardFound, Actual: " + resp, resp instanceof LocalShardFound);
988 LOG.info("testOnReceiveFindLocalShardWaitForShardInitialized starting");
992 public void testOnRecoveryJournalIsCleaned() {
993 String persistenceID = "shard-manager-" + shardMrgIDSuffix;
994 InMemoryJournal.addEntry(persistenceID, 1L, new SchemaContextModules(ImmutableSet.of("foo")));
995 InMemoryJournal.addEntry(persistenceID, 2L, new SchemaContextModules(ImmutableSet.of("bar")));
996 InMemoryJournal.addDeleteMessagesCompleteLatch(persistenceID);
998 newTestShardManager();
1000 InMemoryJournal.waitForDeleteMessagesComplete(persistenceID);
1002 // Journal entries up to the last one should've been deleted
1003 Map<Long, Object> journal = InMemoryJournal.get(persistenceID);
1004 synchronized (journal) {
1005 assertEquals("Journal size", 0, journal.size());
1010 public void testRoleChangeNotificationAndShardLeaderStateChangedReleaseReady() throws Exception {
1011 TestShardManager shardManager = newTestShardManager();
1013 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1014 shardManager.onReceiveCommand(new RoleChangeNotification(
1015 memberId, RaftState.Candidate.name(), RaftState.Leader.name()));
1017 verify(ready, never()).countDown();
1019 shardManager.onReceiveCommand(new ShardLeaderStateChanged(memberId, memberId,
1020 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1022 verify(ready, times(1)).countDown();
1026 public void testRoleChangeNotificationToFollowerWithShardLeaderStateChangedReleaseReady() throws Exception {
1027 new JavaTestKit(getSystem()) {
1029 TestShardManager shardManager = newTestShardManager();
1031 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1032 shardManager.onReceiveCommand(new RoleChangeNotification(memberId, null, RaftState.Follower.name()));
1034 verify(ready, never()).countDown();
1037 .onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
1039 shardManager.onReceiveCommand(
1040 new ShardLeaderStateChanged(memberId, "member-2-shard-default-" + shardMrgIDSuffix,
1041 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1043 verify(ready, times(1)).countDown();
1049 public void testReadyCountDownForMemberUpAfterLeaderStateChanged() throws Exception {
1050 new JavaTestKit(getSystem()) {
1052 TestShardManager shardManager = newTestShardManager();
1054 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1055 shardManager.onReceiveCommand(new RoleChangeNotification(memberId, null, RaftState.Follower.name()));
1057 verify(ready, never()).countDown();
1059 shardManager.onReceiveCommand(
1060 new ShardLeaderStateChanged(memberId, "member-2-shard-default-" + shardMrgIDSuffix,
1061 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1064 .onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
1066 verify(ready, times(1)).countDown();
1072 public void testRoleChangeNotificationDoNothingForUnknownShard() throws Exception {
1073 TestShardManager shardManager = newTestShardManager();
1075 shardManager.onReceiveCommand(new RoleChangeNotification(
1076 "unknown", RaftState.Candidate.name(), RaftState.Leader.name()));
1078 verify(ready, never()).countDown();
1082 public void testByDefaultSyncStatusIsFalse() throws Exception {
1083 TestShardManager shardManager = newTestShardManager();
1085 assertEquals(false, shardManager.getMBean().getSyncStatus());
1089 public void testWhenShardIsLeaderSyncStatusIsTrue() throws Exception {
1090 TestShardManager shardManager = newTestShardManager();
1092 shardManager.onReceiveCommand(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix,
1093 RaftState.Follower.name(), RaftState.Leader.name()));
1095 assertEquals(true, shardManager.getMBean().getSyncStatus());
1099 public void testWhenShardIsCandidateSyncStatusIsFalse() throws Exception {
1100 TestShardManager shardManager = newTestShardManager();
1102 String shardId = "member-1-shard-default-" + shardMrgIDSuffix;
1103 shardManager.onReceiveCommand(new RoleChangeNotification(shardId,
1104 RaftState.Follower.name(), RaftState.Candidate.name()));
1106 assertEquals(false, shardManager.getMBean().getSyncStatus());
1108 // Send a FollowerInitialSyncStatus with status = true for the replica whose current state is candidate
1109 shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(
1112 assertEquals(false, shardManager.getMBean().getSyncStatus());
1116 public void testWhenShardIsFollowerSyncStatusDependsOnFollowerInitialSyncStatus() throws Exception {
1117 TestShardManager shardManager = newTestShardManager();
1119 String shardId = "member-1-shard-default-" + shardMrgIDSuffix;
1120 shardManager.onReceiveCommand(new RoleChangeNotification(shardId,
1121 RaftState.Candidate.name(), RaftState.Follower.name()));
1123 // Initially will be false
1124 assertEquals(false, shardManager.getMBean().getSyncStatus());
1126 // Send status true will make sync status true
1127 shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(true, shardId));
1129 assertEquals(true, shardManager.getMBean().getSyncStatus());
1131 // Send status false will make sync status false
1132 shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(false, shardId));
1134 assertEquals(false, shardManager.getMBean().getSyncStatus());
1139 public void testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards() throws Exception {
1140 LOG.info("testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards starting");
1141 TestShardManager shardManager = newTestShardManager(newShardMgrProps(new MockConfiguration() {
1143 public List<String> getMemberShardNames(MemberName memberName) {
1144 return Arrays.asList("default", "astronauts");
1148 // Initially will be false
1149 assertEquals(false, shardManager.getMBean().getSyncStatus());
1151 // Make default shard leader
1152 String defaultShardId = "member-1-shard-default-" + shardMrgIDSuffix;
1153 shardManager.onReceiveCommand(new RoleChangeNotification(defaultShardId,
1154 RaftState.Follower.name(), RaftState.Leader.name()));
1156 // default = Leader, astronauts is unknown so sync status remains false
1157 assertEquals(false, shardManager.getMBean().getSyncStatus());
1159 // Make astronauts shard leader as well
1160 String astronautsShardId = "member-1-shard-astronauts-" + shardMrgIDSuffix;
1161 shardManager.onReceiveCommand(new RoleChangeNotification(astronautsShardId,
1162 RaftState.Follower.name(), RaftState.Leader.name()));
1164 // Now sync status should be true
1165 assertEquals(true, shardManager.getMBean().getSyncStatus());
1167 // Make astronauts a Follower
1168 shardManager.onReceiveCommand(new RoleChangeNotification(astronautsShardId,
1169 RaftState.Leader.name(), RaftState.Follower.name()));
1171 // Sync status is not true
1172 assertEquals(false, shardManager.getMBean().getSyncStatus());
1174 // Make the astronauts follower sync status true
1175 shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(true, astronautsShardId));
1177 // Sync status is now true
1178 assertEquals(true, shardManager.getMBean().getSyncStatus());
1180 LOG.info("testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards ending");
1184 public void testOnReceiveSwitchShardBehavior() throws Exception {
1185 new JavaTestKit(getSystem()) {
1187 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1189 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1190 shardManager.tell(new ActorInitialized(), mockShardActor);
1192 shardManager.tell(new SwitchShardBehavior(mockShardName, RaftState.Leader, 1000), getRef());
1194 SwitchBehavior switchBehavior = MessageCollectorActor.expectFirstMatching(mockShardActor,
1195 SwitchBehavior.class);
1197 assertEquals(RaftState.Leader, switchBehavior.getNewState());
1198 assertEquals(1000, switchBehavior.getNewTerm());
1203 private static List<MemberName> members(String... names) {
1204 return Arrays.asList(names).stream().map(MemberName::forName).collect(Collectors.toList());
1208 public void testOnCreateShard() {
1209 LOG.info("testOnCreateShard starting");
1210 new JavaTestKit(getSystem()) {
1212 datastoreContextBuilder.shardInitializationTimeout(1, TimeUnit.MINUTES).persistent(true);
1214 ActorRef shardManager = actorFactory
1215 .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1217 SchemaContext schemaContext = TestModel.createTestContext();
1218 shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
1220 DatastoreContext datastoreContext = DatastoreContext.newBuilder().shardElectionTimeoutFactor(100)
1221 .persistent(false).build();
1222 Shard.Builder shardBuilder = Shard.builder();
1224 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1225 "foo", null, members("member-1", "member-5", "member-6"));
1226 shardManager.tell(new CreateShard(config, shardBuilder, datastoreContext), getRef());
1228 expectMsgClass(duration("5 seconds"), Success.class);
1230 shardManager.tell(new FindLocalShard("foo", true), getRef());
1232 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1234 assertEquals("isRecoveryApplicable", false, shardBuilder.getDatastoreContext().isPersistent());
1235 assertTrue("Epxected ShardPeerAddressResolver", shardBuilder.getDatastoreContext().getShardRaftConfig()
1236 .getPeerAddressResolver() instanceof ShardPeerAddressResolver);
1237 assertEquals("peerMembers", Sets.newHashSet(
1238 ShardIdentifier.create("foo", MemberName.forName("member-5"), shardMrgIDSuffix).toString(),
1239 ShardIdentifier.create("foo", MemberName.forName("member-6"), shardMrgIDSuffix).toString()),
1240 shardBuilder.getPeerAddresses().keySet());
1241 assertEquals("ShardIdentifier", ShardIdentifier.create("foo", MEMBER_1, shardMrgIDSuffix),
1242 shardBuilder.getId());
1243 assertSame("schemaContext", schemaContext, shardBuilder.getSchemaContext());
1245 // Send CreateShard with same name - should return Success with
1248 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1250 Success success = expectMsgClass(duration("5 seconds"), Success.class);
1251 assertNotNull("Success status is null", success.status());
1255 LOG.info("testOnCreateShard ending");
1259 public void testOnCreateShardWithLocalMemberNotInShardConfig() {
1260 LOG.info("testOnCreateShardWithLocalMemberNotInShardConfig starting");
1261 new JavaTestKit(getSystem()) {
1263 datastoreContextBuilder.shardInitializationTimeout(1, TimeUnit.MINUTES).persistent(true);
1265 ActorRef shardManager = actorFactory
1266 .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1268 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1270 Shard.Builder shardBuilder = Shard.builder();
1271 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1272 "foo", null, members("member-5", "member-6"));
1274 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1275 expectMsgClass(duration("5 seconds"), Success.class);
1277 shardManager.tell(new FindLocalShard("foo", true), getRef());
1278 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1280 assertEquals("peerMembers size", 0, shardBuilder.getPeerAddresses().size());
1281 assertEquals("schemaContext", DisableElectionsRaftPolicy.class.getName(), shardBuilder
1282 .getDatastoreContext().getShardRaftConfig().getCustomRaftPolicyImplementationClass());
1286 LOG.info("testOnCreateShardWithLocalMemberNotInShardConfig ending");
1290 public void testOnCreateShardWithNoInitialSchemaContext() {
1291 LOG.info("testOnCreateShardWithNoInitialSchemaContext starting");
1292 new JavaTestKit(getSystem()) {
1294 ActorRef shardManager = actorFactory
1295 .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1297 Shard.Builder shardBuilder = Shard.builder();
1299 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1300 "foo", null, members("member-1"));
1301 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1303 expectMsgClass(duration("5 seconds"), Success.class);
1305 SchemaContext schemaContext = TestModel.createTestContext();
1306 shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
1308 shardManager.tell(new FindLocalShard("foo", true), getRef());
1310 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1312 assertSame("schemaContext", schemaContext, shardBuilder.getSchemaContext());
1313 assertNotNull("schemaContext is null", shardBuilder.getDatastoreContext());
1317 LOG.info("testOnCreateShardWithNoInitialSchemaContext ending");
1321 public void testGetSnapshot() throws Exception {
1322 LOG.info("testGetSnapshot starting");
1323 JavaTestKit kit = new JavaTestKit(getSystem());
1325 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1326 .put("shard1", Arrays.asList("member-1")).put("shard2", Arrays.asList("member-1"))
1327 .put("astronauts", Collections.<String>emptyList()).build());
1329 TestActorRef<TestShardManager> shardManager = actorFactory
1330 .createTestActor(newShardMgrProps(mockConfig).withDispatcher(Dispatchers.DefaultDispatcherId()));
1332 shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1333 Failure failure = kit.expectMsgClass(Failure.class);
1334 assertEquals("Failure cause type", IllegalStateException.class, failure.cause().getClass());
1336 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1338 waitForShardInitialized(shardManager, "shard1", kit);
1339 waitForShardInitialized(shardManager, "shard2", kit);
1341 shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1343 DatastoreSnapshot datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1345 assertEquals("getType", shardMrgIDSuffix, datastoreSnapshot.getType());
1346 assertNull("Expected null ShardManagerSnapshot", datastoreSnapshot.getShardManagerSnapshot());
1348 Function<ShardSnapshot, String> shardNameTransformer = s -> s.getName();
1350 assertEquals("Shard names", Sets.newHashSet("shard1", "shard2"), Sets.newHashSet(
1351 Lists.transform(datastoreSnapshot.getShardSnapshots(), shardNameTransformer)));
1353 // Add a new replica
1355 JavaTestKit mockShardLeaderKit = new JavaTestKit(getSystem());
1357 TestShardManager shardManagerInstance = shardManager.underlyingActor();
1358 shardManagerInstance.setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1360 shardManager.tell(new AddShardReplica("astronauts"), kit.getRef());
1361 mockShardLeaderKit.expectMsgClass(AddServer.class);
1362 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.OK, ""));
1363 kit.expectMsgClass(Status.Success.class);
1364 waitForShardInitialized(shardManager, "astronauts", kit);
1366 // Send another GetSnapshot and verify
1368 shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1369 datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1371 assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"), Sets.newHashSet(
1372 Lists.transform(datastoreSnapshot.getShardSnapshots(), shardNameTransformer)));
1374 ShardManagerSnapshot snapshot = datastoreSnapshot.getShardManagerSnapshot();
1375 assertNotNull("Expected ShardManagerSnapshot", snapshot);
1376 assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"),
1377 Sets.newHashSet(snapshot.getShardList()));
1379 LOG.info("testGetSnapshot ending");
1383 public void testRestoreFromSnapshot() throws Exception {
1384 LOG.info("testRestoreFromSnapshot starting");
1386 datastoreContextBuilder.shardInitializationTimeout(3, TimeUnit.SECONDS);
1388 JavaTestKit kit = new JavaTestKit(getSystem());
1390 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1391 .put("shard1", Collections.<String>emptyList()).put("shard2", Collections.<String>emptyList())
1392 .put("astronauts", Collections.<String>emptyList()).build());
1394 ShardManagerSnapshot snapshot =
1395 new ShardManagerSnapshot(Arrays.asList("shard1", "shard2", "astronauts"), Collections.emptyMap());
1396 DatastoreSnapshot restoreFromSnapshot = new DatastoreSnapshot(shardMrgIDSuffix, snapshot,
1397 Collections.<ShardSnapshot>emptyList());
1398 TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(newTestShardMgrBuilder(mockConfig)
1399 .restoreFromSnapshot(restoreFromSnapshot).props().withDispatcher(Dispatchers.DefaultDispatcherId()));
1401 shardManager.underlyingActor().waitForRecoveryComplete();
1403 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1405 waitForShardInitialized(shardManager, "shard1", kit);
1406 waitForShardInitialized(shardManager, "shard2", kit);
1407 waitForShardInitialized(shardManager, "astronauts", kit);
1409 shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1411 DatastoreSnapshot datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1413 assertEquals("getType", shardMrgIDSuffix, datastoreSnapshot.getType());
1415 assertNotNull("Expected ShardManagerSnapshot", datastoreSnapshot.getShardManagerSnapshot());
1416 assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"),
1417 Sets.newHashSet(datastoreSnapshot.getShardManagerSnapshot().getShardList()));
1419 LOG.info("testRestoreFromSnapshot ending");
1423 public void testAddShardReplicaForNonExistentShardConfig() throws Exception {
1424 new JavaTestKit(getSystem()) {
1426 ActorRef shardManager = actorFactory
1427 .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1429 shardManager.tell(new AddShardReplica("model-inventory"), getRef());
1430 Status.Failure resp = expectMsgClass(duration("2 seconds"), Status.Failure.class);
1432 assertEquals("Failure obtained", true, resp.cause() instanceof IllegalArgumentException);
1438 public void testAddShardReplica() throws Exception {
1439 LOG.info("testAddShardReplica starting");
1440 MockConfiguration mockConfig = new MockConfiguration(
1441 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
1442 .put("astronauts", Arrays.asList("member-2")).build());
1444 final String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
1445 datastoreContextBuilder.shardManagerPersistenceId(shardManagerID);
1447 // Create an ActorSystem ShardManager actor for member-1.
1448 final ActorSystem system1 = newActorSystem("Member1");
1449 Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1450 ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
1451 final TestActorRef<TestShardManager> newReplicaShardManager = TestActorRef.create(system1,
1452 newTestShardMgrBuilder(mockConfig).shardActor(mockDefaultShardActor)
1453 .cluster(new ClusterWrapperImpl(system1)).props()
1454 .withDispatcher(Dispatchers.DefaultDispatcherId()),
1457 // Create an ActorSystem ShardManager actor for member-2.
1458 final ActorSystem system2 = newActorSystem("Member2");
1459 Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1461 String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
1462 String name = ShardIdentifier.create("astronauts", MEMBER_2, "config").toString();
1463 final TestActorRef<MockRespondActor> mockShardLeaderActor = TestActorRef.create(system2,
1464 Props.create(MockRespondActor.class, AddServer.class,
1465 new AddServerReply(ServerChangeStatus.OK, memberId2))
1466 .withDispatcher(Dispatchers.DefaultDispatcherId()),
1468 final TestActorRef<TestShardManager> leaderShardManager = TestActorRef.create(system2,
1469 newTestShardMgrBuilder(mockConfig).shardActor(mockShardLeaderActor)
1470 .cluster(new ClusterWrapperImpl(system2)).props()
1471 .withDispatcher(Dispatchers.DefaultDispatcherId()),
1474 new JavaTestKit(system1) {
1476 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1477 leaderShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1479 leaderShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1481 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
1482 leaderShardManager.tell(
1483 new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
1484 mockShardLeaderActor);
1485 leaderShardManager.tell(
1486 new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
1487 mockShardLeaderActor);
1489 newReplicaShardManager.underlyingActor().waitForMemberUp();
1490 leaderShardManager.underlyingActor().waitForMemberUp();
1492 // Have a dummy snapshot to be overwritten by the new data
1494 String[] restoredShards = { "default", "people" };
1495 ShardManagerSnapshot snapshot =
1496 new ShardManagerSnapshot(Arrays.asList(restoredShards), Collections.emptyMap());
1497 InMemorySnapshotStore.addSnapshot(shardManagerID, snapshot);
1498 Uninterruptibles.sleepUninterruptibly(2, TimeUnit.MILLISECONDS);
1500 InMemorySnapshotStore.addSnapshotSavedLatch(shardManagerID);
1501 InMemorySnapshotStore.addSnapshotDeletedLatch(shardManagerID);
1503 // construct a mock response message
1504 newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
1505 AddServer addServerMsg = MessageCollectorActor.expectFirstMatching(mockShardLeaderActor,
1507 String addServerId = "member-1-shard-astronauts-" + shardMrgIDSuffix;
1508 assertEquals("AddServer serverId", addServerId, addServerMsg.getNewServerId());
1509 expectMsgClass(duration("5 seconds"), Status.Success.class);
1511 InMemorySnapshotStore.waitForSavedSnapshot(shardManagerID, ShardManagerSnapshot.class);
1512 InMemorySnapshotStore.waitForDeletedSnapshot(shardManagerID);
1513 List<ShardManagerSnapshot> persistedSnapshots = InMemorySnapshotStore.getSnapshots(shardManagerID,
1514 ShardManagerSnapshot.class);
1515 assertEquals("Number of snapshots persisted", 1, persistedSnapshots.size());
1516 ShardManagerSnapshot shardManagerSnapshot = persistedSnapshots.get(0);
1517 assertEquals("Persisted local shards", Sets.newHashSet("default", "astronauts"),
1518 Sets.newHashSet(shardManagerSnapshot.getShardList()));
1521 LOG.info("testAddShardReplica ending");
1525 public void testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader() throws Exception {
1526 LOG.info("testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader starting");
1527 new JavaTestKit(getSystem()) {
1529 TestActorRef<TestShardManager> shardManager = actorFactory
1530 .createTestActor(newPropsShardMgrWithMockShardActor(), shardMgrID);
1532 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1533 shardManager.tell(new ActorInitialized(), mockShardActor);
1535 String leaderId = "leader-member-shard-default-" + shardMrgIDSuffix;
1536 AddServerReply addServerReply = new AddServerReply(ServerChangeStatus.ALREADY_EXISTS, null);
1537 ActorRef leaderShardActor = shardManager.underlyingActor().getContext()
1538 .actorOf(Props.create(MockRespondActor.class, AddServer.class, addServerReply), leaderId);
1540 MockClusterWrapper.sendMemberUp(shardManager, "leader-member", leaderShardActor.path().toString());
1542 String newReplicaId = "member-1-shard-default-" + shardMrgIDSuffix;
1544 new RoleChangeNotification(newReplicaId, RaftState.Candidate.name(), RaftState.Follower.name()),
1547 new ShardLeaderStateChanged(newReplicaId, leaderId, DataStoreVersions.CURRENT_VERSION),
1550 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1552 MessageCollectorActor.expectFirstMatching(leaderShardActor, AddServer.class);
1554 Failure resp = expectMsgClass(duration("5 seconds"), Failure.class);
1555 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1557 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1558 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1560 // Send message again to verify previous in progress state is
1563 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1564 resp = expectMsgClass(duration("5 seconds"), Failure.class);
1565 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1567 // Send message again with an AddServer timeout to verify the
1568 // pre-existing shard actor isn't terminated.
1571 newDatastoreContextFactory(
1572 datastoreContextBuilder.shardLeaderElectionTimeout(100, TimeUnit.MILLISECONDS).build()),
1574 leaderShardActor.tell(MockRespondActor.CLEAR_RESPONSE, ActorRef.noSender());
1575 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1576 expectMsgClass(duration("5 seconds"), Failure.class);
1578 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1579 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1583 LOG.info("testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader ending");
1587 public void testAddShardReplicaWithPreExistingLocalReplicaLeader() throws Exception {
1588 LOG.info("testAddShardReplicaWithPreExistingLocalReplicaLeader starting");
1589 new JavaTestKit(getSystem()) {
1591 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1592 ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1594 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1595 shardManager.tell(new ActorInitialized(), mockShardActor);
1596 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
1597 DataStoreVersions.CURRENT_VERSION), getRef());
1599 new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
1602 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1603 Failure resp = expectMsgClass(duration("5 seconds"), Failure.class);
1604 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1606 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1607 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1611 LOG.info("testAddShardReplicaWithPreExistingLocalReplicaLeader ending");
1615 public void testAddShardReplicaWithAddServerReplyFailure() throws Exception {
1616 LOG.info("testAddShardReplicaWithAddServerReplyFailure starting");
1617 new JavaTestKit(getSystem()) {
1619 JavaTestKit mockShardLeaderKit = new JavaTestKit(getSystem());
1621 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1622 .put("astronauts", Arrays.asList("member-2")).build());
1624 ActorRef mockNewReplicaShardActor = newMockShardActor(getSystem(), "astronauts", "member-1");
1625 final TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(
1626 newTestShardMgrBuilder(mockConfig).shardActor(mockNewReplicaShardActor).props(), shardMgrID);
1627 shardManager.underlyingActor()
1628 .setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1630 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1632 JavaTestKit terminateWatcher = new JavaTestKit(getSystem());
1633 terminateWatcher.watch(mockNewReplicaShardActor);
1635 shardManager.tell(new AddShardReplica("astronauts"), getRef());
1637 AddServer addServerMsg = mockShardLeaderKit.expectMsgClass(AddServer.class);
1638 assertEquals("AddServer serverId", "member-1-shard-astronauts-" + shardMrgIDSuffix,
1639 addServerMsg.getNewServerId());
1640 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.TIMEOUT, null));
1642 Failure failure = expectMsgClass(duration("5 seconds"), Failure.class);
1643 assertEquals("Failure cause", TimeoutException.class, failure.cause().getClass());
1645 shardManager.tell(new FindLocalShard("astronauts", false), getRef());
1646 expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
1648 terminateWatcher.expectTerminated(mockNewReplicaShardActor);
1650 shardManager.tell(new AddShardReplica("astronauts"), getRef());
1651 mockShardLeaderKit.expectMsgClass(AddServer.class);
1652 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.NO_LEADER, null));
1653 failure = expectMsgClass(duration("5 seconds"), Failure.class);
1654 assertEquals("Failure cause", NoShardLeaderException.class, failure.cause().getClass());
1658 LOG.info("testAddShardReplicaWithAddServerReplyFailure ending");
1662 public void testAddShardReplicaWithAlreadyInProgress() throws Exception {
1663 testServerChangeWhenAlreadyInProgress("astronauts", new AddShardReplica("astronauts"),
1664 AddServer.class, new AddShardReplica("astronauts"));
1668 public void testAddShardReplicaWithFindPrimaryTimeout() throws Exception {
1669 LOG.info("testAddShardReplicaWithFindPrimaryTimeout starting");
1670 datastoreContextBuilder.shardInitializationTimeout(100, TimeUnit.MILLISECONDS);
1671 new JavaTestKit(getSystem()) {
1673 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1674 .put("astronauts", Arrays.asList("member-2")).build());
1676 final ActorRef newReplicaShardManager = actorFactory
1677 .createActor(newTestShardMgrBuilder(mockConfig).shardActor(mockShardActor).props(), shardMgrID);
1679 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1680 MockClusterWrapper.sendMemberUp(newReplicaShardManager, "member-2",
1681 AddressFromURIString.parse("akka://non-existent@127.0.0.1:5").toString());
1683 newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
1684 Status.Failure resp = expectMsgClass(duration("5 seconds"), Status.Failure.class);
1685 assertEquals("Failure obtained", true, resp.cause() instanceof RuntimeException);
1689 LOG.info("testAddShardReplicaWithFindPrimaryTimeout ending");
1693 public void testRemoveShardReplicaForNonExistentShard() throws Exception {
1694 new JavaTestKit(getSystem()) {
1696 ActorRef shardManager = actorFactory
1697 .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1699 shardManager.tell(new RemoveShardReplica("model-inventory", MEMBER_1), getRef());
1700 Status.Failure resp = expectMsgClass(duration("10 seconds"), Status.Failure.class);
1701 assertEquals("Failure obtained", true, resp.cause() instanceof PrimaryNotFoundException);
1710 public void testRemoveShardReplicaLocal() throws Exception {
1711 new JavaTestKit(getSystem()) {
1713 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1715 final TestActorRef<MockRespondActor> respondActor = actorFactory
1716 .createTestActor(Props.create(MockRespondActor.class, RemoveServer.class,
1717 new RemoveServerReply(ServerChangeStatus.OK, null)), memberId);
1719 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
1721 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1722 shardManager.tell(new ActorInitialized(), respondActor);
1723 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
1724 DataStoreVersions.CURRENT_VERSION), getRef());
1726 new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
1729 shardManager.tell(new RemoveShardReplica(Shard.DEFAULT_NAME, MEMBER_1), getRef());
1730 final RemoveServer removeServer = MessageCollectorActor.expectFirstMatching(respondActor,
1731 RemoveServer.class);
1732 assertEquals(ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString(),
1733 removeServer.getServerId());
1734 expectMsgClass(duration("5 seconds"), Success.class);
1740 public void testRemoveShardReplicaRemote() throws Exception {
1741 MockConfiguration mockConfig = new MockConfiguration(
1742 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
1743 .put("astronauts", Arrays.asList("member-1")).build());
1745 String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
1747 // Create an ActorSystem ShardManager actor for member-1.
1748 final ActorSystem system1 = newActorSystem("Member1");
1749 Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1750 ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
1752 final TestActorRef<TestShardManager> newReplicaShardManager = TestActorRef.create(system1,
1753 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockDefaultShardActor).cluster(
1754 new ClusterWrapperImpl(system1)).props().withDispatcher(Dispatchers.DefaultDispatcherId()),
1757 // Create an ActorSystem ShardManager actor for member-2.
1758 final ActorSystem system2 = newActorSystem("Member2");
1759 Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1761 String name = ShardIdentifier.create("default", MEMBER_2, shardMrgIDSuffix).toString();
1762 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
1763 final TestActorRef<MockRespondActor> mockShardLeaderActor =
1764 TestActorRef.create(system2, Props.create(MockRespondActor.class, RemoveServer.class,
1765 new RemoveServerReply(ServerChangeStatus.OK, memberId2)), name);
1767 LOG.error("Mock Shard Leader Actor : {}", mockShardLeaderActor);
1769 final TestActorRef<TestShardManager> leaderShardManager = TestActorRef.create(system2,
1770 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockShardLeaderActor).cluster(
1771 new ClusterWrapperImpl(system2)).props().withDispatcher(Dispatchers.DefaultDispatcherId()),
1774 // Because mockShardLeaderActor is created at the top level of the actor system it has an address like so,
1775 // akka://cluster-test@127.0.0.1:2559/user/member-2-shard-default-config1
1776 // However when a shard manager has a local shard which is a follower and a leader that is remote it will
1777 // try to compute an address for the remote shard leader using the ShardPeerAddressResolver. This address will
1779 // akka://cluster-test@127.0.0.1:2559/user/shardmanager-config1/member-2-shard-default-config1
1780 // In this specific case if we did a FindPrimary for shard default from member-1 we would come up
1781 // with the address of an actor which does not exist, therefore any message sent to that actor would go to
1783 // To work around this problem we create a ForwardingActor with the right address and pass to it the
1784 // mockShardLeaderActor. The ForwardingActor simply forwards all messages to the mockShardLeaderActor and every
1785 // thing works as expected
1786 final ActorRef actorRef = leaderShardManager.underlyingActor().context()
1787 .actorOf(Props.create(ForwardingActor.class, mockShardLeaderActor),
1788 "member-2-shard-default-" + shardMrgIDSuffix);
1790 LOG.error("Forwarding actor : {}", actorRef);
1792 new JavaTestKit(system1) {
1794 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1795 leaderShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1797 leaderShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1798 newReplicaShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1800 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
1801 leaderShardManager.tell(
1802 new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
1803 mockShardLeaderActor);
1804 leaderShardManager.tell(
1805 new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
1806 mockShardLeaderActor);
1808 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
1809 newReplicaShardManager.tell(
1810 new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class), leaderVersion),
1812 newReplicaShardManager.tell(
1813 new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
1816 newReplicaShardManager.underlyingActor().waitForMemberUp();
1817 leaderShardManager.underlyingActor().waitForMemberUp();
1819 // construct a mock response message
1820 newReplicaShardManager.tell(new RemoveShardReplica("default", MEMBER_1), getRef());
1821 RemoveServer removeServer = MessageCollectorActor.expectFirstMatching(mockShardLeaderActor,
1822 RemoveServer.class);
1823 String removeServerId = ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString();
1824 assertEquals("RemoveServer serverId", removeServerId, removeServer.getServerId());
1825 expectMsgClass(duration("5 seconds"), Status.Success.class);
1831 public void testRemoveShardReplicaWhenAnotherRemoveShardReplicaAlreadyInProgress() throws Exception {
1832 testServerChangeWhenAlreadyInProgress("astronauts", new RemoveShardReplica("astronauts", MEMBER_2),
1833 RemoveServer.class, new RemoveShardReplica("astronauts", MEMBER_3));
1837 public void testRemoveShardReplicaWhenAddShardReplicaAlreadyInProgress() throws Exception {
1838 testServerChangeWhenAlreadyInProgress("astronauts", new AddShardReplica("astronauts"),
1839 AddServer.class, new RemoveShardReplica("astronauts", MEMBER_2));
1843 public void testServerChangeWhenAlreadyInProgress(final String shardName, final Object firstServerChange,
1844 final Class<?> firstForwardedServerChangeClass,
1845 final Object secondServerChange) throws Exception {
1846 new JavaTestKit(getSystem()) {
1848 JavaTestKit mockShardLeaderKit = new JavaTestKit(getSystem());
1849 final JavaTestKit secondRequestKit = new JavaTestKit(getSystem());
1851 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1852 .put(shardName, Arrays.asList("member-2")).build());
1854 final TestActorRef<TestShardManager> shardManager = TestActorRef.create(getSystem(),
1855 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockShardActor)
1856 .cluster(new MockClusterWrapper()).props()
1857 .withDispatcher(Dispatchers.DefaultDispatcherId()),
1860 shardManager.underlyingActor()
1861 .setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1863 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1865 shardManager.tell(firstServerChange, getRef());
1867 mockShardLeaderKit.expectMsgClass(firstForwardedServerChangeClass);
1869 shardManager.tell(secondServerChange, secondRequestKit.getRef());
1871 secondRequestKit.expectMsgClass(duration("5 seconds"), Failure.class);
1877 public void testServerRemovedShardActorNotRunning() throws Exception {
1878 LOG.info("testServerRemovedShardActorNotRunning starting");
1879 new JavaTestKit(getSystem()) {
1881 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1882 .put("default", Arrays.asList("member-1", "member-2"))
1883 .put("astronauts", Arrays.asList("member-2"))
1884 .put("people", Arrays.asList("member-1", "member-2")).build());
1886 TestActorRef<TestShardManager> shardManager = actorFactory
1887 .createTestActor(newShardMgrProps(mockConfig));
1889 shardManager.underlyingActor().waitForRecoveryComplete();
1890 shardManager.tell(new FindLocalShard("people", false), getRef());
1891 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1893 shardManager.tell(new FindLocalShard("default", false), getRef());
1894 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1896 // Removed the default shard replica from member-1
1897 ShardIdentifier.Builder builder = new ShardIdentifier.Builder();
1898 ShardIdentifier shardId = builder.shardName("default").memberName(MEMBER_1).type(shardMrgIDSuffix)
1900 shardManager.tell(new ServerRemoved(shardId.toString()), getRef());
1902 shardManager.underlyingActor().verifySnapshotPersisted(Sets.newHashSet("people"));
1906 LOG.info("testServerRemovedShardActorNotRunning ending");
1910 public void testServerRemovedShardActorRunning() throws Exception {
1911 LOG.info("testServerRemovedShardActorRunning starting");
1912 new JavaTestKit(getSystem()) {
1914 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1915 .put("default", Arrays.asList("member-1", "member-2"))
1916 .put("astronauts", Arrays.asList("member-2"))
1917 .put("people", Arrays.asList("member-1", "member-2")).build());
1919 String shardId = ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString();
1920 TestActorRef<MessageCollectorActor> shard = actorFactory.createTestActor(MessageCollectorActor.props(),
1923 TestActorRef<TestShardManager> shardManager = actorFactory
1924 .createTestActor(newTestShardMgrBuilder(mockConfig).addShardActor("default", shard).props());
1926 shardManager.underlyingActor().waitForRecoveryComplete();
1928 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1929 shardManager.tell(new ActorInitialized(), shard);
1931 waitForShardInitialized(shardManager, "people", this);
1932 waitForShardInitialized(shardManager, "default", this);
1934 // Removed the default shard replica from member-1
1935 shardManager.tell(new ServerRemoved(shardId), getRef());
1937 shardManager.underlyingActor().verifySnapshotPersisted(Sets.newHashSet("people"));
1939 MessageCollectorActor.expectFirstMatching(shard, Shutdown.class);
1943 LOG.info("testServerRemovedShardActorRunning ending");
1947 public void testShardPersistenceWithRestoredData() throws Exception {
1948 LOG.info("testShardPersistenceWithRestoredData starting");
1949 new JavaTestKit(getSystem()) {
1951 MockConfiguration mockConfig =
1952 new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1953 .put("default", Arrays.asList("member-1", "member-2"))
1954 .put("astronauts", Arrays.asList("member-2"))
1955 .put("people", Arrays.asList("member-1", "member-2")).build());
1956 String[] restoredShards = {"default", "astronauts"};
1957 ShardManagerSnapshot snapshot =
1958 new ShardManagerSnapshot(Arrays.asList(restoredShards), Collections.emptyMap());
1959 InMemorySnapshotStore.addSnapshot("shard-manager-" + shardMrgIDSuffix, snapshot);
1961 // create shardManager to come up with restored data
1962 TestActorRef<TestShardManager> newRestoredShardManager = actorFactory
1963 .createTestActor(newShardMgrProps(mockConfig));
1965 newRestoredShardManager.underlyingActor().waitForRecoveryComplete();
1967 newRestoredShardManager.tell(new FindLocalShard("people", false), getRef());
1968 LocalShardNotFound notFound = expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
1969 assertEquals("for uninitialized shard", "people", notFound.getShardName());
1971 // Verify a local shard is created for the restored shards,
1972 // although we expect a NotInitializedException for the shards
1973 // as the actor initialization
1974 // message is not sent for them
1975 newRestoredShardManager.tell(new FindLocalShard("default", false), getRef());
1976 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1978 newRestoredShardManager.tell(new FindLocalShard("astronauts", false), getRef());
1979 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1983 LOG.info("testShardPersistenceWithRestoredData ending");
1987 public void testShutDown() throws Exception {
1988 LOG.info("testShutDown starting");
1989 new JavaTestKit(getSystem()) {
1991 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1992 .put("shard1", Arrays.asList("member-1")).put("shard2", Arrays.asList("member-1")).build());
1994 String shardId1 = ShardIdentifier.create("shard1", MEMBER_1, shardMrgIDSuffix).toString();
1995 TestActorRef<MessageCollectorActor> shard1 = actorFactory.createTestActor(
1996 MessageCollectorActor.props().withDispatcher(Dispatchers.DefaultDispatcherId()), shardId1);
1998 String shardId2 = ShardIdentifier.create("shard2", MEMBER_1, shardMrgIDSuffix).toString();
1999 TestActorRef<MessageCollectorActor> shard2 = actorFactory.createTestActor(
2000 MessageCollectorActor.props().withDispatcher(Dispatchers.DefaultDispatcherId()), shardId2);
2002 TestActorRef<TestShardManager> shardManager = actorFactory
2003 .createTestActor(newTestShardMgrBuilder(mockConfig).addShardActor("shard1", shard1)
2004 .addShardActor("shard2", shard2).props()
2005 .withDispatcher(Dispatchers.DefaultDispatcherId()));
2007 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
2008 shardManager.tell(new ActorInitialized(), shard1);
2009 shardManager.tell(new ActorInitialized(), shard2);
2011 FiniteDuration duration = FiniteDuration.create(5, TimeUnit.SECONDS);
2012 Future<Boolean> stopFuture = Patterns.gracefulStop(shardManager, duration, Shutdown.INSTANCE);
2014 MessageCollectorActor.expectFirstMatching(shard1, Shutdown.class);
2015 MessageCollectorActor.expectFirstMatching(shard2, Shutdown.class);
2018 Await.ready(stopFuture, FiniteDuration.create(500, TimeUnit.MILLISECONDS));
2019 fail("ShardManager actor stopped without waiting for the Shards to be stopped");
2020 } catch (TimeoutException e) {
2024 actorFactory.killActor(shard1, this);
2025 actorFactory.killActor(shard2, this);
2027 Boolean stopped = Await.result(stopFuture, duration);
2028 assertEquals("Stopped", Boolean.TRUE, stopped);
2032 LOG.info("testShutDown ending");
2036 public void testChangeServersVotingStatus() throws Exception {
2037 new JavaTestKit(getSystem()) {
2039 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
2041 TestActorRef<MockRespondActor> respondActor = actorFactory
2042 .createTestActor(Props.create(MockRespondActor.class, ChangeServersVotingStatus.class,
2043 new ServerChangeReply(ServerChangeStatus.OK, null)), memberId);
2045 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
2047 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
2048 shardManager.tell(new ActorInitialized(), respondActor);
2049 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
2050 DataStoreVersions.CURRENT_VERSION), getRef());
2052 new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
2056 new ChangeShardMembersVotingStatus("default", ImmutableMap.of("member-2", Boolean.TRUE)),
2059 ChangeServersVotingStatus actualChangeStatusMsg = MessageCollectorActor
2060 .expectFirstMatching(respondActor, ChangeServersVotingStatus.class);
2061 assertEquals("ChangeServersVotingStatus map", actualChangeStatusMsg.getServerVotingStatusMap(),
2062 ImmutableMap.of(ShardIdentifier
2063 .create("default", MemberName.forName("member-2"), shardMrgIDSuffix).toString(),
2066 expectMsgClass(duration("5 seconds"), Success.class);
2072 public void testChangeServersVotingStatusWithNoLeader() throws Exception {
2073 new JavaTestKit(getSystem()) {
2075 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
2077 TestActorRef<MockRespondActor> respondActor = actorFactory
2078 .createTestActor(Props.create(MockRespondActor.class, ChangeServersVotingStatus.class,
2079 new ServerChangeReply(ServerChangeStatus.NO_LEADER, null)), memberId);
2081 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
2083 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
2084 shardManager.tell(new ActorInitialized(), respondActor);
2085 shardManager.tell(new RoleChangeNotification(memberId, null, RaftState.Follower.name()), respondActor);
2088 new ChangeShardMembersVotingStatus("default", ImmutableMap.of("member-2", Boolean.TRUE)),
2091 MessageCollectorActor.expectFirstMatching(respondActor, ChangeServersVotingStatus.class);
2093 Status.Failure resp = expectMsgClass(duration("5 seconds"), Status.Failure.class);
2094 assertEquals("Failure resposnse", true, resp.cause() instanceof NoShardLeaderException);
2099 public static class TestShardManager extends ShardManager {
2100 private final CountDownLatch recoveryComplete = new CountDownLatch(1);
2101 private final CountDownLatch snapshotPersist = new CountDownLatch(1);
2102 private ShardManagerSnapshot snapshot;
2103 private final Map<String, ActorRef> shardActors;
2104 private final ActorRef shardActor;
2105 private CountDownLatch findPrimaryMessageReceived = new CountDownLatch(1);
2106 private CountDownLatch memberUpReceived = new CountDownLatch(1);
2107 private CountDownLatch memberRemovedReceived = new CountDownLatch(1);
2108 private CountDownLatch memberUnreachableReceived = new CountDownLatch(1);
2109 private CountDownLatch memberReachableReceived = new CountDownLatch(1);
2110 private volatile MessageInterceptor messageInterceptor;
2112 private TestShardManager(Builder builder) {
2114 shardActor = builder.shardActor;
2115 shardActors = builder.shardActors;
2119 protected void handleRecover(Object message) throws Exception {
2121 super.handleRecover(message);
2123 if (message instanceof RecoveryCompleted) {
2124 recoveryComplete.countDown();
2129 private void countDownIfOther(final Member member, CountDownLatch latch) {
2130 if (!getCluster().getCurrentMemberName().equals(memberToName(member))) {
2136 public void handleCommand(Object message) throws Exception {
2138 if (messageInterceptor != null && messageInterceptor.canIntercept(message)) {
2139 getSender().tell(messageInterceptor.apply(message), getSelf());
2141 super.handleCommand(message);
2144 if (message instanceof FindPrimary) {
2145 findPrimaryMessageReceived.countDown();
2146 } else if (message instanceof ClusterEvent.MemberUp) {
2147 countDownIfOther(((ClusterEvent.MemberUp) message).member(), memberUpReceived);
2148 } else if (message instanceof ClusterEvent.MemberRemoved) {
2149 countDownIfOther(((ClusterEvent.MemberRemoved) message).member(), memberRemovedReceived);
2150 } else if (message instanceof ClusterEvent.UnreachableMember) {
2151 countDownIfOther(((ClusterEvent.UnreachableMember) message).member(), memberUnreachableReceived);
2152 } else if (message instanceof ClusterEvent.ReachableMember) {
2153 countDownIfOther(((ClusterEvent.ReachableMember) message).member(), memberReachableReceived);
2158 void setMessageInterceptor(MessageInterceptor messageInterceptor) {
2159 this.messageInterceptor = messageInterceptor;
2162 void waitForRecoveryComplete() {
2163 assertEquals("Recovery complete", true,
2164 Uninterruptibles.awaitUninterruptibly(recoveryComplete, 5, TimeUnit.SECONDS));
2167 public void waitForMemberUp() {
2168 assertEquals("MemberUp received", true,
2169 Uninterruptibles.awaitUninterruptibly(memberUpReceived, 5, TimeUnit.SECONDS));
2170 memberUpReceived = new CountDownLatch(1);
2173 void waitForMemberRemoved() {
2174 assertEquals("MemberRemoved received", true,
2175 Uninterruptibles.awaitUninterruptibly(memberRemovedReceived, 5, TimeUnit.SECONDS));
2176 memberRemovedReceived = new CountDownLatch(1);
2179 void waitForUnreachableMember() {
2180 assertEquals("UnreachableMember received", true,
2181 Uninterruptibles.awaitUninterruptibly(memberUnreachableReceived, 5, TimeUnit.SECONDS
2183 memberUnreachableReceived = new CountDownLatch(1);
2186 void waitForReachableMember() {
2187 assertEquals("ReachableMember received", true,
2188 Uninterruptibles.awaitUninterruptibly(memberReachableReceived, 5, TimeUnit.SECONDS));
2189 memberReachableReceived = new CountDownLatch(1);
2192 void verifyFindPrimary() {
2193 assertEquals("FindPrimary received", true,
2194 Uninterruptibles.awaitUninterruptibly(findPrimaryMessageReceived, 5, TimeUnit.SECONDS));
2195 findPrimaryMessageReceived = new CountDownLatch(1);
2198 public static Builder builder(DatastoreContext.Builder datastoreContextBuilder) {
2199 return new Builder(datastoreContextBuilder);
2202 public static class Builder extends AbstractGenericCreator<Builder, TestShardManager> {
2203 private ActorRef shardActor;
2204 private final Map<String, ActorRef> shardActors = new HashMap<>();
2206 Builder(DatastoreContext.Builder datastoreContextBuilder) {
2207 super(TestShardManager.class);
2208 datastoreContextFactory(newDatastoreContextFactory(datastoreContextBuilder.build()));
2211 Builder shardActor(ActorRef newShardActor) {
2212 this.shardActor = newShardActor;
2216 Builder addShardActor(String shardName, ActorRef actorRef) {
2217 shardActors.put(shardName, actorRef);
2223 public void saveSnapshot(Object obj) {
2224 snapshot = (ShardManagerSnapshot) obj;
2225 snapshotPersist.countDown();
2226 super.saveSnapshot(obj);
2229 void verifySnapshotPersisted(Set<String> shardList) {
2230 assertEquals("saveSnapshot invoked", true,
2231 Uninterruptibles.awaitUninterruptibly(snapshotPersist, 5, TimeUnit.SECONDS));
2232 assertEquals("Shard Persisted", shardList, Sets.newHashSet(snapshot.getShardList()));
2236 protected ActorRef newShardActor(SchemaContext schemaContext, ShardInformation info) {
2237 if (shardActors.get(info.getShardName()) != null) {
2238 return shardActors.get(info.getShardName());
2241 if (shardActor != null) {
2245 return super.newShardActor(schemaContext, info);
2249 private abstract static class AbstractGenericCreator<T extends AbstractGenericCreator<T, ?>, C extends ShardManager>
2250 extends AbstractShardManagerCreator<T> {
2251 private final Class<C> shardManagerClass;
2253 AbstractGenericCreator(Class<C> shardManagerClass) {
2254 this.shardManagerClass = shardManagerClass;
2255 cluster(new MockClusterWrapper()).configuration(new MockConfiguration()).waitTillReadyCountDownLatch(ready)
2256 .primaryShardInfoCache(new PrimaryShardInfoFutureCache());
2260 public Props props() {
2262 return Props.create(shardManagerClass, this);
2266 private static class GenericCreator<C extends ShardManager> extends AbstractGenericCreator<GenericCreator<C>, C> {
2267 GenericCreator(Class<C> shardManagerClass) {
2268 super(shardManagerClass);
2272 private static class DelegatingShardManagerCreator implements Creator<ShardManager> {
2273 private static final long serialVersionUID = 1L;
2274 private final Creator<ShardManager> delegate;
2276 DelegatingShardManagerCreator(Creator<ShardManager> delegate) {
2277 this.delegate = delegate;
2281 public ShardManager create() throws Exception {
2282 return delegate.create();
2286 interface MessageInterceptor extends Function<Object, Object> {
2287 boolean canIntercept(Object message);
2290 private static MessageInterceptor newFindPrimaryInterceptor(final ActorRef primaryActor) {
2291 return new MessageInterceptor() {
2293 public Object apply(Object message) {
2294 return new RemotePrimaryShardFound(Serialization.serializedActorPath(primaryActor), (short) 1);
2298 public boolean canIntercept(Object message) {
2299 return message instanceof FindPrimary;
2304 private static class MockRespondActor extends MessageCollectorActor {
2305 static final String CLEAR_RESPONSE = "clear-response";
2307 private Object responseMsg;
2308 private final Class<?> requestClass;
2310 @SuppressWarnings("unused")
2311 MockRespondActor(Class<?> requestClass, Object responseMsg) {
2312 this.requestClass = requestClass;
2313 this.responseMsg = responseMsg;
2317 public void onReceive(Object message) throws Exception {
2318 if (message.equals(CLEAR_RESPONSE)) {
2321 super.onReceive(message);
2322 if (message.getClass().equals(requestClass) && responseMsg != null) {
2323 getSender().tell(responseMsg, getSelf());