2 * Copyright (c) 2014, 2015 Cisco Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
9 package org.opendaylight.controller.cluster.datastore.shardmanager;
11 import static org.junit.Assert.assertEquals;
12 import static org.junit.Assert.assertNotNull;
13 import static org.junit.Assert.assertNull;
14 import static org.junit.Assert.assertSame;
15 import static org.junit.Assert.assertTrue;
16 import static org.junit.Assert.fail;
17 import static org.mockito.Mockito.mock;
18 import static org.mockito.Mockito.never;
19 import static org.mockito.Mockito.times;
20 import static org.mockito.Mockito.verify;
22 import akka.actor.ActorRef;
23 import akka.actor.ActorSystem;
24 import akka.actor.AddressFromURIString;
25 import akka.actor.Props;
26 import akka.actor.Status;
27 import akka.actor.Status.Failure;
28 import akka.actor.Status.Success;
29 import akka.cluster.Cluster;
30 import akka.cluster.ClusterEvent;
31 import akka.cluster.Member;
32 import akka.dispatch.Dispatchers;
33 import akka.dispatch.OnComplete;
34 import akka.japi.Creator;
35 import akka.pattern.Patterns;
36 import akka.persistence.RecoveryCompleted;
37 import akka.serialization.Serialization;
38 import akka.testkit.TestActorRef;
39 import akka.testkit.javadsl.TestKit;
40 import akka.util.Timeout;
41 import com.google.common.base.Function;
42 import com.google.common.base.Stopwatch;
43 import com.google.common.collect.ImmutableMap;
44 import com.google.common.collect.Lists;
45 import com.google.common.collect.Sets;
46 import com.google.common.util.concurrent.Uninterruptibles;
48 import java.util.AbstractMap;
49 import java.util.Arrays;
50 import java.util.Collection;
51 import java.util.Collections;
52 import java.util.HashMap;
53 import java.util.List;
55 import java.util.Map.Entry;
57 import java.util.concurrent.CountDownLatch;
58 import java.util.concurrent.TimeUnit;
59 import java.util.concurrent.TimeoutException;
60 import java.util.stream.Collectors;
61 import org.junit.AfterClass;
62 import org.junit.BeforeClass;
63 import org.junit.Test;
64 import org.mockito.Mockito;
65 import org.opendaylight.controller.cluster.access.concepts.MemberName;
66 import org.opendaylight.controller.cluster.datastore.AbstractShardManagerTest;
67 import org.opendaylight.controller.cluster.datastore.ClusterWrapperImpl;
68 import org.opendaylight.controller.cluster.datastore.DataStoreVersions;
69 import org.opendaylight.controller.cluster.datastore.DatastoreContext;
70 import org.opendaylight.controller.cluster.datastore.DatastoreContextFactory;
71 import org.opendaylight.controller.cluster.datastore.DistributedDataStore;
72 import org.opendaylight.controller.cluster.datastore.Shard;
73 import org.opendaylight.controller.cluster.datastore.config.ConfigurationImpl;
74 import org.opendaylight.controller.cluster.datastore.config.EmptyModuleShardConfigProvider;
75 import org.opendaylight.controller.cluster.datastore.config.ModuleShardConfiguration;
76 import org.opendaylight.controller.cluster.datastore.exceptions.AlreadyExistsException;
77 import org.opendaylight.controller.cluster.datastore.exceptions.NoShardLeaderException;
78 import org.opendaylight.controller.cluster.datastore.exceptions.NotInitializedException;
79 import org.opendaylight.controller.cluster.datastore.exceptions.PrimaryNotFoundException;
80 import org.opendaylight.controller.cluster.datastore.identifiers.ShardIdentifier;
81 import org.opendaylight.controller.cluster.datastore.identifiers.ShardManagerIdentifier;
82 import org.opendaylight.controller.cluster.datastore.messages.ActorInitialized;
83 import org.opendaylight.controller.cluster.datastore.messages.AddShardReplica;
84 import org.opendaylight.controller.cluster.datastore.messages.ChangeShardMembersVotingStatus;
85 import org.opendaylight.controller.cluster.datastore.messages.CreateShard;
86 import org.opendaylight.controller.cluster.datastore.messages.FindLocalShard;
87 import org.opendaylight.controller.cluster.datastore.messages.FindPrimary;
88 import org.opendaylight.controller.cluster.datastore.messages.LocalPrimaryShardFound;
89 import org.opendaylight.controller.cluster.datastore.messages.LocalShardFound;
90 import org.opendaylight.controller.cluster.datastore.messages.LocalShardNotFound;
91 import org.opendaylight.controller.cluster.datastore.messages.PeerDown;
92 import org.opendaylight.controller.cluster.datastore.messages.PeerUp;
93 import org.opendaylight.controller.cluster.datastore.messages.PrimaryShardInfo;
94 import org.opendaylight.controller.cluster.datastore.messages.RemotePrimaryShardFound;
95 import org.opendaylight.controller.cluster.datastore.messages.RemoveShardReplica;
96 import org.opendaylight.controller.cluster.datastore.messages.ShardLeaderStateChanged;
97 import org.opendaylight.controller.cluster.datastore.messages.UpdateSchemaContext;
98 import org.opendaylight.controller.cluster.datastore.persisted.DatastoreSnapshot;
99 import org.opendaylight.controller.cluster.datastore.persisted.DatastoreSnapshot.ShardSnapshot;
100 import org.opendaylight.controller.cluster.datastore.persisted.ShardManagerSnapshot;
101 import org.opendaylight.controller.cluster.datastore.utils.ForwardingActor;
102 import org.opendaylight.controller.cluster.datastore.utils.MockClusterWrapper;
103 import org.opendaylight.controller.cluster.datastore.utils.MockConfiguration;
104 import org.opendaylight.controller.cluster.datastore.utils.PrimaryShardInfoFutureCache;
105 import org.opendaylight.controller.cluster.notifications.LeaderStateChanged;
106 import org.opendaylight.controller.cluster.notifications.RegisterRoleChangeListener;
107 import org.opendaylight.controller.cluster.notifications.RoleChangeNotification;
108 import org.opendaylight.controller.cluster.raft.RaftState;
109 import org.opendaylight.controller.cluster.raft.base.messages.FollowerInitialSyncUpStatus;
110 import org.opendaylight.controller.cluster.raft.base.messages.SwitchBehavior;
111 import org.opendaylight.controller.cluster.raft.client.messages.GetSnapshot;
112 import org.opendaylight.controller.cluster.raft.client.messages.Shutdown;
113 import org.opendaylight.controller.cluster.raft.messages.AddServer;
114 import org.opendaylight.controller.cluster.raft.messages.AddServerReply;
115 import org.opendaylight.controller.cluster.raft.messages.ChangeServersVotingStatus;
116 import org.opendaylight.controller.cluster.raft.messages.RemoveServer;
117 import org.opendaylight.controller.cluster.raft.messages.RemoveServerReply;
118 import org.opendaylight.controller.cluster.raft.messages.ServerChangeReply;
119 import org.opendaylight.controller.cluster.raft.messages.ServerChangeStatus;
120 import org.opendaylight.controller.cluster.raft.messages.ServerRemoved;
121 import org.opendaylight.controller.cluster.raft.policy.DisableElectionsRaftPolicy;
122 import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore;
123 import org.opendaylight.controller.cluster.raft.utils.MessageCollectorActor;
124 import org.opendaylight.controller.md.cluster.datastore.model.TestModel;
125 import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTree;
126 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
127 import org.slf4j.Logger;
128 import org.slf4j.LoggerFactory;
129 import scala.concurrent.Await;
130 import scala.concurrent.Future;
131 import scala.concurrent.duration.FiniteDuration;
133 public class ShardManagerTest extends AbstractShardManagerTest {
134 private static final Logger LOG = LoggerFactory.getLogger(ShardManagerTest.class);
135 private static final MemberName MEMBER_2 = MemberName.forName("member-2");
136 private static final MemberName MEMBER_3 = MemberName.forName("member-3");
138 private static SchemaContext TEST_SCHEMA_CONTEXT;
140 private final String shardMgrID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
143 public static void beforeClass() {
144 TEST_SCHEMA_CONTEXT = TestModel.createTestContext();
148 public static void afterClass() {
149 TEST_SCHEMA_CONTEXT = null;
152 private ActorSystem newActorSystem(final String config) {
153 return newActorSystem("cluster-test", config);
156 private ActorRef newMockShardActor(final ActorSystem system, final String shardName, final String memberName) {
157 String name = ShardIdentifier.create(shardName, MemberName.forName(memberName), "config").toString();
158 if (system == getSystem()) {
159 return actorFactory.createActor(MessageCollectorActor.props(), name);
162 return system.actorOf(MessageCollectorActor.props(), name);
165 private Props newShardMgrProps() {
166 return newShardMgrProps(new MockConfiguration());
169 private static DatastoreContextFactory newDatastoreContextFactory(final DatastoreContext datastoreContext) {
170 DatastoreContextFactory mockFactory = mock(DatastoreContextFactory.class);
171 Mockito.doReturn(datastoreContext).when(mockFactory).getBaseDatastoreContext();
172 Mockito.doReturn(datastoreContext).when(mockFactory).getShardDatastoreContext(Mockito.anyString());
176 private TestShardManager.Builder newTestShardMgrBuilderWithMockShardActor() {
177 return newTestShardMgrBuilderWithMockShardActor(mockShardActor);
180 private TestShardManager.Builder newTestShardMgrBuilderWithMockShardActor(final ActorRef shardActor) {
181 return TestShardManager.builder(datastoreContextBuilder).shardActor(shardActor)
182 .distributedDataStore(mock(DistributedDataStore.class));
186 private Props newPropsShardMgrWithMockShardActor() {
187 return newTestShardMgrBuilderWithMockShardActor().props().withDispatcher(
188 Dispatchers.DefaultDispatcherId());
191 private Props newPropsShardMgrWithMockShardActor(final ActorRef shardActor) {
192 return newTestShardMgrBuilderWithMockShardActor(shardActor).props()
193 .withDispatcher(Dispatchers.DefaultDispatcherId());
197 private TestShardManager newTestShardManager() {
198 return newTestShardManager(newShardMgrProps());
201 private TestShardManager newTestShardManager(final Props props) {
202 TestActorRef<TestShardManager> shardManagerActor = actorFactory.createTestActor(props);
203 TestShardManager shardManager = shardManagerActor.underlyingActor();
204 shardManager.waitForRecoveryComplete();
208 private static void waitForShardInitialized(final ActorRef shardManager, final String shardName,
210 AssertionError last = null;
211 Stopwatch sw = Stopwatch.createStarted();
212 while (sw.elapsed(TimeUnit.SECONDS) <= 5) {
214 shardManager.tell(new FindLocalShard(shardName, true), kit.getRef());
215 kit.expectMsgClass(LocalShardFound.class);
217 } catch (AssertionError e) {
221 Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
227 @SuppressWarnings("unchecked")
228 private static <T> T expectMsgClassOrFailure(final Class<T> msgClass, final TestKit kit, final String msg) {
229 Object reply = kit.expectMsgAnyClassOf(kit.duration("5 sec"), msgClass, Failure.class);
230 if (reply instanceof Failure) {
231 throw new AssertionError(msg + " failed", ((Failure)reply).cause());
238 public void testPerShardDatastoreContext() throws Exception {
239 LOG.info("testPerShardDatastoreContext starting");
240 final DatastoreContextFactory mockFactory = newDatastoreContextFactory(
241 datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
244 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(6).build())
245 .when(mockFactory).getShardDatastoreContext("default");
248 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(7).build())
249 .when(mockFactory).getShardDatastoreContext("topology");
251 final MockConfiguration mockConfig = new MockConfiguration() {
253 public Collection<String> getMemberShardNames(final MemberName memberName) {
254 return Arrays.asList("default", "topology");
258 public Collection<MemberName> getMembersFromShardName(final String shardName) {
259 return members("member-1");
263 final ActorRef defaultShardActor = actorFactory.createActor(
264 MessageCollectorActor.props(), actorFactory.generateActorId("default"));
265 final ActorRef topologyShardActor = actorFactory.createActor(
266 MessageCollectorActor.props(), actorFactory.generateActorId("topology"));
268 final Map<String, Entry<ActorRef, DatastoreContext>> shardInfoMap = Collections.synchronizedMap(
269 new HashMap<String, Entry<ActorRef, DatastoreContext>>());
270 shardInfoMap.put("default", new AbstractMap.SimpleEntry<>(defaultShardActor, null));
271 shardInfoMap.put("topology", new AbstractMap.SimpleEntry<>(topologyShardActor, null));
273 final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
274 final CountDownLatch newShardActorLatch = new CountDownLatch(2);
275 class LocalShardManager extends ShardManager {
276 LocalShardManager(final AbstractShardManagerCreator<?> creator) {
281 protected ActorRef newShardActor(final ShardInformation info) {
282 Entry<ActorRef, DatastoreContext> entry = shardInfoMap.get(info.getShardName());
285 ref = entry.getKey();
286 entry.setValue(info.getDatastoreContext());
289 newShardActorLatch.countDown();
294 final Creator<ShardManager> creator = new Creator<ShardManager>() {
295 private static final long serialVersionUID = 1L;
297 public ShardManager create() {
298 return new LocalShardManager(
299 new GenericCreator<>(LocalShardManager.class).datastoreContextFactory(mockFactory)
300 .primaryShardInfoCache(primaryShardInfoCache).configuration(mockConfig));
304 TestKit kit = new TestKit(getSystem());
306 final ActorRef shardManager = actorFactory.createActor(Props.create(
307 new DelegatingShardManagerCreator(creator)).withDispatcher(Dispatchers.DefaultDispatcherId()));
309 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), kit.getRef());
311 assertEquals("Shard actors created", true, newShardActorLatch.await(5, TimeUnit.SECONDS));
312 assertEquals("getShardElectionTimeoutFactor", 6,
313 shardInfoMap.get("default").getValue().getShardElectionTimeoutFactor());
314 assertEquals("getShardElectionTimeoutFactor", 7,
315 shardInfoMap.get("topology").getValue().getShardElectionTimeoutFactor());
317 DatastoreContextFactory newMockFactory = newDatastoreContextFactory(
318 datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
320 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(66).build())
321 .when(newMockFactory).getShardDatastoreContext("default");
324 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(77).build())
325 .when(newMockFactory).getShardDatastoreContext("topology");
327 shardManager.tell(newMockFactory, kit.getRef());
329 DatastoreContext newContext = MessageCollectorActor.expectFirstMatching(defaultShardActor,
330 DatastoreContext.class);
331 assertEquals("getShardElectionTimeoutFactor", 66, newContext.getShardElectionTimeoutFactor());
333 newContext = MessageCollectorActor.expectFirstMatching(topologyShardActor, DatastoreContext.class);
334 assertEquals("getShardElectionTimeoutFactor", 77, newContext.getShardElectionTimeoutFactor());
336 LOG.info("testPerShardDatastoreContext ending");
340 public void testOnReceiveFindPrimaryForNonExistentShard() {
341 new TestKit(getSystem()) {
343 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
345 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
347 shardManager.tell(new FindPrimary("non-existent", false), getRef());
349 expectMsgClass(duration("5 seconds"), PrimaryNotFoundException.class);
355 public void testOnReceiveFindPrimaryForLocalLeaderShard() {
356 LOG.info("testOnReceiveFindPrimaryForLocalLeaderShard starting");
357 new TestKit(getSystem()) {
359 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
361 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
363 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
364 shardManager.tell(new ActorInitialized(), mockShardActor);
366 DataTree mockDataTree = mock(DataTree.class);
367 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
368 DataStoreVersions.CURRENT_VERSION), getRef());
370 MessageCollectorActor.expectFirstMatching(mockShardActor, RegisterRoleChangeListener.class);
372 new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
375 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
377 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
378 LocalPrimaryShardFound.class);
379 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
380 primaryFound.getPrimaryPath().contains("member-1-shard-default"));
381 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
385 LOG.info("testOnReceiveFindPrimaryForLocalLeaderShard ending");
389 public void testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp() {
390 LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp starting");
391 new TestKit(getSystem()) {
393 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
395 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
396 shardManager.tell(new ActorInitialized(), mockShardActor);
398 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
399 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
401 new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
403 shardManager.tell(new LeaderStateChanged(memberId1, memberId2, DataStoreVersions.CURRENT_VERSION),
406 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
408 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
412 LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp ending");
416 public void testOnReceiveFindPrimaryForNonLocalLeaderShard() {
417 LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShard starting");
418 new TestKit(getSystem()) {
420 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
422 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
423 shardManager.tell(new ActorInitialized(), mockShardActor);
425 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
426 MockClusterWrapper.sendMemberUp(shardManager, "member-2", getRef().path().toString());
428 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
430 new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
432 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
433 shardManager.tell(new ShardLeaderStateChanged(memberId1, memberId2, leaderVersion), mockShardActor);
435 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
437 RemotePrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
438 RemotePrimaryShardFound.class);
439 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
440 primaryFound.getPrimaryPath().contains("member-2-shard-default"));
441 assertEquals("getPrimaryVersion", leaderVersion, primaryFound.getPrimaryVersion());
445 LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShard ending");
449 public void testOnReceiveFindPrimaryForUninitializedShard() {
450 new TestKit(getSystem()) {
452 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
454 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
456 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
462 public void testOnReceiveFindPrimaryForInitializedShardWithNoRole() {
463 new TestKit(getSystem()) {
465 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
467 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
468 shardManager.tell(new ActorInitialized(), mockShardActor);
470 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
472 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
478 public void testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId() {
479 LOG.info("testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId starting");
480 new TestKit(getSystem()) {
482 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
484 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
485 shardManager.tell(new ActorInitialized(), mockShardActor);
487 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
489 new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Follower.name()),
492 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
494 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
496 DataTree mockDataTree = mock(DataTree.class);
497 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
498 DataStoreVersions.CURRENT_VERSION), mockShardActor);
500 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
502 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
503 LocalPrimaryShardFound.class);
504 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
505 primaryFound.getPrimaryPath().contains("member-1-shard-default"));
506 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
510 LOG.info("testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId starting");
514 public void testOnReceiveFindPrimaryWaitForShardLeader() {
515 LOG.info("testOnReceiveFindPrimaryWaitForShardLeader starting");
516 datastoreContextBuilder.shardInitializationTimeout(10, TimeUnit.SECONDS);
517 new TestKit(getSystem()) {
519 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
521 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
523 // We're passing waitUntilInitialized = true to FindPrimary so
524 // the response should be
525 // delayed until we send ActorInitialized and
526 // RoleChangeNotification.
527 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
529 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
531 shardManager.tell(new ActorInitialized(), mockShardActor);
533 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
535 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
537 new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
540 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
542 DataTree mockDataTree = mock(DataTree.class);
543 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
544 DataStoreVersions.CURRENT_VERSION), mockShardActor);
546 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
547 LocalPrimaryShardFound.class);
548 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
549 primaryFound.getPrimaryPath().contains("member-1-shard-default"));
550 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
552 expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
556 LOG.info("testOnReceiveFindPrimaryWaitForShardLeader ending");
560 public void testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard() {
561 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard starting");
562 new TestKit(getSystem()) {
564 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
566 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
568 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
570 expectMsgClass(duration("2 seconds"), NotInitializedException.class);
572 shardManager.tell(new ActorInitialized(), mockShardActor);
574 expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
578 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard ending");
582 public void testOnReceiveFindPrimaryWaitForReadyWithCandidateShard() {
583 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithCandidateShard starting");
584 new TestKit(getSystem()) {
586 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
588 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
589 shardManager.tell(new ActorInitialized(), mockShardActor);
590 shardManager.tell(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix, null,
591 RaftState.Candidate.name()), mockShardActor);
593 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
595 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
599 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithCandidateShard ending");
603 public void testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard() {
604 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard starting");
605 new TestKit(getSystem()) {
607 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
609 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
610 shardManager.tell(new ActorInitialized(), mockShardActor);
611 shardManager.tell(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix, null,
612 RaftState.IsolatedLeader.name()), mockShardActor);
614 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
616 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
620 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard ending");
624 public void testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard() {
625 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard starting");
626 new TestKit(getSystem()) {
628 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
630 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
631 shardManager.tell(new ActorInitialized(), mockShardActor);
633 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
635 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
639 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard ending");
643 public void testOnReceiveFindPrimaryForRemoteShard() {
644 LOG.info("testOnReceiveFindPrimaryForRemoteShard starting");
645 String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
647 // Create an ActorSystem ShardManager actor for member-1.
649 final ActorSystem system1 = newActorSystem("Member1");
650 Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
652 final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
653 newTestShardMgrBuilderWithMockShardActor().cluster(
654 new ClusterWrapperImpl(system1)).props().withDispatcher(
655 Dispatchers.DefaultDispatcherId()), shardManagerID);
657 // Create an ActorSystem ShardManager actor for member-2.
659 final ActorSystem system2 = newActorSystem("Member2");
661 Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
663 final ActorRef mockShardActor2 = newMockShardActor(system2, "astronauts", "member-2");
665 MockConfiguration mockConfig2 = new MockConfiguration(
666 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
667 .put("astronauts", Arrays.asList("member-2")).build());
669 final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
670 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
671 new ClusterWrapperImpl(system2)).props().withDispatcher(
672 Dispatchers.DefaultDispatcherId()), shardManagerID);
674 new TestKit(system1) {
676 shardManager1.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
677 shardManager2.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
679 shardManager2.tell(new ActorInitialized(), mockShardActor2);
681 String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
682 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
684 new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
687 new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
690 shardManager1.underlyingActor().waitForMemberUp();
691 shardManager1.tell(new FindPrimary("astronauts", false), getRef());
693 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
694 String path = found.getPrimaryPath();
695 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-astronauts-config"));
696 assertEquals("getPrimaryVersion", leaderVersion, found.getPrimaryVersion());
698 shardManager2.underlyingActor().verifyFindPrimary();
700 // This part times out quite a bit on jenkins for some reason
702 // Cluster.get(system2).down(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
704 // shardManager1.underlyingActor().waitForMemberRemoved();
706 // shardManager1.tell(new FindPrimary("astronauts", false), getRef());
708 // expectMsgClass(duration("5 seconds"), PrimaryNotFoundException.class);
712 LOG.info("testOnReceiveFindPrimaryForRemoteShard ending");
716 public void testShardAvailabilityOnChangeOfMemberReachability() {
717 LOG.info("testShardAvailabilityOnChangeOfMemberReachability starting");
718 String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
720 // Create an ActorSystem ShardManager actor for member-1.
722 final ActorSystem system1 = newActorSystem("Member1");
723 Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
725 final ActorRef mockShardActor1 = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
727 final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
728 newTestShardMgrBuilder().shardActor(mockShardActor1).cluster(
729 new ClusterWrapperImpl(system1)).props().withDispatcher(
730 Dispatchers.DefaultDispatcherId()), shardManagerID);
732 // Create an ActorSystem ShardManager actor for member-2.
734 final ActorSystem system2 = newActorSystem("Member2");
736 Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
738 final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
740 MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
741 .put("default", Arrays.asList("member-1", "member-2")).build());
743 final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
744 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
745 new ClusterWrapperImpl(system2)).props().withDispatcher(
746 Dispatchers.DefaultDispatcherId()), shardManagerID);
748 new TestKit(system1) {
750 shardManager1.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
751 shardManager2.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
752 shardManager1.tell(new ActorInitialized(), mockShardActor1);
753 shardManager2.tell(new ActorInitialized(), mockShardActor2);
755 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
756 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
757 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class),
758 DataStoreVersions.CURRENT_VERSION), mockShardActor1);
760 new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
762 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class),
763 DataStoreVersions.CURRENT_VERSION), mockShardActor2);
765 new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
767 shardManager1.underlyingActor().waitForMemberUp();
769 shardManager1.tell(new FindPrimary("default", true), getRef());
771 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
772 String path = found.getPrimaryPath();
773 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
775 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
776 "akka://cluster-test@127.0.0.1:2558"), getRef());
778 shardManager1.underlyingActor().waitForUnreachableMember();
780 PeerDown peerDown = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
781 assertEquals("getMemberName", MEMBER_2, peerDown.getMemberName());
782 MessageCollectorActor.clearMessages(mockShardActor1);
785 MockClusterWrapper.createMemberRemoved("member-2", "akka://cluster-test@127.0.0.1:2558"),
788 MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
790 shardManager1.tell(new FindPrimary("default", true), getRef());
792 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
795 MockClusterWrapper.createReachableMember("member-2", "akka://cluster-test@127.0.0.1:2558"),
798 shardManager1.underlyingActor().waitForReachableMember();
800 PeerUp peerUp = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
801 assertEquals("getMemberName", MEMBER_2, peerUp.getMemberName());
802 MessageCollectorActor.clearMessages(mockShardActor1);
804 shardManager1.tell(new FindPrimary("default", true), getRef());
806 RemotePrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
807 String path1 = found1.getPrimaryPath();
808 assertTrue("Unexpected primary path " + path1, path1.contains("member-2-shard-default-config"));
811 MockClusterWrapper.createMemberUp("member-2", "akka://cluster-test@127.0.0.1:2558"),
814 MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
816 // Test FindPrimary wait succeeds after reachable member event.
818 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
819 "akka://cluster-test@127.0.0.1:2558"), getRef());
820 shardManager1.underlyingActor().waitForUnreachableMember();
822 shardManager1.tell(new FindPrimary("default", true), getRef());
825 MockClusterWrapper.createReachableMember("member-2", "akka://cluster-test@127.0.0.1:2558"),
828 RemotePrimaryShardFound found2 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
829 String path2 = found2.getPrimaryPath();
830 assertTrue("Unexpected primary path " + path2, path2.contains("member-2-shard-default-config"));
834 LOG.info("testShardAvailabilityOnChangeOfMemberReachability ending");
838 public void testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange() {
839 LOG.info("testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange starting");
840 String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
842 // Create an ActorSystem ShardManager actor for member-1.
844 final ActorSystem system1 = newActorSystem("Member1");
845 Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
847 final ActorRef mockShardActor1 = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
849 final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
850 final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
851 newTestShardMgrBuilder().shardActor(mockShardActor1).cluster(new ClusterWrapperImpl(system1))
852 .primaryShardInfoCache(primaryShardInfoCache).props()
853 .withDispatcher(Dispatchers.DefaultDispatcherId()),
856 // Create an ActorSystem ShardManager actor for member-2.
858 final ActorSystem system2 = newActorSystem("Member2");
860 Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
862 final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
864 MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
865 .put("default", Arrays.asList("member-1", "member-2")).build());
867 final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
868 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
869 new ClusterWrapperImpl(system2)).props().withDispatcher(
870 Dispatchers.DefaultDispatcherId()), shardManagerID);
872 new TestKit(system1) {
874 shardManager1.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
875 shardManager2.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
876 shardManager1.tell(new ActorInitialized(), mockShardActor1);
877 shardManager2.tell(new ActorInitialized(), mockShardActor2);
879 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
880 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
881 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class),
882 DataStoreVersions.CURRENT_VERSION), mockShardActor1);
884 new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
886 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class),
887 DataStoreVersions.CURRENT_VERSION), mockShardActor2);
889 new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
891 shardManager1.underlyingActor().waitForMemberUp();
893 shardManager1.tell(new FindPrimary("default", true), getRef());
895 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
896 String path = found.getPrimaryPath();
897 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
899 primaryShardInfoCache.putSuccessful("default", new PrimaryShardInfo(
900 system1.actorSelection(mockShardActor1.path()), DataStoreVersions.CURRENT_VERSION));
902 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
903 "akka://cluster-test@127.0.0.1:2558"), getRef());
905 shardManager1.underlyingActor().waitForUnreachableMember();
907 shardManager1.tell(new FindPrimary("default", true), getRef());
909 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
911 assertNull("Expected primaryShardInfoCache entry removed",
912 primaryShardInfoCache.getIfPresent("default"));
914 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId1, mock(DataTree.class),
915 DataStoreVersions.CURRENT_VERSION), mockShardActor1);
917 new RoleChangeNotification(memberId1, RaftState.Follower.name(), RaftState.Leader.name()),
920 shardManager1.tell(new FindPrimary("default", true), getRef());
922 LocalPrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
923 String path1 = found1.getPrimaryPath();
924 assertTrue("Unexpected primary path " + path1, path1.contains("member-1-shard-default-config"));
929 LOG.info("testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange ending");
933 public void testShardAvailabilityChangeOnMemberWithNameContainedInLeaderIdUnreachable() {
934 LOG.info("testShardAvailabilityChangeOnMemberWithNameContainedInLeaderIdUnreachable starting");
935 String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
937 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
938 .put("default", Arrays.asList("member-256", "member-2")).build());
940 // Create an ActorSystem, ShardManager and actor for member-256.
942 final ActorSystem system256 = newActorSystem("Member256");
943 // 2562 is the tcp port of Member256 in src/test/resources/application.conf.
944 Cluster.get(system256).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2562"));
946 final ActorRef mockShardActor256 = newMockShardActor(system256, Shard.DEFAULT_NAME, "member-256");
948 final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
950 // ShardManager must be created with shard configuration to let its localShards has shards.
951 final TestActorRef<TestShardManager> shardManager256 = TestActorRef.create(system256,
952 newTestShardMgrBuilder(mockConfig).shardActor(mockShardActor256)
953 .cluster(new ClusterWrapperImpl(system256))
954 .primaryShardInfoCache(primaryShardInfoCache).props()
955 .withDispatcher(Dispatchers.DefaultDispatcherId()),
958 // Create an ActorSystem, ShardManager and actor for member-2 whose name is contained in member-256.
960 final ActorSystem system2 = newActorSystem("Member2");
962 // Join member-2 into the cluster of member-256.
963 Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2562"));
965 final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
967 final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
968 newTestShardMgrBuilder(mockConfig).shardActor(mockShardActor2).cluster(
969 new ClusterWrapperImpl(system2)).props().withDispatcher(
970 Dispatchers.DefaultDispatcherId()), shardManagerID);
972 new TestKit(system256) {
974 shardManager256.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
975 shardManager2.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
976 shardManager256.tell(new ActorInitialized(), mockShardActor256);
977 shardManager2.tell(new ActorInitialized(), mockShardActor2);
979 String memberId256 = "member-256-shard-default-" + shardMrgIDSuffix;
980 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
981 shardManager256.tell(new ShardLeaderStateChanged(memberId256, memberId256, mock(DataTree.class),
982 DataStoreVersions.CURRENT_VERSION), mockShardActor256);
983 shardManager256.tell(
984 new RoleChangeNotification(memberId256, RaftState.Candidate.name(), RaftState.Leader.name()),
986 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId256, mock(DataTree.class),
987 DataStoreVersions.CURRENT_VERSION), mockShardActor2);
989 new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Follower.name()),
991 shardManager256.underlyingActor().waitForMemberUp();
993 shardManager256.tell(new FindPrimary("default", true), getRef());
995 LocalPrimaryShardFound found = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
996 String path = found.getPrimaryPath();
997 assertTrue("Unexpected primary path " + path + " which must on member-256",
998 path.contains("member-256-shard-default-config"));
1000 PrimaryShardInfo primaryShardInfo = new PrimaryShardInfo(
1001 system256.actorSelection(mockShardActor256.path()), DataStoreVersions.CURRENT_VERSION);
1002 primaryShardInfoCache.putSuccessful("default", primaryShardInfo);
1004 // Simulate member-2 become unreachable.
1005 shardManager256.tell(MockClusterWrapper.createUnreachableMember("member-2",
1006 "akka://cluster-test@127.0.0.1:2558"), getRef());
1007 shardManager256.underlyingActor().waitForUnreachableMember();
1009 // Make sure leader shard on member-256 is still leader and still in the cache.
1010 shardManager256.tell(new FindPrimary("default", true), getRef());
1011 found = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
1012 path = found.getPrimaryPath();
1013 assertTrue("Unexpected primary path " + path + " which must still not on member-256",
1014 path.contains("member-256-shard-default-config"));
1015 Future<PrimaryShardInfo> futurePrimaryShard = primaryShardInfoCache.getIfPresent("default");
1016 futurePrimaryShard.onComplete(new OnComplete<PrimaryShardInfo>() {
1018 public void onComplete(final Throwable failure, final PrimaryShardInfo futurePrimaryShardInfo) {
1019 if (failure != null) {
1020 assertTrue("Primary shard info is unexpectedly removed from primaryShardInfoCache", false);
1022 assertEquals("Expected primaryShardInfoCache entry",
1023 primaryShardInfo, futurePrimaryShardInfo);
1026 }, system256.dispatchers().defaultGlobalDispatcher());
1030 LOG.info("testShardAvailabilityChangeOnMemberWithNameContainedInLeaderIdUnreachable ending");
1034 public void testOnReceiveFindLocalShardForNonExistentShard() {
1035 new TestKit(getSystem()) {
1037 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1039 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1041 shardManager.tell(new FindLocalShard("non-existent", false), getRef());
1043 LocalShardNotFound notFound = expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
1045 assertEquals("getShardName", "non-existent", notFound.getShardName());
1051 public void testOnReceiveFindLocalShardForExistentShard() {
1052 new TestKit(getSystem()) {
1054 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1056 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1057 shardManager.tell(new ActorInitialized(), mockShardActor);
1059 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1061 LocalShardFound found = expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1063 assertTrue("Found path contains " + found.getPath().path().toString(),
1064 found.getPath().path().toString().contains("member-1-shard-default-config"));
1070 public void testOnReceiveFindLocalShardForNotInitializedShard() {
1071 new TestKit(getSystem()) {
1073 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1075 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1077 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1083 public void testOnReceiveFindLocalShardWaitForShardInitialized() throws Exception {
1084 LOG.info("testOnReceiveFindLocalShardWaitForShardInitialized starting");
1085 new TestKit(getSystem()) {
1087 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1089 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1091 // We're passing waitUntilInitialized = true to FindLocalShard
1092 // so the response should be
1093 // delayed until we send ActorInitialized.
1094 Future<Object> future = Patterns.ask(shardManager, new FindLocalShard(Shard.DEFAULT_NAME, true),
1095 new Timeout(5, TimeUnit.SECONDS));
1097 shardManager.tell(new ActorInitialized(), mockShardActor);
1099 Object resp = Await.result(future, duration("5 seconds"));
1100 assertTrue("Expected: LocalShardFound, Actual: " + resp, resp instanceof LocalShardFound);
1104 LOG.info("testOnReceiveFindLocalShardWaitForShardInitialized starting");
1108 public void testRoleChangeNotificationAndShardLeaderStateChangedReleaseReady() throws Exception {
1109 TestShardManager shardManager = newTestShardManager();
1111 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1112 shardManager.onReceiveCommand(new RoleChangeNotification(
1113 memberId, RaftState.Candidate.name(), RaftState.Leader.name()));
1115 verify(ready, never()).countDown();
1117 shardManager.onReceiveCommand(new ShardLeaderStateChanged(memberId, memberId,
1118 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1120 verify(ready, times(1)).countDown();
1124 public void testRoleChangeNotificationToFollowerWithShardLeaderStateChangedReleaseReady() throws Exception {
1125 new TestKit(getSystem()) {
1127 TestShardManager shardManager = newTestShardManager();
1129 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1130 shardManager.onReceiveCommand(new RoleChangeNotification(memberId, null, RaftState.Follower.name()));
1132 verify(ready, never()).countDown();
1135 .onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
1137 shardManager.onReceiveCommand(
1138 new ShardLeaderStateChanged(memberId, "member-2-shard-default-" + shardMrgIDSuffix,
1139 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1141 verify(ready, times(1)).countDown();
1147 public void testReadyCountDownForMemberUpAfterLeaderStateChanged() throws Exception {
1148 new TestKit(getSystem()) {
1150 TestShardManager shardManager = newTestShardManager();
1152 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1153 shardManager.onReceiveCommand(new RoleChangeNotification(memberId, null, RaftState.Follower.name()));
1155 verify(ready, never()).countDown();
1157 shardManager.onReceiveCommand(
1158 new ShardLeaderStateChanged(memberId, "member-2-shard-default-" + shardMrgIDSuffix,
1159 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1162 .onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
1164 verify(ready, times(1)).countDown();
1170 public void testRoleChangeNotificationDoNothingForUnknownShard() throws Exception {
1171 TestShardManager shardManager = newTestShardManager();
1173 shardManager.onReceiveCommand(new RoleChangeNotification(
1174 "unknown", RaftState.Candidate.name(), RaftState.Leader.name()));
1176 verify(ready, never()).countDown();
1180 public void testByDefaultSyncStatusIsFalse() {
1181 TestShardManager shardManager = newTestShardManager();
1183 assertEquals(false, shardManager.getMBean().getSyncStatus());
1187 public void testWhenShardIsLeaderSyncStatusIsTrue() throws Exception {
1188 TestShardManager shardManager = newTestShardManager();
1190 shardManager.onReceiveCommand(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix,
1191 RaftState.Follower.name(), RaftState.Leader.name()));
1193 assertEquals(true, shardManager.getMBean().getSyncStatus());
1197 public void testWhenShardIsCandidateSyncStatusIsFalse() throws Exception {
1198 TestShardManager shardManager = newTestShardManager();
1200 String shardId = "member-1-shard-default-" + shardMrgIDSuffix;
1201 shardManager.onReceiveCommand(new RoleChangeNotification(shardId,
1202 RaftState.Follower.name(), RaftState.Candidate.name()));
1204 assertEquals(false, shardManager.getMBean().getSyncStatus());
1206 // Send a FollowerInitialSyncStatus with status = true for the replica whose current state is candidate
1207 shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(
1210 assertEquals(false, shardManager.getMBean().getSyncStatus());
1214 public void testWhenShardIsFollowerSyncStatusDependsOnFollowerInitialSyncStatus() throws Exception {
1215 TestShardManager shardManager = newTestShardManager();
1217 String shardId = "member-1-shard-default-" + shardMrgIDSuffix;
1218 shardManager.onReceiveCommand(new RoleChangeNotification(shardId,
1219 RaftState.Candidate.name(), RaftState.Follower.name()));
1221 // Initially will be false
1222 assertEquals(false, shardManager.getMBean().getSyncStatus());
1224 // Send status true will make sync status true
1225 shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(true, shardId));
1227 assertEquals(true, shardManager.getMBean().getSyncStatus());
1229 // Send status false will make sync status false
1230 shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(false, shardId));
1232 assertEquals(false, shardManager.getMBean().getSyncStatus());
1237 public void testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards() throws Exception {
1238 LOG.info("testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards starting");
1239 TestShardManager shardManager = newTestShardManager(newShardMgrProps(new MockConfiguration() {
1241 public List<String> getMemberShardNames(final MemberName memberName) {
1242 return Arrays.asList("default", "astronauts");
1246 // Initially will be false
1247 assertEquals(false, shardManager.getMBean().getSyncStatus());
1249 // Make default shard leader
1250 String defaultShardId = "member-1-shard-default-" + shardMrgIDSuffix;
1251 shardManager.onReceiveCommand(new RoleChangeNotification(defaultShardId,
1252 RaftState.Follower.name(), RaftState.Leader.name()));
1254 // default = Leader, astronauts is unknown so sync status remains false
1255 assertEquals(false, shardManager.getMBean().getSyncStatus());
1257 // Make astronauts shard leader as well
1258 String astronautsShardId = "member-1-shard-astronauts-" + shardMrgIDSuffix;
1259 shardManager.onReceiveCommand(new RoleChangeNotification(astronautsShardId,
1260 RaftState.Follower.name(), RaftState.Leader.name()));
1262 // Now sync status should be true
1263 assertEquals(true, shardManager.getMBean().getSyncStatus());
1265 // Make astronauts a Follower
1266 shardManager.onReceiveCommand(new RoleChangeNotification(astronautsShardId,
1267 RaftState.Leader.name(), RaftState.Follower.name()));
1269 // Sync status is not true
1270 assertEquals(false, shardManager.getMBean().getSyncStatus());
1272 // Make the astronauts follower sync status true
1273 shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(true, astronautsShardId));
1275 // Sync status is now true
1276 assertEquals(true, shardManager.getMBean().getSyncStatus());
1278 LOG.info("testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards ending");
1282 public void testOnReceiveSwitchShardBehavior() {
1283 new TestKit(getSystem()) {
1285 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1287 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1288 shardManager.tell(new ActorInitialized(), mockShardActor);
1290 shardManager.tell(new SwitchShardBehavior(mockShardName, RaftState.Leader, 1000), getRef());
1292 SwitchBehavior switchBehavior = MessageCollectorActor.expectFirstMatching(mockShardActor,
1293 SwitchBehavior.class);
1295 assertEquals(RaftState.Leader, switchBehavior.getNewState());
1296 assertEquals(1000, switchBehavior.getNewTerm());
1301 private static List<MemberName> members(final String... names) {
1302 return Arrays.asList(names).stream().map(MemberName::forName).collect(Collectors.toList());
1306 public void testOnCreateShard() {
1307 LOG.info("testOnCreateShard starting");
1308 new TestKit(getSystem()) {
1310 datastoreContextBuilder.shardInitializationTimeout(1, TimeUnit.MINUTES).persistent(true);
1312 ActorRef shardManager = actorFactory
1313 .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1314 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1316 SchemaContext schemaContext = TEST_SCHEMA_CONTEXT;
1317 shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
1319 DatastoreContext datastoreContext = DatastoreContext.newBuilder().shardElectionTimeoutFactor(100)
1320 .persistent(false).build();
1321 Shard.Builder shardBuilder = Shard.builder();
1323 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1324 "foo", null, members("member-1", "member-5", "member-6"));
1325 shardManager.tell(new CreateShard(config, shardBuilder, datastoreContext), getRef());
1327 expectMsgClass(duration("5 seconds"), Success.class);
1329 shardManager.tell(new FindLocalShard("foo", true), getRef());
1331 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1333 assertEquals("isRecoveryApplicable", false, shardBuilder.getDatastoreContext().isPersistent());
1334 assertTrue("Epxected ShardPeerAddressResolver", shardBuilder.getDatastoreContext().getShardRaftConfig()
1335 .getPeerAddressResolver() instanceof ShardPeerAddressResolver);
1336 assertEquals("peerMembers", Sets.newHashSet(
1337 ShardIdentifier.create("foo", MemberName.forName("member-5"), shardMrgIDSuffix).toString(),
1338 ShardIdentifier.create("foo", MemberName.forName("member-6"), shardMrgIDSuffix).toString()),
1339 shardBuilder.getPeerAddresses().keySet());
1340 assertEquals("ShardIdentifier", ShardIdentifier.create("foo", MEMBER_1, shardMrgIDSuffix),
1341 shardBuilder.getId());
1342 assertSame("schemaContext", schemaContext, shardBuilder.getSchemaContext());
1344 // Send CreateShard with same name - should return Success with
1347 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1349 Success success = expectMsgClass(duration("5 seconds"), Success.class);
1350 assertNotNull("Success status is null", success.status());
1354 LOG.info("testOnCreateShard ending");
1358 public void testOnCreateShardWithLocalMemberNotInShardConfig() {
1359 LOG.info("testOnCreateShardWithLocalMemberNotInShardConfig starting");
1360 new TestKit(getSystem()) {
1362 datastoreContextBuilder.shardInitializationTimeout(1, TimeUnit.MINUTES).persistent(true);
1364 ActorRef shardManager = actorFactory
1365 .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1366 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1368 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), ActorRef.noSender());
1370 Shard.Builder shardBuilder = Shard.builder();
1371 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1372 "foo", null, members("member-5", "member-6"));
1374 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1375 expectMsgClass(duration("5 seconds"), Success.class);
1377 shardManager.tell(new FindLocalShard("foo", true), getRef());
1378 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1380 assertEquals("peerMembers size", 0, shardBuilder.getPeerAddresses().size());
1381 assertEquals("schemaContext", DisableElectionsRaftPolicy.class.getName(), shardBuilder
1382 .getDatastoreContext().getShardRaftConfig().getCustomRaftPolicyImplementationClass());
1386 LOG.info("testOnCreateShardWithLocalMemberNotInShardConfig ending");
1390 public void testOnCreateShardWithNoInitialSchemaContext() {
1391 LOG.info("testOnCreateShardWithNoInitialSchemaContext starting");
1392 new TestKit(getSystem()) {
1394 ActorRef shardManager = actorFactory
1395 .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1396 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1398 Shard.Builder shardBuilder = Shard.builder();
1400 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1401 "foo", null, members("member-1"));
1402 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1404 expectMsgClass(duration("5 seconds"), Success.class);
1406 SchemaContext schemaContext = TEST_SCHEMA_CONTEXT;
1407 shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
1409 shardManager.tell(new FindLocalShard("foo", true), getRef());
1411 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1413 assertSame("schemaContext", schemaContext, shardBuilder.getSchemaContext());
1414 assertNotNull("schemaContext is null", shardBuilder.getDatastoreContext());
1418 LOG.info("testOnCreateShardWithNoInitialSchemaContext ending");
1422 public void testGetSnapshot() {
1423 LOG.info("testGetSnapshot starting");
1424 TestKit kit = new TestKit(getSystem());
1426 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1427 .put("shard1", Arrays.asList("member-1")).put("shard2", Arrays.asList("member-1"))
1428 .put("astronauts", Collections.<String>emptyList()).build());
1430 TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(newShardMgrProps(mockConfig)
1431 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1433 shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1434 Failure failure = kit.expectMsgClass(Failure.class);
1435 assertEquals("Failure cause type", IllegalStateException.class, failure.cause().getClass());
1437 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), ActorRef.noSender());
1439 waitForShardInitialized(shardManager, "shard1", kit);
1440 waitForShardInitialized(shardManager, "shard2", kit);
1442 shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1444 DatastoreSnapshot datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1446 assertEquals("getType", shardMrgIDSuffix, datastoreSnapshot.getType());
1447 assertNull("Expected null ShardManagerSnapshot", datastoreSnapshot.getShardManagerSnapshot());
1449 Function<ShardSnapshot, String> shardNameTransformer = ShardSnapshot::getName;
1451 assertEquals("Shard names", Sets.newHashSet("shard1", "shard2"), Sets.newHashSet(
1452 datastoreSnapshot.getShardSnapshots().stream().map(shardNameTransformer).collect(Collectors.toSet())));
1454 // Add a new replica
1456 TestKit mockShardLeaderKit = new TestKit(getSystem());
1458 TestShardManager shardManagerInstance = shardManager.underlyingActor();
1459 shardManagerInstance.setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1461 shardManager.tell(new AddShardReplica("astronauts"), kit.getRef());
1462 mockShardLeaderKit.expectMsgClass(AddServer.class);
1463 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.OK, ""));
1464 kit.expectMsgClass(Status.Success.class);
1465 waitForShardInitialized(shardManager, "astronauts", kit);
1467 // Send another GetSnapshot and verify
1469 shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1470 datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1472 assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"), Sets.newHashSet(
1473 Lists.transform(datastoreSnapshot.getShardSnapshots(), shardNameTransformer)));
1475 ShardManagerSnapshot snapshot = datastoreSnapshot.getShardManagerSnapshot();
1476 assertNotNull("Expected ShardManagerSnapshot", snapshot);
1477 assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"),
1478 Sets.newHashSet(snapshot.getShardList()));
1480 LOG.info("testGetSnapshot ending");
1484 public void testRestoreFromSnapshot() {
1485 LOG.info("testRestoreFromSnapshot starting");
1487 datastoreContextBuilder.shardInitializationTimeout(3, TimeUnit.SECONDS);
1489 TestKit kit = new TestKit(getSystem());
1491 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1492 .put("shard1", Collections.<String>emptyList()).put("shard2", Collections.<String>emptyList())
1493 .put("astronauts", Collections.<String>emptyList()).build());
1495 ShardManagerSnapshot snapshot =
1496 new ShardManagerSnapshot(Arrays.asList("shard1", "shard2", "astronauts"), Collections.emptyMap());
1497 DatastoreSnapshot restoreFromSnapshot = new DatastoreSnapshot(shardMrgIDSuffix, snapshot,
1498 Collections.<ShardSnapshot>emptyList());
1499 TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(newTestShardMgrBuilder(mockConfig)
1500 .restoreFromSnapshot(restoreFromSnapshot).props().withDispatcher(Dispatchers.DefaultDispatcherId()));
1502 shardManager.underlyingActor().waitForRecoveryComplete();
1504 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), ActorRef.noSender());
1506 waitForShardInitialized(shardManager, "shard1", kit);
1507 waitForShardInitialized(shardManager, "shard2", kit);
1508 waitForShardInitialized(shardManager, "astronauts", kit);
1510 shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1512 DatastoreSnapshot datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1514 assertEquals("getType", shardMrgIDSuffix, datastoreSnapshot.getType());
1516 assertNotNull("Expected ShardManagerSnapshot", datastoreSnapshot.getShardManagerSnapshot());
1517 assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"),
1518 Sets.newHashSet(datastoreSnapshot.getShardManagerSnapshot().getShardList()));
1520 LOG.info("testRestoreFromSnapshot ending");
1524 public void testAddShardReplicaForNonExistentShardConfig() {
1525 new TestKit(getSystem()) {
1527 ActorRef shardManager = actorFactory
1528 .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1529 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1531 shardManager.tell(new AddShardReplica("model-inventory"), getRef());
1532 Status.Failure resp = expectMsgClass(duration("2 seconds"), Status.Failure.class);
1534 assertEquals("Failure obtained", true, resp.cause() instanceof IllegalArgumentException);
1540 public void testAddShardReplica() {
1541 LOG.info("testAddShardReplica starting");
1542 MockConfiguration mockConfig = new MockConfiguration(
1543 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
1544 .put("astronauts", Arrays.asList("member-2")).build());
1546 final String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
1547 datastoreContextBuilder.shardManagerPersistenceId(shardManagerID);
1549 // Create an ActorSystem ShardManager actor for member-1.
1550 final ActorSystem system1 = newActorSystem("Member1");
1551 Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1552 ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
1553 final TestActorRef<TestShardManager> newReplicaShardManager = TestActorRef.create(system1,
1554 newTestShardMgrBuilder(mockConfig).shardActor(mockDefaultShardActor)
1555 .cluster(new ClusterWrapperImpl(system1)).props()
1556 .withDispatcher(Dispatchers.DefaultDispatcherId()),
1559 // Create an ActorSystem ShardManager actor for member-2.
1560 final ActorSystem system2 = newActorSystem("Member2");
1561 Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1563 String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
1564 String name = ShardIdentifier.create("astronauts", MEMBER_2, "config").toString();
1565 final TestActorRef<MockRespondActor> mockShardLeaderActor = TestActorRef.create(system2,
1566 Props.create(MockRespondActor.class, AddServer.class,
1567 new AddServerReply(ServerChangeStatus.OK, memberId2))
1568 .withDispatcher(Dispatchers.DefaultDispatcherId()),
1570 final TestActorRef<TestShardManager> leaderShardManager = TestActorRef.create(system2,
1571 newTestShardMgrBuilder(mockConfig).shardActor(mockShardLeaderActor)
1572 .cluster(new ClusterWrapperImpl(system2)).props()
1573 .withDispatcher(Dispatchers.DefaultDispatcherId()),
1576 new TestKit(system1) {
1578 newReplicaShardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1579 leaderShardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1581 leaderShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1583 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
1584 leaderShardManager.tell(
1585 new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
1586 mockShardLeaderActor);
1587 leaderShardManager.tell(
1588 new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
1589 mockShardLeaderActor);
1591 newReplicaShardManager.underlyingActor().waitForMemberUp();
1592 leaderShardManager.underlyingActor().waitForMemberUp();
1594 // Have a dummy snapshot to be overwritten by the new data
1596 String[] restoredShards = { "default", "people" };
1597 ShardManagerSnapshot snapshot =
1598 new ShardManagerSnapshot(Arrays.asList(restoredShards), Collections.emptyMap());
1599 InMemorySnapshotStore.addSnapshot(shardManagerID, snapshot);
1600 Uninterruptibles.sleepUninterruptibly(2, TimeUnit.MILLISECONDS);
1602 InMemorySnapshotStore.addSnapshotSavedLatch(shardManagerID);
1603 InMemorySnapshotStore.addSnapshotDeletedLatch(shardManagerID);
1605 // construct a mock response message
1606 newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
1607 AddServer addServerMsg = MessageCollectorActor.expectFirstMatching(mockShardLeaderActor,
1609 String addServerId = "member-1-shard-astronauts-" + shardMrgIDSuffix;
1610 assertEquals("AddServer serverId", addServerId, addServerMsg.getNewServerId());
1611 expectMsgClass(duration("5 seconds"), Status.Success.class);
1613 InMemorySnapshotStore.waitForSavedSnapshot(shardManagerID, ShardManagerSnapshot.class);
1614 InMemorySnapshotStore.waitForDeletedSnapshot(shardManagerID);
1615 List<ShardManagerSnapshot> persistedSnapshots = InMemorySnapshotStore.getSnapshots(shardManagerID,
1616 ShardManagerSnapshot.class);
1617 assertEquals("Number of snapshots persisted", 1, persistedSnapshots.size());
1618 ShardManagerSnapshot shardManagerSnapshot = persistedSnapshots.get(0);
1619 assertEquals("Persisted local shards", Sets.newHashSet("default", "astronauts"),
1620 Sets.newHashSet(shardManagerSnapshot.getShardList()));
1623 LOG.info("testAddShardReplica ending");
1627 public void testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader() {
1628 LOG.info("testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader starting");
1629 new TestKit(getSystem()) {
1631 TestActorRef<TestShardManager> shardManager = actorFactory
1632 .createTestActor(newPropsShardMgrWithMockShardActor(), shardMgrID);
1634 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1635 shardManager.tell(new ActorInitialized(), mockShardActor);
1637 String leaderId = "leader-member-shard-default-" + shardMrgIDSuffix;
1638 AddServerReply addServerReply = new AddServerReply(ServerChangeStatus.ALREADY_EXISTS, null);
1639 ActorRef leaderShardActor = shardManager.underlyingActor().getContext()
1640 .actorOf(Props.create(MockRespondActor.class, AddServer.class, addServerReply), leaderId);
1642 MockClusterWrapper.sendMemberUp(shardManager, "leader-member", leaderShardActor.path().toString());
1644 String newReplicaId = "member-1-shard-default-" + shardMrgIDSuffix;
1646 new RoleChangeNotification(newReplicaId, RaftState.Candidate.name(), RaftState.Follower.name()),
1649 new ShardLeaderStateChanged(newReplicaId, leaderId, DataStoreVersions.CURRENT_VERSION),
1652 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1654 MessageCollectorActor.expectFirstMatching(leaderShardActor, AddServer.class);
1656 Failure resp = expectMsgClass(duration("5 seconds"), Failure.class);
1657 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1659 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1660 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1662 // Send message again to verify previous in progress state is
1665 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1666 resp = expectMsgClass(duration("5 seconds"), Failure.class);
1667 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1669 // Send message again with an AddServer timeout to verify the
1670 // pre-existing shard actor isn't terminated.
1673 newDatastoreContextFactory(
1674 datastoreContextBuilder.shardLeaderElectionTimeout(100, TimeUnit.MILLISECONDS).build()),
1676 leaderShardActor.tell(MockRespondActor.CLEAR_RESPONSE, ActorRef.noSender());
1677 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1678 expectMsgClass(duration("5 seconds"), Failure.class);
1680 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1681 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1685 LOG.info("testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader ending");
1689 public void testAddShardReplicaWithPreExistingLocalReplicaLeader() {
1690 LOG.info("testAddShardReplicaWithPreExistingLocalReplicaLeader starting");
1691 new TestKit(getSystem()) {
1693 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1694 ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1696 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1697 shardManager.tell(new ActorInitialized(), mockShardActor);
1698 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
1699 DataStoreVersions.CURRENT_VERSION), getRef());
1701 new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
1704 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1705 Failure resp = expectMsgClass(duration("5 seconds"), Failure.class);
1706 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1708 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1709 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1713 LOG.info("testAddShardReplicaWithPreExistingLocalReplicaLeader ending");
1717 public void testAddShardReplicaWithAddServerReplyFailure() {
1718 LOG.info("testAddShardReplicaWithAddServerReplyFailure starting");
1719 new TestKit(getSystem()) {
1721 TestKit mockShardLeaderKit = new TestKit(getSystem());
1723 MockConfiguration mockConfig = new MockConfiguration(
1724 ImmutableMap.of("astronauts", Arrays.asList("member-2")));
1726 ActorRef mockNewReplicaShardActor = newMockShardActor(getSystem(), "astronauts", "member-1");
1727 final TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(
1728 newTestShardMgrBuilder(mockConfig).shardActor(mockNewReplicaShardActor).props()
1729 .withDispatcher(Dispatchers.DefaultDispatcherId()), shardMgrID);
1730 shardManager.underlyingActor()
1731 .setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1733 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1735 TestKit terminateWatcher = new TestKit(getSystem());
1736 terminateWatcher.watch(mockNewReplicaShardActor);
1738 shardManager.tell(new AddShardReplica("astronauts"), getRef());
1740 AddServer addServerMsg = mockShardLeaderKit.expectMsgClass(AddServer.class);
1741 assertEquals("AddServer serverId", "member-1-shard-astronauts-" + shardMrgIDSuffix,
1742 addServerMsg.getNewServerId());
1743 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.TIMEOUT, null));
1745 Failure failure = expectMsgClass(duration("5 seconds"), Failure.class);
1746 assertEquals("Failure cause", TimeoutException.class, failure.cause().getClass());
1748 shardManager.tell(new FindLocalShard("astronauts", false), getRef());
1749 expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
1751 terminateWatcher.expectTerminated(mockNewReplicaShardActor);
1753 shardManager.tell(new AddShardReplica("astronauts"), getRef());
1754 mockShardLeaderKit.expectMsgClass(AddServer.class);
1755 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.NO_LEADER, null));
1756 failure = expectMsgClass(duration("5 seconds"), Failure.class);
1757 assertEquals("Failure cause", NoShardLeaderException.class, failure.cause().getClass());
1761 LOG.info("testAddShardReplicaWithAddServerReplyFailure ending");
1765 public void testAddShardReplicaWithAlreadyInProgress() {
1766 testServerChangeWhenAlreadyInProgress("astronauts", new AddShardReplica("astronauts"),
1767 AddServer.class, new AddShardReplica("astronauts"));
1771 public void testAddShardReplicaWithFindPrimaryTimeout() {
1772 LOG.info("testAddShardReplicaWithFindPrimaryTimeout starting");
1773 datastoreContextBuilder.shardInitializationTimeout(100, TimeUnit.MILLISECONDS);
1774 new TestKit(getSystem()) {
1776 MockConfiguration mockConfig = new MockConfiguration(
1777 ImmutableMap.of("astronauts", Arrays.asList("member-2")));
1779 final ActorRef newReplicaShardManager = actorFactory
1780 .createActor(newTestShardMgrBuilder(mockConfig).shardActor(mockShardActor).props()
1781 .withDispatcher(Dispatchers.DefaultDispatcherId()), shardMgrID);
1783 newReplicaShardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1784 MockClusterWrapper.sendMemberUp(newReplicaShardManager, "member-2",
1785 AddressFromURIString.parse("akka://non-existent@127.0.0.1:5").toString());
1787 newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
1788 Status.Failure resp = expectMsgClass(duration("5 seconds"), Status.Failure.class);
1789 assertEquals("Failure obtained", true, resp.cause() instanceof RuntimeException);
1793 LOG.info("testAddShardReplicaWithFindPrimaryTimeout ending");
1797 public void testRemoveShardReplicaForNonExistentShard() {
1798 new TestKit(getSystem()) {
1800 ActorRef shardManager = actorFactory
1801 .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1802 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1804 shardManager.tell(new RemoveShardReplica("model-inventory", MEMBER_1), getRef());
1805 Status.Failure resp = expectMsgClass(duration("10 seconds"), Status.Failure.class);
1806 assertEquals("Failure obtained", true, resp.cause() instanceof PrimaryNotFoundException);
1815 public void testRemoveShardReplicaLocal() {
1816 new TestKit(getSystem()) {
1818 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1820 final ActorRef respondActor = actorFactory.createActor(Props.create(MockRespondActor.class,
1821 RemoveServer.class, new RemoveServerReply(ServerChangeStatus.OK, null)), memberId);
1823 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
1825 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1826 shardManager.tell(new ActorInitialized(), respondActor);
1827 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
1828 DataStoreVersions.CURRENT_VERSION), getRef());
1830 new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
1833 shardManager.tell(new RemoveShardReplica(Shard.DEFAULT_NAME, MEMBER_1), getRef());
1834 final RemoveServer removeServer = MessageCollectorActor.expectFirstMatching(respondActor,
1835 RemoveServer.class);
1836 assertEquals(ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString(),
1837 removeServer.getServerId());
1838 expectMsgClass(duration("5 seconds"), Success.class);
1844 public void testRemoveShardReplicaRemote() {
1845 MockConfiguration mockConfig = new MockConfiguration(
1846 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
1847 .put("astronauts", Arrays.asList("member-1")).build());
1849 String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
1851 // Create an ActorSystem ShardManager actor for member-1.
1852 final ActorSystem system1 = newActorSystem("Member1");
1853 Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1854 ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
1856 final TestActorRef<TestShardManager> newReplicaShardManager = TestActorRef.create(system1,
1857 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockDefaultShardActor).cluster(
1858 new ClusterWrapperImpl(system1)).props().withDispatcher(Dispatchers.DefaultDispatcherId()),
1861 // Create an ActorSystem ShardManager actor for member-2.
1862 final ActorSystem system2 = newActorSystem("Member2");
1863 Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1865 String name = ShardIdentifier.create("default", MEMBER_2, shardMrgIDSuffix).toString();
1866 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
1867 final TestActorRef<MockRespondActor> mockShardLeaderActor =
1868 TestActorRef.create(system2, Props.create(MockRespondActor.class, RemoveServer.class,
1869 new RemoveServerReply(ServerChangeStatus.OK, memberId2)), name);
1871 LOG.error("Mock Shard Leader Actor : {}", mockShardLeaderActor);
1873 final TestActorRef<TestShardManager> leaderShardManager = TestActorRef.create(system2,
1874 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockShardLeaderActor).cluster(
1875 new ClusterWrapperImpl(system2)).props().withDispatcher(Dispatchers.DefaultDispatcherId()),
1878 // Because mockShardLeaderActor is created at the top level of the actor system it has an address like so,
1879 // akka://cluster-test@127.0.0.1:2559/user/member-2-shard-default-config1
1880 // However when a shard manager has a local shard which is a follower and a leader that is remote it will
1881 // try to compute an address for the remote shard leader using the ShardPeerAddressResolver. This address will
1883 // akka://cluster-test@127.0.0.1:2559/user/shardmanager-config1/member-2-shard-default-config1
1884 // In this specific case if we did a FindPrimary for shard default from member-1 we would come up
1885 // with the address of an actor which does not exist, therefore any message sent to that actor would go to
1887 // To work around this problem we create a ForwardingActor with the right address and pass to it the
1888 // mockShardLeaderActor. The ForwardingActor simply forwards all messages to the mockShardLeaderActor and every
1889 // thing works as expected
1890 final ActorRef actorRef = leaderShardManager.underlyingActor().context()
1891 .actorOf(Props.create(ForwardingActor.class, mockShardLeaderActor),
1892 "member-2-shard-default-" + shardMrgIDSuffix);
1894 LOG.error("Forwarding actor : {}", actorRef);
1896 new TestKit(system1) {
1898 newReplicaShardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1899 leaderShardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1901 leaderShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1902 newReplicaShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1904 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
1905 leaderShardManager.tell(
1906 new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
1907 mockShardLeaderActor);
1908 leaderShardManager.tell(
1909 new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
1910 mockShardLeaderActor);
1912 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
1913 newReplicaShardManager.tell(
1914 new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class), leaderVersion),
1916 newReplicaShardManager.tell(
1917 new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
1920 newReplicaShardManager.underlyingActor().waitForMemberUp();
1921 leaderShardManager.underlyingActor().waitForMemberUp();
1923 // construct a mock response message
1924 newReplicaShardManager.tell(new RemoveShardReplica("default", MEMBER_1), getRef());
1925 RemoveServer removeServer = MessageCollectorActor.expectFirstMatching(mockShardLeaderActor,
1926 RemoveServer.class);
1927 String removeServerId = ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString();
1928 assertEquals("RemoveServer serverId", removeServerId, removeServer.getServerId());
1929 expectMsgClass(duration("5 seconds"), Status.Success.class);
1935 public void testRemoveShardReplicaWhenAnotherRemoveShardReplicaAlreadyInProgress() {
1936 testServerChangeWhenAlreadyInProgress("astronauts", new RemoveShardReplica("astronauts", MEMBER_2),
1937 RemoveServer.class, new RemoveShardReplica("astronauts", MEMBER_3));
1941 public void testRemoveShardReplicaWhenAddShardReplicaAlreadyInProgress() {
1942 testServerChangeWhenAlreadyInProgress("astronauts", new AddShardReplica("astronauts"),
1943 AddServer.class, new RemoveShardReplica("astronauts", MEMBER_2));
1947 public void testServerChangeWhenAlreadyInProgress(final String shardName, final Object firstServerChange,
1948 final Class<?> firstForwardedServerChangeClass,
1949 final Object secondServerChange) {
1950 new TestKit(getSystem()) {
1952 TestKit mockShardLeaderKit = new TestKit(getSystem());
1953 final TestKit secondRequestKit = new TestKit(getSystem());
1955 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1956 .put(shardName, Arrays.asList("member-2")).build());
1958 final TestActorRef<TestShardManager> shardManager = TestActorRef.create(getSystem(),
1959 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockShardActor)
1960 .cluster(new MockClusterWrapper()).props()
1961 .withDispatcher(Dispatchers.DefaultDispatcherId()),
1964 shardManager.underlyingActor()
1965 .setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1967 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1969 shardManager.tell(firstServerChange, getRef());
1971 mockShardLeaderKit.expectMsgClass(firstForwardedServerChangeClass);
1973 shardManager.tell(secondServerChange, secondRequestKit.getRef());
1975 secondRequestKit.expectMsgClass(duration("5 seconds"), Failure.class);
1981 public void testServerRemovedShardActorNotRunning() {
1982 LOG.info("testServerRemovedShardActorNotRunning starting");
1983 new TestKit(getSystem()) {
1985 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1986 .put("default", Arrays.asList("member-1", "member-2"))
1987 .put("astronauts", Arrays.asList("member-2"))
1988 .put("people", Arrays.asList("member-1", "member-2")).build());
1990 TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(
1991 newShardMgrProps(mockConfig).withDispatcher(Dispatchers.DefaultDispatcherId()));
1993 shardManager.underlyingActor().waitForRecoveryComplete();
1994 shardManager.tell(new FindLocalShard("people", false), getRef());
1995 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1997 shardManager.tell(new FindLocalShard("default", false), getRef());
1998 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
2000 // Removed the default shard replica from member-1
2001 ShardIdentifier.Builder builder = new ShardIdentifier.Builder();
2002 ShardIdentifier shardId = builder.shardName("default").memberName(MEMBER_1).type(shardMrgIDSuffix)
2004 shardManager.tell(new ServerRemoved(shardId.toString()), getRef());
2006 shardManager.underlyingActor().verifySnapshotPersisted(Sets.newHashSet("people"));
2010 LOG.info("testServerRemovedShardActorNotRunning ending");
2014 public void testServerRemovedShardActorRunning() {
2015 LOG.info("testServerRemovedShardActorRunning starting");
2016 new TestKit(getSystem()) {
2018 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
2019 .put("default", Arrays.asList("member-1", "member-2"))
2020 .put("astronauts", Arrays.asList("member-2"))
2021 .put("people", Arrays.asList("member-1", "member-2")).build());
2023 String shardId = ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString();
2024 ActorRef shard = actorFactory.createActor(MessageCollectorActor.props(), shardId);
2026 TestActorRef<TestShardManager> shardManager = actorFactory
2027 .createTestActor(newTestShardMgrBuilder(mockConfig).addShardActor("default", shard).props()
2028 .withDispatcher(Dispatchers.DefaultDispatcherId()));
2030 shardManager.underlyingActor().waitForRecoveryComplete();
2032 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
2033 shardManager.tell(new ActorInitialized(), shard);
2035 waitForShardInitialized(shardManager, "people", this);
2036 waitForShardInitialized(shardManager, "default", this);
2038 // Removed the default shard replica from member-1
2039 shardManager.tell(new ServerRemoved(shardId), getRef());
2041 shardManager.underlyingActor().verifySnapshotPersisted(Sets.newHashSet("people"));
2043 MessageCollectorActor.expectFirstMatching(shard, Shutdown.class);
2047 LOG.info("testServerRemovedShardActorRunning ending");
2051 public void testShardPersistenceWithRestoredData() {
2052 LOG.info("testShardPersistenceWithRestoredData starting");
2053 new TestKit(getSystem()) {
2055 MockConfiguration mockConfig =
2056 new MockConfiguration(ImmutableMap.<String, List<String>>builder()
2057 .put("default", Arrays.asList("member-1", "member-2"))
2058 .put("astronauts", Arrays.asList("member-2"))
2059 .put("people", Arrays.asList("member-1", "member-2")).build());
2060 String[] restoredShards = {"default", "astronauts"};
2061 ShardManagerSnapshot snapshot =
2062 new ShardManagerSnapshot(Arrays.asList(restoredShards), Collections.emptyMap());
2063 InMemorySnapshotStore.addSnapshot("shard-manager-" + shardMrgIDSuffix, snapshot);
2065 // create shardManager to come up with restored data
2066 TestActorRef<TestShardManager> newRestoredShardManager = actorFactory.createTestActor(
2067 newShardMgrProps(mockConfig).withDispatcher(Dispatchers.DefaultDispatcherId()));
2069 newRestoredShardManager.underlyingActor().waitForRecoveryComplete();
2071 newRestoredShardManager.tell(new FindLocalShard("people", false), getRef());
2072 LocalShardNotFound notFound = expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
2073 assertEquals("for uninitialized shard", "people", notFound.getShardName());
2075 // Verify a local shard is created for the restored shards,
2076 // although we expect a NotInitializedException for the shards
2077 // as the actor initialization
2078 // message is not sent for them
2079 newRestoredShardManager.tell(new FindLocalShard("default", false), getRef());
2080 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
2082 newRestoredShardManager.tell(new FindLocalShard("astronauts", false), getRef());
2083 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
2087 LOG.info("testShardPersistenceWithRestoredData ending");
2091 public void testShutDown() throws Exception {
2092 LOG.info("testShutDown starting");
2093 new TestKit(getSystem()) {
2095 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
2096 .put("shard1", Arrays.asList("member-1")).put("shard2", Arrays.asList("member-1")).build());
2098 String shardId1 = ShardIdentifier.create("shard1", MEMBER_1, shardMrgIDSuffix).toString();
2099 ActorRef shard1 = actorFactory.createActor(MessageCollectorActor.props(), shardId1);
2101 String shardId2 = ShardIdentifier.create("shard2", MEMBER_1, shardMrgIDSuffix).toString();
2102 ActorRef shard2 = actorFactory.createActor(MessageCollectorActor.props(), shardId2);
2104 ActorRef shardManager = actorFactory.createActor(newTestShardMgrBuilder(mockConfig)
2105 .addShardActor("shard1", shard1).addShardActor("shard2", shard2).props());
2107 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
2108 shardManager.tell(new ActorInitialized(), shard1);
2109 shardManager.tell(new ActorInitialized(), shard2);
2111 FiniteDuration duration = FiniteDuration.create(5, TimeUnit.SECONDS);
2112 Future<Boolean> stopFuture = Patterns.gracefulStop(shardManager, duration, Shutdown.INSTANCE);
2114 MessageCollectorActor.expectFirstMatching(shard1, Shutdown.class);
2115 MessageCollectorActor.expectFirstMatching(shard2, Shutdown.class);
2118 Await.ready(stopFuture, FiniteDuration.create(500, TimeUnit.MILLISECONDS));
2119 fail("ShardManager actor stopped without waiting for the Shards to be stopped");
2120 } catch (TimeoutException e) {
2124 actorFactory.killActor(shard1, this);
2125 actorFactory.killActor(shard2, this);
2127 Boolean stopped = Await.result(stopFuture, duration);
2128 assertEquals("Stopped", Boolean.TRUE, stopped);
2132 LOG.info("testShutDown ending");
2136 public void testChangeServersVotingStatus() {
2137 new TestKit(getSystem()) {
2139 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
2141 ActorRef respondActor = actorFactory
2142 .createActor(Props.create(MockRespondActor.class, ChangeServersVotingStatus.class,
2143 new ServerChangeReply(ServerChangeStatus.OK, null)), memberId);
2145 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
2147 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
2148 shardManager.tell(new ActorInitialized(), respondActor);
2149 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
2150 DataStoreVersions.CURRENT_VERSION), getRef());
2152 new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
2156 new ChangeShardMembersVotingStatus("default", ImmutableMap.of("member-2", Boolean.TRUE)),
2159 ChangeServersVotingStatus actualChangeStatusMsg = MessageCollectorActor
2160 .expectFirstMatching(respondActor, ChangeServersVotingStatus.class);
2161 assertEquals("ChangeServersVotingStatus map", actualChangeStatusMsg.getServerVotingStatusMap(),
2162 ImmutableMap.of(ShardIdentifier
2163 .create("default", MemberName.forName("member-2"), shardMrgIDSuffix).toString(),
2166 expectMsgClass(duration("5 seconds"), Success.class);
2172 public void testChangeServersVotingStatusWithNoLeader() {
2173 new TestKit(getSystem()) {
2175 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
2177 ActorRef respondActor = actorFactory
2178 .createActor(Props.create(MockRespondActor.class, ChangeServersVotingStatus.class,
2179 new ServerChangeReply(ServerChangeStatus.NO_LEADER, null)), memberId);
2181 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
2183 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
2184 shardManager.tell(new ActorInitialized(), respondActor);
2185 shardManager.tell(new RoleChangeNotification(memberId, null, RaftState.Follower.name()), respondActor);
2188 new ChangeShardMembersVotingStatus("default", ImmutableMap.of("member-2", Boolean.TRUE)),
2191 MessageCollectorActor.expectFirstMatching(respondActor, ChangeServersVotingStatus.class);
2193 Status.Failure resp = expectMsgClass(duration("5 seconds"), Status.Failure.class);
2194 assertEquals("Failure resposnse", true, resp.cause() instanceof NoShardLeaderException);
2199 public static class TestShardManager extends ShardManager {
2200 private final CountDownLatch recoveryComplete = new CountDownLatch(1);
2201 private final CountDownLatch snapshotPersist = new CountDownLatch(1);
2202 private ShardManagerSnapshot snapshot;
2203 private final Map<String, ActorRef> shardActors;
2204 private final ActorRef shardActor;
2205 private CountDownLatch findPrimaryMessageReceived = new CountDownLatch(1);
2206 private CountDownLatch memberUpReceived = new CountDownLatch(1);
2207 private CountDownLatch memberRemovedReceived = new CountDownLatch(1);
2208 private CountDownLatch memberUnreachableReceived = new CountDownLatch(1);
2209 private CountDownLatch memberReachableReceived = new CountDownLatch(1);
2210 private volatile MessageInterceptor messageInterceptor;
2212 TestShardManager(final Builder builder) {
2214 shardActor = builder.shardActor;
2215 shardActors = builder.shardActors;
2219 protected void handleRecover(final Object message) throws Exception {
2221 super.handleRecover(message);
2223 if (message instanceof RecoveryCompleted) {
2224 recoveryComplete.countDown();
2229 private void countDownIfOther(final Member member, final CountDownLatch latch) {
2230 if (!getCluster().getCurrentMemberName().equals(memberToName(member))) {
2236 public void handleCommand(final Object message) throws Exception {
2238 if (messageInterceptor != null && messageInterceptor.canIntercept(message)) {
2239 getSender().tell(messageInterceptor.apply(message), getSelf());
2241 super.handleCommand(message);
2244 if (message instanceof FindPrimary) {
2245 findPrimaryMessageReceived.countDown();
2246 } else if (message instanceof ClusterEvent.MemberUp) {
2247 countDownIfOther(((ClusterEvent.MemberUp) message).member(), memberUpReceived);
2248 } else if (message instanceof ClusterEvent.MemberRemoved) {
2249 countDownIfOther(((ClusterEvent.MemberRemoved) message).member(), memberRemovedReceived);
2250 } else if (message instanceof ClusterEvent.UnreachableMember) {
2251 countDownIfOther(((ClusterEvent.UnreachableMember) message).member(), memberUnreachableReceived);
2252 } else if (message instanceof ClusterEvent.ReachableMember) {
2253 countDownIfOther(((ClusterEvent.ReachableMember) message).member(), memberReachableReceived);
2258 void setMessageInterceptor(final MessageInterceptor messageInterceptor) {
2259 this.messageInterceptor = messageInterceptor;
2262 void waitForRecoveryComplete() {
2263 assertEquals("Recovery complete", true,
2264 Uninterruptibles.awaitUninterruptibly(recoveryComplete, 5, TimeUnit.SECONDS));
2267 public void waitForMemberUp() {
2268 assertEquals("MemberUp received", true,
2269 Uninterruptibles.awaitUninterruptibly(memberUpReceived, 5, TimeUnit.SECONDS));
2270 memberUpReceived = new CountDownLatch(1);
2273 void waitForMemberRemoved() {
2274 assertEquals("MemberRemoved received", true,
2275 Uninterruptibles.awaitUninterruptibly(memberRemovedReceived, 5, TimeUnit.SECONDS));
2276 memberRemovedReceived = new CountDownLatch(1);
2279 void waitForUnreachableMember() {
2280 assertEquals("UnreachableMember received", true,
2281 Uninterruptibles.awaitUninterruptibly(memberUnreachableReceived, 5, TimeUnit.SECONDS
2283 memberUnreachableReceived = new CountDownLatch(1);
2286 void waitForReachableMember() {
2287 assertEquals("ReachableMember received", true,
2288 Uninterruptibles.awaitUninterruptibly(memberReachableReceived, 5, TimeUnit.SECONDS));
2289 memberReachableReceived = new CountDownLatch(1);
2292 void verifyFindPrimary() {
2293 assertEquals("FindPrimary received", true,
2294 Uninterruptibles.awaitUninterruptibly(findPrimaryMessageReceived, 5, TimeUnit.SECONDS));
2295 findPrimaryMessageReceived = new CountDownLatch(1);
2298 public static Builder builder(final DatastoreContext.Builder datastoreContextBuilder) {
2299 return new Builder(datastoreContextBuilder);
2302 public static class Builder extends AbstractGenericCreator<Builder, TestShardManager> {
2303 private ActorRef shardActor;
2304 private final Map<String, ActorRef> shardActors = new HashMap<>();
2306 Builder(final DatastoreContext.Builder datastoreContextBuilder) {
2307 super(TestShardManager.class);
2308 datastoreContextFactory(newDatastoreContextFactory(datastoreContextBuilder.build()));
2311 Builder shardActor(final ActorRef newShardActor) {
2312 this.shardActor = newShardActor;
2316 Builder addShardActor(final String shardName, final ActorRef actorRef) {
2317 shardActors.put(shardName, actorRef);
2323 public void saveSnapshot(final Object obj) {
2324 snapshot = (ShardManagerSnapshot) obj;
2325 snapshotPersist.countDown();
2326 super.saveSnapshot(obj);
2329 void verifySnapshotPersisted(final Set<String> shardList) {
2330 assertEquals("saveSnapshot invoked", true,
2331 Uninterruptibles.awaitUninterruptibly(snapshotPersist, 5, TimeUnit.SECONDS));
2332 assertEquals("Shard Persisted", shardList, Sets.newHashSet(snapshot.getShardList()));
2336 protected ActorRef newShardActor(final ShardInformation info) {
2337 if (shardActors.get(info.getShardName()) != null) {
2338 return shardActors.get(info.getShardName());
2341 if (shardActor != null) {
2345 return super.newShardActor(info);
2349 private abstract static class AbstractGenericCreator<T extends AbstractGenericCreator<T, ?>, C extends ShardManager>
2350 extends AbstractShardManagerCreator<T> {
2351 private final Class<C> shardManagerClass;
2353 AbstractGenericCreator(final Class<C> shardManagerClass) {
2354 this.shardManagerClass = shardManagerClass;
2355 cluster(new MockClusterWrapper()).configuration(new MockConfiguration()).waitTillReadyCountDownLatch(ready)
2356 .primaryShardInfoCache(new PrimaryShardInfoFutureCache());
2360 public Props props() {
2362 return Props.create(shardManagerClass, this);
2366 private static class GenericCreator<C extends ShardManager> extends AbstractGenericCreator<GenericCreator<C>, C> {
2367 GenericCreator(final Class<C> shardManagerClass) {
2368 super(shardManagerClass);
2372 private static class DelegatingShardManagerCreator implements Creator<ShardManager> {
2373 private static final long serialVersionUID = 1L;
2374 private final Creator<ShardManager> delegate;
2376 DelegatingShardManagerCreator(final Creator<ShardManager> delegate) {
2377 this.delegate = delegate;
2381 public ShardManager create() throws Exception {
2382 return delegate.create();
2386 interface MessageInterceptor extends Function<Object, Object> {
2387 boolean canIntercept(Object message);
2390 private static MessageInterceptor newFindPrimaryInterceptor(final ActorRef primaryActor) {
2391 return new MessageInterceptor() {
2393 public Object apply(final Object message) {
2394 return new RemotePrimaryShardFound(Serialization.serializedActorPath(primaryActor), (short) 1);
2398 public boolean canIntercept(final Object message) {
2399 return message instanceof FindPrimary;
2404 private static class MockRespondActor extends MessageCollectorActor {
2405 static final String CLEAR_RESPONSE = "clear-response";
2407 private Object responseMsg;
2408 private final Class<?> requestClass;
2410 @SuppressWarnings("unused")
2411 MockRespondActor(final Class<?> requestClass, final Object responseMsg) {
2412 this.requestClass = requestClass;
2413 this.responseMsg = responseMsg;
2417 public void onReceive(final Object message) throws Exception {
2418 if (message.equals(CLEAR_RESPONSE)) {
2421 super.onReceive(message);
2422 if (message.getClass().equals(requestClass) && responseMsg != null) {
2423 getSender().tell(responseMsg, getSelf());