2 * Copyright (c) 2014, 2015 Cisco Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
9 package org.opendaylight.controller.cluster.datastore.shardmanager;
11 import static org.junit.Assert.assertEquals;
12 import static org.junit.Assert.assertNotNull;
13 import static org.junit.Assert.assertNull;
14 import static org.junit.Assert.assertSame;
15 import static org.junit.Assert.assertTrue;
16 import static org.junit.Assert.fail;
17 import static org.mockito.Mockito.mock;
18 import static org.mockito.Mockito.never;
19 import static org.mockito.Mockito.times;
20 import static org.mockito.Mockito.verify;
22 import akka.actor.ActorRef;
23 import akka.actor.ActorSystem;
24 import akka.actor.AddressFromURIString;
25 import akka.actor.Props;
26 import akka.actor.Status;
27 import akka.actor.Status.Failure;
28 import akka.actor.Status.Success;
29 import akka.cluster.Cluster;
30 import akka.cluster.ClusterEvent;
31 import akka.cluster.Member;
32 import akka.dispatch.Dispatchers;
33 import akka.dispatch.OnComplete;
34 import akka.japi.Creator;
35 import akka.pattern.Patterns;
36 import akka.persistence.RecoveryCompleted;
37 import akka.serialization.Serialization;
38 import akka.testkit.TestActorRef;
39 import akka.testkit.javadsl.TestKit;
40 import akka.util.Timeout;
41 import com.google.common.base.Function;
42 import com.google.common.base.Stopwatch;
43 import com.google.common.collect.ImmutableMap;
44 import com.google.common.collect.Lists;
45 import com.google.common.collect.Sets;
46 import com.google.common.util.concurrent.Uninterruptibles;
48 import java.util.AbstractMap;
49 import java.util.Arrays;
50 import java.util.Collection;
51 import java.util.Collections;
52 import java.util.HashMap;
53 import java.util.List;
55 import java.util.Map.Entry;
57 import java.util.concurrent.CountDownLatch;
58 import java.util.concurrent.TimeUnit;
59 import java.util.concurrent.TimeoutException;
60 import java.util.stream.Collectors;
61 import org.junit.Test;
62 import org.mockito.Mockito;
63 import org.opendaylight.controller.cluster.access.concepts.MemberName;
64 import org.opendaylight.controller.cluster.datastore.AbstractShardManagerTest;
65 import org.opendaylight.controller.cluster.datastore.ClusterWrapperImpl;
66 import org.opendaylight.controller.cluster.datastore.DataStoreVersions;
67 import org.opendaylight.controller.cluster.datastore.DatastoreContext;
68 import org.opendaylight.controller.cluster.datastore.DatastoreContextFactory;
69 import org.opendaylight.controller.cluster.datastore.DistributedDataStore;
70 import org.opendaylight.controller.cluster.datastore.Shard;
71 import org.opendaylight.controller.cluster.datastore.config.ConfigurationImpl;
72 import org.opendaylight.controller.cluster.datastore.config.EmptyModuleShardConfigProvider;
73 import org.opendaylight.controller.cluster.datastore.config.ModuleShardConfiguration;
74 import org.opendaylight.controller.cluster.datastore.exceptions.AlreadyExistsException;
75 import org.opendaylight.controller.cluster.datastore.exceptions.NoShardLeaderException;
76 import org.opendaylight.controller.cluster.datastore.exceptions.NotInitializedException;
77 import org.opendaylight.controller.cluster.datastore.exceptions.PrimaryNotFoundException;
78 import org.opendaylight.controller.cluster.datastore.identifiers.ShardIdentifier;
79 import org.opendaylight.controller.cluster.datastore.identifiers.ShardManagerIdentifier;
80 import org.opendaylight.controller.cluster.datastore.messages.ActorInitialized;
81 import org.opendaylight.controller.cluster.datastore.messages.AddShardReplica;
82 import org.opendaylight.controller.cluster.datastore.messages.ChangeShardMembersVotingStatus;
83 import org.opendaylight.controller.cluster.datastore.messages.CreateShard;
84 import org.opendaylight.controller.cluster.datastore.messages.FindLocalShard;
85 import org.opendaylight.controller.cluster.datastore.messages.FindPrimary;
86 import org.opendaylight.controller.cluster.datastore.messages.LocalPrimaryShardFound;
87 import org.opendaylight.controller.cluster.datastore.messages.LocalShardFound;
88 import org.opendaylight.controller.cluster.datastore.messages.LocalShardNotFound;
89 import org.opendaylight.controller.cluster.datastore.messages.PeerDown;
90 import org.opendaylight.controller.cluster.datastore.messages.PeerUp;
91 import org.opendaylight.controller.cluster.datastore.messages.PrimaryShardInfo;
92 import org.opendaylight.controller.cluster.datastore.messages.RemotePrimaryShardFound;
93 import org.opendaylight.controller.cluster.datastore.messages.RemoveShardReplica;
94 import org.opendaylight.controller.cluster.datastore.messages.ShardLeaderStateChanged;
95 import org.opendaylight.controller.cluster.datastore.messages.UpdateSchemaContext;
96 import org.opendaylight.controller.cluster.datastore.persisted.DatastoreSnapshot;
97 import org.opendaylight.controller.cluster.datastore.persisted.DatastoreSnapshot.ShardSnapshot;
98 import org.opendaylight.controller.cluster.datastore.persisted.ShardManagerSnapshot;
99 import org.opendaylight.controller.cluster.datastore.utils.ForwardingActor;
100 import org.opendaylight.controller.cluster.datastore.utils.MockClusterWrapper;
101 import org.opendaylight.controller.cluster.datastore.utils.MockConfiguration;
102 import org.opendaylight.controller.cluster.datastore.utils.PrimaryShardInfoFutureCache;
103 import org.opendaylight.controller.cluster.notifications.LeaderStateChanged;
104 import org.opendaylight.controller.cluster.notifications.RegisterRoleChangeListener;
105 import org.opendaylight.controller.cluster.notifications.RoleChangeNotification;
106 import org.opendaylight.controller.cluster.raft.RaftState;
107 import org.opendaylight.controller.cluster.raft.base.messages.FollowerInitialSyncUpStatus;
108 import org.opendaylight.controller.cluster.raft.base.messages.SwitchBehavior;
109 import org.opendaylight.controller.cluster.raft.client.messages.GetSnapshot;
110 import org.opendaylight.controller.cluster.raft.client.messages.Shutdown;
111 import org.opendaylight.controller.cluster.raft.messages.AddServer;
112 import org.opendaylight.controller.cluster.raft.messages.AddServerReply;
113 import org.opendaylight.controller.cluster.raft.messages.ChangeServersVotingStatus;
114 import org.opendaylight.controller.cluster.raft.messages.RemoveServer;
115 import org.opendaylight.controller.cluster.raft.messages.RemoveServerReply;
116 import org.opendaylight.controller.cluster.raft.messages.ServerChangeReply;
117 import org.opendaylight.controller.cluster.raft.messages.ServerChangeStatus;
118 import org.opendaylight.controller.cluster.raft.messages.ServerRemoved;
119 import org.opendaylight.controller.cluster.raft.policy.DisableElectionsRaftPolicy;
120 import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore;
121 import org.opendaylight.controller.cluster.raft.utils.MessageCollectorActor;
122 import org.opendaylight.controller.md.cluster.datastore.model.TestModel;
123 import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTree;
124 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
125 import org.slf4j.Logger;
126 import org.slf4j.LoggerFactory;
127 import scala.concurrent.Await;
128 import scala.concurrent.Future;
129 import scala.concurrent.duration.FiniteDuration;
131 public class ShardManagerTest extends AbstractShardManagerTest {
132 private static final Logger LOG = LoggerFactory.getLogger(ShardManagerTest.class);
133 private static final MemberName MEMBER_2 = MemberName.forName("member-2");
134 private static final MemberName MEMBER_3 = MemberName.forName("member-3");
136 private final String shardMgrID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
138 private ActorSystem newActorSystem(final String config) {
139 return newActorSystem("cluster-test", config);
142 private ActorRef newMockShardActor(final ActorSystem system, final String shardName, final String memberName) {
143 String name = ShardIdentifier.create(shardName, MemberName.forName(memberName), "config").toString();
144 if (system == getSystem()) {
145 return actorFactory.createActor(MessageCollectorActor.props(), name);
148 return system.actorOf(MessageCollectorActor.props(), name);
151 private Props newShardMgrProps() {
152 return newShardMgrProps(new MockConfiguration());
155 private static DatastoreContextFactory newDatastoreContextFactory(final DatastoreContext datastoreContext) {
156 DatastoreContextFactory mockFactory = mock(DatastoreContextFactory.class);
157 Mockito.doReturn(datastoreContext).when(mockFactory).getBaseDatastoreContext();
158 Mockito.doReturn(datastoreContext).when(mockFactory).getShardDatastoreContext(Mockito.anyString());
162 private TestShardManager.Builder newTestShardMgrBuilderWithMockShardActor() {
163 return newTestShardMgrBuilderWithMockShardActor(mockShardActor);
166 private TestShardManager.Builder newTestShardMgrBuilderWithMockShardActor(final ActorRef shardActor) {
167 return TestShardManager.builder(datastoreContextBuilder).shardActor(shardActor)
168 .distributedDataStore(mock(DistributedDataStore.class));
172 private Props newPropsShardMgrWithMockShardActor() {
173 return newTestShardMgrBuilderWithMockShardActor().props().withDispatcher(
174 Dispatchers.DefaultDispatcherId());
177 private Props newPropsShardMgrWithMockShardActor(final ActorRef shardActor) {
178 return newTestShardMgrBuilderWithMockShardActor(shardActor).props()
179 .withDispatcher(Dispatchers.DefaultDispatcherId());
183 private TestShardManager newTestShardManager() {
184 return newTestShardManager(newShardMgrProps());
187 private TestShardManager newTestShardManager(final Props props) {
188 TestActorRef<TestShardManager> shardManagerActor = actorFactory.createTestActor(props);
189 TestShardManager shardManager = shardManagerActor.underlyingActor();
190 shardManager.waitForRecoveryComplete();
194 private static void waitForShardInitialized(final ActorRef shardManager, final String shardName,
196 AssertionError last = null;
197 Stopwatch sw = Stopwatch.createStarted();
198 while (sw.elapsed(TimeUnit.SECONDS) <= 5) {
200 shardManager.tell(new FindLocalShard(shardName, true), kit.getRef());
201 kit.expectMsgClass(LocalShardFound.class);
203 } catch (AssertionError e) {
207 Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
213 @SuppressWarnings("unchecked")
214 private static <T> T expectMsgClassOrFailure(final Class<T> msgClass, final TestKit kit, final String msg) {
215 Object reply = kit.expectMsgAnyClassOf(kit.duration("5 sec"), msgClass, Failure.class);
216 if (reply instanceof Failure) {
217 throw new AssertionError(msg + " failed", ((Failure)reply).cause());
224 public void testPerShardDatastoreContext() throws Exception {
225 LOG.info("testPerShardDatastoreContext starting");
226 final DatastoreContextFactory mockFactory = newDatastoreContextFactory(
227 datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
230 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(6).build())
231 .when(mockFactory).getShardDatastoreContext("default");
234 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(7).build())
235 .when(mockFactory).getShardDatastoreContext("topology");
237 final MockConfiguration mockConfig = new MockConfiguration() {
239 public Collection<String> getMemberShardNames(final MemberName memberName) {
240 return Arrays.asList("default", "topology");
244 public Collection<MemberName> getMembersFromShardName(final String shardName) {
245 return members("member-1");
249 final ActorRef defaultShardActor = actorFactory.createActor(
250 MessageCollectorActor.props(), actorFactory.generateActorId("default"));
251 final ActorRef topologyShardActor = actorFactory.createActor(
252 MessageCollectorActor.props(), actorFactory.generateActorId("topology"));
254 final Map<String, Entry<ActorRef, DatastoreContext>> shardInfoMap = Collections.synchronizedMap(
255 new HashMap<String, Entry<ActorRef, DatastoreContext>>());
256 shardInfoMap.put("default", new AbstractMap.SimpleEntry<>(defaultShardActor, null));
257 shardInfoMap.put("topology", new AbstractMap.SimpleEntry<>(topologyShardActor, null));
259 final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
260 final CountDownLatch newShardActorLatch = new CountDownLatch(2);
261 class LocalShardManager extends ShardManager {
262 LocalShardManager(final AbstractShardManagerCreator<?> creator) {
267 protected ActorRef newShardActor(final ShardInformation info) {
268 Entry<ActorRef, DatastoreContext> entry = shardInfoMap.get(info.getShardName());
271 ref = entry.getKey();
272 entry.setValue(info.getDatastoreContext());
275 newShardActorLatch.countDown();
280 final Creator<ShardManager> creator = new Creator<ShardManager>() {
281 private static final long serialVersionUID = 1L;
283 public ShardManager create() {
284 return new LocalShardManager(
285 new GenericCreator<>(LocalShardManager.class).datastoreContextFactory(mockFactory)
286 .primaryShardInfoCache(primaryShardInfoCache).configuration(mockConfig));
290 TestKit kit = new TestKit(getSystem());
292 final ActorRef shardManager = actorFactory.createActor(Props.create(
293 new DelegatingShardManagerCreator(creator)).withDispatcher(Dispatchers.DefaultDispatcherId()));
295 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), kit.getRef());
297 assertEquals("Shard actors created", true, newShardActorLatch.await(5, TimeUnit.SECONDS));
298 assertEquals("getShardElectionTimeoutFactor", 6,
299 shardInfoMap.get("default").getValue().getShardElectionTimeoutFactor());
300 assertEquals("getShardElectionTimeoutFactor", 7,
301 shardInfoMap.get("topology").getValue().getShardElectionTimeoutFactor());
303 DatastoreContextFactory newMockFactory = newDatastoreContextFactory(
304 datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
306 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(66).build())
307 .when(newMockFactory).getShardDatastoreContext("default");
310 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(77).build())
311 .when(newMockFactory).getShardDatastoreContext("topology");
313 shardManager.tell(newMockFactory, kit.getRef());
315 DatastoreContext newContext = MessageCollectorActor.expectFirstMatching(defaultShardActor,
316 DatastoreContext.class);
317 assertEquals("getShardElectionTimeoutFactor", 66, newContext.getShardElectionTimeoutFactor());
319 newContext = MessageCollectorActor.expectFirstMatching(topologyShardActor, DatastoreContext.class);
320 assertEquals("getShardElectionTimeoutFactor", 77, newContext.getShardElectionTimeoutFactor());
322 LOG.info("testPerShardDatastoreContext ending");
326 public void testOnReceiveFindPrimaryForNonExistentShard() {
327 new TestKit(getSystem()) {
329 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
331 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
333 shardManager.tell(new FindPrimary("non-existent", false), getRef());
335 expectMsgClass(duration("5 seconds"), PrimaryNotFoundException.class);
341 public void testOnReceiveFindPrimaryForLocalLeaderShard() {
342 LOG.info("testOnReceiveFindPrimaryForLocalLeaderShard starting");
343 new TestKit(getSystem()) {
345 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
347 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
349 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
350 shardManager.tell(new ActorInitialized(), mockShardActor);
352 DataTree mockDataTree = mock(DataTree.class);
353 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
354 DataStoreVersions.CURRENT_VERSION), getRef());
356 MessageCollectorActor.expectFirstMatching(mockShardActor, RegisterRoleChangeListener.class);
358 new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
361 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
363 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
364 LocalPrimaryShardFound.class);
365 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
366 primaryFound.getPrimaryPath().contains("member-1-shard-default"));
367 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
371 LOG.info("testOnReceiveFindPrimaryForLocalLeaderShard ending");
375 public void testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp() {
376 LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp starting");
377 new TestKit(getSystem()) {
379 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
381 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
382 shardManager.tell(new ActorInitialized(), mockShardActor);
384 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
385 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
387 new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
389 shardManager.tell(new LeaderStateChanged(memberId1, memberId2, DataStoreVersions.CURRENT_VERSION),
392 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
394 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
398 LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp ending");
402 public void testOnReceiveFindPrimaryForNonLocalLeaderShard() {
403 LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShard starting");
404 new TestKit(getSystem()) {
406 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
408 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
409 shardManager.tell(new ActorInitialized(), mockShardActor);
411 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
412 MockClusterWrapper.sendMemberUp(shardManager, "member-2", getRef().path().toString());
414 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
416 new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
418 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
419 shardManager.tell(new ShardLeaderStateChanged(memberId1, memberId2, leaderVersion), mockShardActor);
421 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
423 RemotePrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
424 RemotePrimaryShardFound.class);
425 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
426 primaryFound.getPrimaryPath().contains("member-2-shard-default"));
427 assertEquals("getPrimaryVersion", leaderVersion, primaryFound.getPrimaryVersion());
431 LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShard ending");
435 public void testOnReceiveFindPrimaryForUninitializedShard() {
436 new TestKit(getSystem()) {
438 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
440 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
442 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
448 public void testOnReceiveFindPrimaryForInitializedShardWithNoRole() {
449 new TestKit(getSystem()) {
451 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
453 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
454 shardManager.tell(new ActorInitialized(), mockShardActor);
456 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
458 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
464 public void testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId() {
465 LOG.info("testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId starting");
466 new TestKit(getSystem()) {
468 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
470 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
471 shardManager.tell(new ActorInitialized(), mockShardActor);
473 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
475 new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Follower.name()),
478 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
480 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
482 DataTree mockDataTree = mock(DataTree.class);
483 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
484 DataStoreVersions.CURRENT_VERSION), mockShardActor);
486 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
488 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
489 LocalPrimaryShardFound.class);
490 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
491 primaryFound.getPrimaryPath().contains("member-1-shard-default"));
492 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
496 LOG.info("testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId starting");
500 public void testOnReceiveFindPrimaryWaitForShardLeader() {
501 LOG.info("testOnReceiveFindPrimaryWaitForShardLeader starting");
502 datastoreContextBuilder.shardInitializationTimeout(10, TimeUnit.SECONDS);
503 new TestKit(getSystem()) {
505 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
507 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
509 // We're passing waitUntilInitialized = true to FindPrimary so
510 // the response should be
511 // delayed until we send ActorInitialized and
512 // RoleChangeNotification.
513 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
515 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
517 shardManager.tell(new ActorInitialized(), mockShardActor);
519 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
521 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
523 new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
526 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
528 DataTree mockDataTree = mock(DataTree.class);
529 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
530 DataStoreVersions.CURRENT_VERSION), mockShardActor);
532 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
533 LocalPrimaryShardFound.class);
534 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
535 primaryFound.getPrimaryPath().contains("member-1-shard-default"));
536 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
538 expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
542 LOG.info("testOnReceiveFindPrimaryWaitForShardLeader ending");
546 public void testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard() {
547 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard starting");
548 new TestKit(getSystem()) {
550 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
552 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
554 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
556 expectMsgClass(duration("2 seconds"), NotInitializedException.class);
558 shardManager.tell(new ActorInitialized(), mockShardActor);
560 expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
564 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard ending");
568 public void testOnReceiveFindPrimaryWaitForReadyWithCandidateShard() {
569 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithCandidateShard starting");
570 new TestKit(getSystem()) {
572 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
574 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
575 shardManager.tell(new ActorInitialized(), mockShardActor);
576 shardManager.tell(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix, null,
577 RaftState.Candidate.name()), mockShardActor);
579 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
581 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
585 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithCandidateShard ending");
589 public void testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard() {
590 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard starting");
591 new TestKit(getSystem()) {
593 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
595 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
596 shardManager.tell(new ActorInitialized(), mockShardActor);
597 shardManager.tell(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix, null,
598 RaftState.IsolatedLeader.name()), mockShardActor);
600 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
602 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
606 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard ending");
610 public void testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard() {
611 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard starting");
612 new TestKit(getSystem()) {
614 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
616 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
617 shardManager.tell(new ActorInitialized(), mockShardActor);
619 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
621 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
625 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard ending");
629 public void testOnReceiveFindPrimaryForRemoteShard() {
630 LOG.info("testOnReceiveFindPrimaryForRemoteShard starting");
631 String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
633 // Create an ActorSystem ShardManager actor for member-1.
635 final ActorSystem system1 = newActorSystem("Member1");
636 Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
638 final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
639 newTestShardMgrBuilderWithMockShardActor().cluster(
640 new ClusterWrapperImpl(system1)).props().withDispatcher(
641 Dispatchers.DefaultDispatcherId()), shardManagerID);
643 // Create an ActorSystem ShardManager actor for member-2.
645 final ActorSystem system2 = newActorSystem("Member2");
647 Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
649 final ActorRef mockShardActor2 = newMockShardActor(system2, "astronauts", "member-2");
651 MockConfiguration mockConfig2 = new MockConfiguration(
652 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
653 .put("astronauts", Arrays.asList("member-2")).build());
655 final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
656 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
657 new ClusterWrapperImpl(system2)).props().withDispatcher(
658 Dispatchers.DefaultDispatcherId()), shardManagerID);
660 new TestKit(system1) {
662 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
663 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
665 shardManager2.tell(new ActorInitialized(), mockShardActor2);
667 String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
668 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
670 new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
673 new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
676 shardManager1.underlyingActor().waitForMemberUp();
677 shardManager1.tell(new FindPrimary("astronauts", false), getRef());
679 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
680 String path = found.getPrimaryPath();
681 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-astronauts-config"));
682 assertEquals("getPrimaryVersion", leaderVersion, found.getPrimaryVersion());
684 shardManager2.underlyingActor().verifyFindPrimary();
686 // This part times out quite a bit on jenkins for some reason
688 // Cluster.get(system2).down(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
690 // shardManager1.underlyingActor().waitForMemberRemoved();
692 // shardManager1.tell(new FindPrimary("astronauts", false), getRef());
694 // expectMsgClass(duration("5 seconds"), PrimaryNotFoundException.class);
698 LOG.info("testOnReceiveFindPrimaryForRemoteShard ending");
702 public void testShardAvailabilityOnChangeOfMemberReachability() {
703 LOG.info("testShardAvailabilityOnChangeOfMemberReachability starting");
704 String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
706 // Create an ActorSystem ShardManager actor for member-1.
708 final ActorSystem system1 = newActorSystem("Member1");
709 Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
711 final ActorRef mockShardActor1 = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
713 final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
714 newTestShardMgrBuilder().shardActor(mockShardActor1).cluster(
715 new ClusterWrapperImpl(system1)).props().withDispatcher(
716 Dispatchers.DefaultDispatcherId()), shardManagerID);
718 // Create an ActorSystem ShardManager actor for member-2.
720 final ActorSystem system2 = newActorSystem("Member2");
722 Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
724 final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
726 MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
727 .put("default", Arrays.asList("member-1", "member-2")).build());
729 final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
730 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
731 new ClusterWrapperImpl(system2)).props().withDispatcher(
732 Dispatchers.DefaultDispatcherId()), shardManagerID);
734 new TestKit(system1) {
736 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
737 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
738 shardManager1.tell(new ActorInitialized(), mockShardActor1);
739 shardManager2.tell(new ActorInitialized(), mockShardActor2);
741 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
742 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
743 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class),
744 DataStoreVersions.CURRENT_VERSION), mockShardActor1);
746 new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
748 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class),
749 DataStoreVersions.CURRENT_VERSION), mockShardActor2);
751 new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
753 shardManager1.underlyingActor().waitForMemberUp();
755 shardManager1.tell(new FindPrimary("default", true), getRef());
757 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
758 String path = found.getPrimaryPath();
759 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
761 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
762 "akka://cluster-test@127.0.0.1:2558"), getRef());
764 shardManager1.underlyingActor().waitForUnreachableMember();
766 PeerDown peerDown = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
767 assertEquals("getMemberName", MEMBER_2, peerDown.getMemberName());
768 MessageCollectorActor.clearMessages(mockShardActor1);
771 MockClusterWrapper.createMemberRemoved("member-2", "akka://cluster-test@127.0.0.1:2558"),
774 MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
776 shardManager1.tell(new FindPrimary("default", true), getRef());
778 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
781 MockClusterWrapper.createReachableMember("member-2", "akka://cluster-test@127.0.0.1:2558"),
784 shardManager1.underlyingActor().waitForReachableMember();
786 PeerUp peerUp = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
787 assertEquals("getMemberName", MEMBER_2, peerUp.getMemberName());
788 MessageCollectorActor.clearMessages(mockShardActor1);
790 shardManager1.tell(new FindPrimary("default", true), getRef());
792 RemotePrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
793 String path1 = found1.getPrimaryPath();
794 assertTrue("Unexpected primary path " + path1, path1.contains("member-2-shard-default-config"));
797 MockClusterWrapper.createMemberUp("member-2", "akka://cluster-test@127.0.0.1:2558"),
800 MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
802 // Test FindPrimary wait succeeds after reachable member event.
804 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
805 "akka://cluster-test@127.0.0.1:2558"), getRef());
806 shardManager1.underlyingActor().waitForUnreachableMember();
808 shardManager1.tell(new FindPrimary("default", true), getRef());
811 MockClusterWrapper.createReachableMember("member-2", "akka://cluster-test@127.0.0.1:2558"),
814 RemotePrimaryShardFound found2 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
815 String path2 = found2.getPrimaryPath();
816 assertTrue("Unexpected primary path " + path2, path2.contains("member-2-shard-default-config"));
820 LOG.info("testShardAvailabilityOnChangeOfMemberReachability ending");
824 public void testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange() {
825 LOG.info("testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange starting");
826 String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
828 // Create an ActorSystem ShardManager actor for member-1.
830 final ActorSystem system1 = newActorSystem("Member1");
831 Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
833 final ActorRef mockShardActor1 = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
835 final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
836 final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
837 newTestShardMgrBuilder().shardActor(mockShardActor1).cluster(new ClusterWrapperImpl(system1))
838 .primaryShardInfoCache(primaryShardInfoCache).props()
839 .withDispatcher(Dispatchers.DefaultDispatcherId()),
842 // Create an ActorSystem ShardManager actor for member-2.
844 final ActorSystem system2 = newActorSystem("Member2");
846 Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
848 final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
850 MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
851 .put("default", Arrays.asList("member-1", "member-2")).build());
853 final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
854 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
855 new ClusterWrapperImpl(system2)).props().withDispatcher(
856 Dispatchers.DefaultDispatcherId()), shardManagerID);
858 new TestKit(system1) {
860 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
861 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
862 shardManager1.tell(new ActorInitialized(), mockShardActor1);
863 shardManager2.tell(new ActorInitialized(), mockShardActor2);
865 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
866 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
867 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class),
868 DataStoreVersions.CURRENT_VERSION), mockShardActor1);
870 new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
872 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class),
873 DataStoreVersions.CURRENT_VERSION), mockShardActor2);
875 new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
877 shardManager1.underlyingActor().waitForMemberUp();
879 shardManager1.tell(new FindPrimary("default", true), getRef());
881 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
882 String path = found.getPrimaryPath();
883 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
885 primaryShardInfoCache.putSuccessful("default", new PrimaryShardInfo(
886 system1.actorSelection(mockShardActor1.path()), DataStoreVersions.CURRENT_VERSION));
888 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
889 "akka://cluster-test@127.0.0.1:2558"), getRef());
891 shardManager1.underlyingActor().waitForUnreachableMember();
893 shardManager1.tell(new FindPrimary("default", true), getRef());
895 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
897 assertNull("Expected primaryShardInfoCache entry removed",
898 primaryShardInfoCache.getIfPresent("default"));
900 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId1, mock(DataTree.class),
901 DataStoreVersions.CURRENT_VERSION), mockShardActor1);
903 new RoleChangeNotification(memberId1, RaftState.Follower.name(), RaftState.Leader.name()),
906 shardManager1.tell(new FindPrimary("default", true), getRef());
908 LocalPrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
909 String path1 = found1.getPrimaryPath();
910 assertTrue("Unexpected primary path " + path1, path1.contains("member-1-shard-default-config"));
915 LOG.info("testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange ending");
919 public void testShardAvailabilityChangeOnMemberWithNameContainedInLeaderIdUnreachable() {
920 LOG.info("testShardAvailabilityChangeOnMemberWithNameContainedInLeaderIdUnreachable starting");
921 String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
923 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
924 .put("default", Arrays.asList("member-256", "member-2")).build());
926 // Create an ActorSystem, ShardManager and actor for member-256.
928 final ActorSystem system256 = newActorSystem("Member256");
929 // 2562 is the tcp port of Member256 in src/test/resources/application.conf.
930 Cluster.get(system256).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2562"));
932 final ActorRef mockShardActor256 = newMockShardActor(system256, Shard.DEFAULT_NAME, "member-256");
934 final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
936 // ShardManager must be created with shard configuration to let its localShards has shards.
937 final TestActorRef<TestShardManager> shardManager256 = TestActorRef.create(system256,
938 newTestShardMgrBuilder(mockConfig).shardActor(mockShardActor256)
939 .cluster(new ClusterWrapperImpl(system256))
940 .primaryShardInfoCache(primaryShardInfoCache).props()
941 .withDispatcher(Dispatchers.DefaultDispatcherId()),
944 // Create an ActorSystem, ShardManager and actor for member-2 whose name is contained in member-256.
946 final ActorSystem system2 = newActorSystem("Member2");
948 // Join member-2 into the cluster of member-256.
949 Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2562"));
951 final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
953 final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
954 newTestShardMgrBuilder(mockConfig).shardActor(mockShardActor2).cluster(
955 new ClusterWrapperImpl(system2)).props().withDispatcher(
956 Dispatchers.DefaultDispatcherId()), shardManagerID);
958 new TestKit(system256) {
960 shardManager256.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
961 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
962 shardManager256.tell(new ActorInitialized(), mockShardActor256);
963 shardManager2.tell(new ActorInitialized(), mockShardActor2);
965 String memberId256 = "member-256-shard-default-" + shardMrgIDSuffix;
966 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
967 shardManager256.tell(new ShardLeaderStateChanged(memberId256, memberId256, mock(DataTree.class),
968 DataStoreVersions.CURRENT_VERSION), mockShardActor256);
969 shardManager256.tell(
970 new RoleChangeNotification(memberId256, RaftState.Candidate.name(), RaftState.Leader.name()),
972 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId256, mock(DataTree.class),
973 DataStoreVersions.CURRENT_VERSION), mockShardActor2);
975 new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Follower.name()),
977 shardManager256.underlyingActor().waitForMemberUp();
979 shardManager256.tell(new FindPrimary("default", true), getRef());
981 LocalPrimaryShardFound found = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
982 String path = found.getPrimaryPath();
983 assertTrue("Unexpected primary path " + path + " which must on member-256",
984 path.contains("member-256-shard-default-config"));
986 PrimaryShardInfo primaryShardInfo = new PrimaryShardInfo(
987 system256.actorSelection(mockShardActor256.path()), DataStoreVersions.CURRENT_VERSION);
988 primaryShardInfoCache.putSuccessful("default", primaryShardInfo);
990 // Simulate member-2 become unreachable.
991 shardManager256.tell(MockClusterWrapper.createUnreachableMember("member-2",
992 "akka://cluster-test@127.0.0.1:2558"), getRef());
993 shardManager256.underlyingActor().waitForUnreachableMember();
995 // Make sure leader shard on member-256 is still leader and still in the cache.
996 shardManager256.tell(new FindPrimary("default", true), getRef());
997 found = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
998 path = found.getPrimaryPath();
999 assertTrue("Unexpected primary path " + path + " which must still not on member-256",
1000 path.contains("member-256-shard-default-config"));
1001 Future<PrimaryShardInfo> futurePrimaryShard = primaryShardInfoCache.getIfPresent("default");
1002 futurePrimaryShard.onComplete(new OnComplete<PrimaryShardInfo>() {
1004 public void onComplete(final Throwable failure, final PrimaryShardInfo futurePrimaryShardInfo) {
1005 if (failure != null) {
1006 assertTrue("Primary shard info is unexpectedly removed from primaryShardInfoCache", false);
1008 assertEquals("Expected primaryShardInfoCache entry",
1009 primaryShardInfo, futurePrimaryShardInfo);
1012 }, system256.dispatchers().defaultGlobalDispatcher());
1016 LOG.info("testShardAvailabilityChangeOnMemberWithNameContainedInLeaderIdUnreachable ending");
1020 public void testOnReceiveFindLocalShardForNonExistentShard() {
1021 new TestKit(getSystem()) {
1023 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1025 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1027 shardManager.tell(new FindLocalShard("non-existent", false), getRef());
1029 LocalShardNotFound notFound = expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
1031 assertEquals("getShardName", "non-existent", notFound.getShardName());
1037 public void testOnReceiveFindLocalShardForExistentShard() {
1038 new TestKit(getSystem()) {
1040 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1042 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1043 shardManager.tell(new ActorInitialized(), mockShardActor);
1045 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1047 LocalShardFound found = expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1049 assertTrue("Found path contains " + found.getPath().path().toString(),
1050 found.getPath().path().toString().contains("member-1-shard-default-config"));
1056 public void testOnReceiveFindLocalShardForNotInitializedShard() {
1057 new TestKit(getSystem()) {
1059 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1061 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1063 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1069 public void testOnReceiveFindLocalShardWaitForShardInitialized() throws Exception {
1070 LOG.info("testOnReceiveFindLocalShardWaitForShardInitialized starting");
1071 new TestKit(getSystem()) {
1073 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1075 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1077 // We're passing waitUntilInitialized = true to FindLocalShard
1078 // so the response should be
1079 // delayed until we send ActorInitialized.
1080 Future<Object> future = Patterns.ask(shardManager, new FindLocalShard(Shard.DEFAULT_NAME, true),
1081 new Timeout(5, TimeUnit.SECONDS));
1083 shardManager.tell(new ActorInitialized(), mockShardActor);
1085 Object resp = Await.result(future, duration("5 seconds"));
1086 assertTrue("Expected: LocalShardFound, Actual: " + resp, resp instanceof LocalShardFound);
1090 LOG.info("testOnReceiveFindLocalShardWaitForShardInitialized starting");
1094 public void testRoleChangeNotificationAndShardLeaderStateChangedReleaseReady() throws Exception {
1095 TestShardManager shardManager = newTestShardManager();
1097 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1098 shardManager.onReceiveCommand(new RoleChangeNotification(
1099 memberId, RaftState.Candidate.name(), RaftState.Leader.name()));
1101 verify(ready, never()).countDown();
1103 shardManager.onReceiveCommand(new ShardLeaderStateChanged(memberId, memberId,
1104 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1106 verify(ready, times(1)).countDown();
1110 public void testRoleChangeNotificationToFollowerWithShardLeaderStateChangedReleaseReady() throws Exception {
1111 new TestKit(getSystem()) {
1113 TestShardManager shardManager = newTestShardManager();
1115 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1116 shardManager.onReceiveCommand(new RoleChangeNotification(memberId, null, RaftState.Follower.name()));
1118 verify(ready, never()).countDown();
1121 .onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
1123 shardManager.onReceiveCommand(
1124 new ShardLeaderStateChanged(memberId, "member-2-shard-default-" + shardMrgIDSuffix,
1125 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1127 verify(ready, times(1)).countDown();
1133 public void testReadyCountDownForMemberUpAfterLeaderStateChanged() throws Exception {
1134 new TestKit(getSystem()) {
1136 TestShardManager shardManager = newTestShardManager();
1138 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1139 shardManager.onReceiveCommand(new RoleChangeNotification(memberId, null, RaftState.Follower.name()));
1141 verify(ready, never()).countDown();
1143 shardManager.onReceiveCommand(
1144 new ShardLeaderStateChanged(memberId, "member-2-shard-default-" + shardMrgIDSuffix,
1145 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1148 .onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
1150 verify(ready, times(1)).countDown();
1156 public void testRoleChangeNotificationDoNothingForUnknownShard() throws Exception {
1157 TestShardManager shardManager = newTestShardManager();
1159 shardManager.onReceiveCommand(new RoleChangeNotification(
1160 "unknown", RaftState.Candidate.name(), RaftState.Leader.name()));
1162 verify(ready, never()).countDown();
1166 public void testByDefaultSyncStatusIsFalse() {
1167 TestShardManager shardManager = newTestShardManager();
1169 assertEquals(false, shardManager.getMBean().getSyncStatus());
1173 public void testWhenShardIsLeaderSyncStatusIsTrue() throws Exception {
1174 TestShardManager shardManager = newTestShardManager();
1176 shardManager.onReceiveCommand(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix,
1177 RaftState.Follower.name(), RaftState.Leader.name()));
1179 assertEquals(true, shardManager.getMBean().getSyncStatus());
1183 public void testWhenShardIsCandidateSyncStatusIsFalse() throws Exception {
1184 TestShardManager shardManager = newTestShardManager();
1186 String shardId = "member-1-shard-default-" + shardMrgIDSuffix;
1187 shardManager.onReceiveCommand(new RoleChangeNotification(shardId,
1188 RaftState.Follower.name(), RaftState.Candidate.name()));
1190 assertEquals(false, shardManager.getMBean().getSyncStatus());
1192 // Send a FollowerInitialSyncStatus with status = true for the replica whose current state is candidate
1193 shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(
1196 assertEquals(false, shardManager.getMBean().getSyncStatus());
1200 public void testWhenShardIsFollowerSyncStatusDependsOnFollowerInitialSyncStatus() throws Exception {
1201 TestShardManager shardManager = newTestShardManager();
1203 String shardId = "member-1-shard-default-" + shardMrgIDSuffix;
1204 shardManager.onReceiveCommand(new RoleChangeNotification(shardId,
1205 RaftState.Candidate.name(), RaftState.Follower.name()));
1207 // Initially will be false
1208 assertEquals(false, shardManager.getMBean().getSyncStatus());
1210 // Send status true will make sync status true
1211 shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(true, shardId));
1213 assertEquals(true, shardManager.getMBean().getSyncStatus());
1215 // Send status false will make sync status false
1216 shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(false, shardId));
1218 assertEquals(false, shardManager.getMBean().getSyncStatus());
1223 public void testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards() throws Exception {
1224 LOG.info("testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards starting");
1225 TestShardManager shardManager = newTestShardManager(newShardMgrProps(new MockConfiguration() {
1227 public List<String> getMemberShardNames(final MemberName memberName) {
1228 return Arrays.asList("default", "astronauts");
1232 // Initially will be false
1233 assertEquals(false, shardManager.getMBean().getSyncStatus());
1235 // Make default shard leader
1236 String defaultShardId = "member-1-shard-default-" + shardMrgIDSuffix;
1237 shardManager.onReceiveCommand(new RoleChangeNotification(defaultShardId,
1238 RaftState.Follower.name(), RaftState.Leader.name()));
1240 // default = Leader, astronauts is unknown so sync status remains false
1241 assertEquals(false, shardManager.getMBean().getSyncStatus());
1243 // Make astronauts shard leader as well
1244 String astronautsShardId = "member-1-shard-astronauts-" + shardMrgIDSuffix;
1245 shardManager.onReceiveCommand(new RoleChangeNotification(astronautsShardId,
1246 RaftState.Follower.name(), RaftState.Leader.name()));
1248 // Now sync status should be true
1249 assertEquals(true, shardManager.getMBean().getSyncStatus());
1251 // Make astronauts a Follower
1252 shardManager.onReceiveCommand(new RoleChangeNotification(astronautsShardId,
1253 RaftState.Leader.name(), RaftState.Follower.name()));
1255 // Sync status is not true
1256 assertEquals(false, shardManager.getMBean().getSyncStatus());
1258 // Make the astronauts follower sync status true
1259 shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(true, astronautsShardId));
1261 // Sync status is now true
1262 assertEquals(true, shardManager.getMBean().getSyncStatus());
1264 LOG.info("testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards ending");
1268 public void testOnReceiveSwitchShardBehavior() {
1269 new TestKit(getSystem()) {
1271 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1273 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1274 shardManager.tell(new ActorInitialized(), mockShardActor);
1276 shardManager.tell(new SwitchShardBehavior(mockShardName, RaftState.Leader, 1000), getRef());
1278 SwitchBehavior switchBehavior = MessageCollectorActor.expectFirstMatching(mockShardActor,
1279 SwitchBehavior.class);
1281 assertEquals(RaftState.Leader, switchBehavior.getNewState());
1282 assertEquals(1000, switchBehavior.getNewTerm());
1287 private static List<MemberName> members(final String... names) {
1288 return Arrays.asList(names).stream().map(MemberName::forName).collect(Collectors.toList());
1292 public void testOnCreateShard() {
1293 LOG.info("testOnCreateShard starting");
1294 new TestKit(getSystem()) {
1296 datastoreContextBuilder.shardInitializationTimeout(1, TimeUnit.MINUTES).persistent(true);
1298 ActorRef shardManager = actorFactory
1299 .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1300 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1302 SchemaContext schemaContext = TestModel.createTestContext();
1303 shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
1305 DatastoreContext datastoreContext = DatastoreContext.newBuilder().shardElectionTimeoutFactor(100)
1306 .persistent(false).build();
1307 Shard.Builder shardBuilder = Shard.builder();
1309 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1310 "foo", null, members("member-1", "member-5", "member-6"));
1311 shardManager.tell(new CreateShard(config, shardBuilder, datastoreContext), getRef());
1313 expectMsgClass(duration("5 seconds"), Success.class);
1315 shardManager.tell(new FindLocalShard("foo", true), getRef());
1317 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1319 assertEquals("isRecoveryApplicable", false, shardBuilder.getDatastoreContext().isPersistent());
1320 assertTrue("Epxected ShardPeerAddressResolver", shardBuilder.getDatastoreContext().getShardRaftConfig()
1321 .getPeerAddressResolver() instanceof ShardPeerAddressResolver);
1322 assertEquals("peerMembers", Sets.newHashSet(
1323 ShardIdentifier.create("foo", MemberName.forName("member-5"), shardMrgIDSuffix).toString(),
1324 ShardIdentifier.create("foo", MemberName.forName("member-6"), shardMrgIDSuffix).toString()),
1325 shardBuilder.getPeerAddresses().keySet());
1326 assertEquals("ShardIdentifier", ShardIdentifier.create("foo", MEMBER_1, shardMrgIDSuffix),
1327 shardBuilder.getId());
1328 assertSame("schemaContext", schemaContext, shardBuilder.getSchemaContext());
1330 // Send CreateShard with same name - should return Success with
1333 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1335 Success success = expectMsgClass(duration("5 seconds"), Success.class);
1336 assertNotNull("Success status is null", success.status());
1340 LOG.info("testOnCreateShard ending");
1344 public void testOnCreateShardWithLocalMemberNotInShardConfig() {
1345 LOG.info("testOnCreateShardWithLocalMemberNotInShardConfig starting");
1346 new TestKit(getSystem()) {
1348 datastoreContextBuilder.shardInitializationTimeout(1, TimeUnit.MINUTES).persistent(true);
1350 ActorRef shardManager = actorFactory
1351 .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1352 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1354 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1356 Shard.Builder shardBuilder = Shard.builder();
1357 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1358 "foo", null, members("member-5", "member-6"));
1360 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1361 expectMsgClass(duration("5 seconds"), Success.class);
1363 shardManager.tell(new FindLocalShard("foo", true), getRef());
1364 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1366 assertEquals("peerMembers size", 0, shardBuilder.getPeerAddresses().size());
1367 assertEquals("schemaContext", DisableElectionsRaftPolicy.class.getName(), shardBuilder
1368 .getDatastoreContext().getShardRaftConfig().getCustomRaftPolicyImplementationClass());
1372 LOG.info("testOnCreateShardWithLocalMemberNotInShardConfig ending");
1376 public void testOnCreateShardWithNoInitialSchemaContext() {
1377 LOG.info("testOnCreateShardWithNoInitialSchemaContext starting");
1378 new TestKit(getSystem()) {
1380 ActorRef shardManager = actorFactory
1381 .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1382 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1384 Shard.Builder shardBuilder = Shard.builder();
1386 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1387 "foo", null, members("member-1"));
1388 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1390 expectMsgClass(duration("5 seconds"), Success.class);
1392 SchemaContext schemaContext = TestModel.createTestContext();
1393 shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
1395 shardManager.tell(new FindLocalShard("foo", true), getRef());
1397 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1399 assertSame("schemaContext", schemaContext, shardBuilder.getSchemaContext());
1400 assertNotNull("schemaContext is null", shardBuilder.getDatastoreContext());
1404 LOG.info("testOnCreateShardWithNoInitialSchemaContext ending");
1408 public void testGetSnapshot() {
1409 LOG.info("testGetSnapshot starting");
1410 TestKit kit = new TestKit(getSystem());
1412 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1413 .put("shard1", Arrays.asList("member-1")).put("shard2", Arrays.asList("member-1"))
1414 .put("astronauts", Collections.<String>emptyList()).build());
1416 TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(newShardMgrProps(mockConfig)
1417 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1419 shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1420 Failure failure = kit.expectMsgClass(Failure.class);
1421 assertEquals("Failure cause type", IllegalStateException.class, failure.cause().getClass());
1423 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1425 waitForShardInitialized(shardManager, "shard1", kit);
1426 waitForShardInitialized(shardManager, "shard2", kit);
1428 shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1430 DatastoreSnapshot datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1432 assertEquals("getType", shardMrgIDSuffix, datastoreSnapshot.getType());
1433 assertNull("Expected null ShardManagerSnapshot", datastoreSnapshot.getShardManagerSnapshot());
1435 Function<ShardSnapshot, String> shardNameTransformer = ShardSnapshot::getName;
1437 assertEquals("Shard names", Sets.newHashSet("shard1", "shard2"), Sets.newHashSet(
1438 Lists.transform(datastoreSnapshot.getShardSnapshots(), shardNameTransformer)));
1440 // Add a new replica
1442 TestKit mockShardLeaderKit = new TestKit(getSystem());
1444 TestShardManager shardManagerInstance = shardManager.underlyingActor();
1445 shardManagerInstance.setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1447 shardManager.tell(new AddShardReplica("astronauts"), kit.getRef());
1448 mockShardLeaderKit.expectMsgClass(AddServer.class);
1449 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.OK, ""));
1450 kit.expectMsgClass(Status.Success.class);
1451 waitForShardInitialized(shardManager, "astronauts", kit);
1453 // Send another GetSnapshot and verify
1455 shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1456 datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1458 assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"), Sets.newHashSet(
1459 Lists.transform(datastoreSnapshot.getShardSnapshots(), shardNameTransformer)));
1461 ShardManagerSnapshot snapshot = datastoreSnapshot.getShardManagerSnapshot();
1462 assertNotNull("Expected ShardManagerSnapshot", snapshot);
1463 assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"),
1464 Sets.newHashSet(snapshot.getShardList()));
1466 LOG.info("testGetSnapshot ending");
1470 public void testRestoreFromSnapshot() {
1471 LOG.info("testRestoreFromSnapshot starting");
1473 datastoreContextBuilder.shardInitializationTimeout(3, TimeUnit.SECONDS);
1475 TestKit kit = new TestKit(getSystem());
1477 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1478 .put("shard1", Collections.<String>emptyList()).put("shard2", Collections.<String>emptyList())
1479 .put("astronauts", Collections.<String>emptyList()).build());
1481 ShardManagerSnapshot snapshot =
1482 new ShardManagerSnapshot(Arrays.asList("shard1", "shard2", "astronauts"), Collections.emptyMap());
1483 DatastoreSnapshot restoreFromSnapshot = new DatastoreSnapshot(shardMrgIDSuffix, snapshot,
1484 Collections.<ShardSnapshot>emptyList());
1485 TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(newTestShardMgrBuilder(mockConfig)
1486 .restoreFromSnapshot(restoreFromSnapshot).props().withDispatcher(Dispatchers.DefaultDispatcherId()));
1488 shardManager.underlyingActor().waitForRecoveryComplete();
1490 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1492 waitForShardInitialized(shardManager, "shard1", kit);
1493 waitForShardInitialized(shardManager, "shard2", kit);
1494 waitForShardInitialized(shardManager, "astronauts", kit);
1496 shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1498 DatastoreSnapshot datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1500 assertEquals("getType", shardMrgIDSuffix, datastoreSnapshot.getType());
1502 assertNotNull("Expected ShardManagerSnapshot", datastoreSnapshot.getShardManagerSnapshot());
1503 assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"),
1504 Sets.newHashSet(datastoreSnapshot.getShardManagerSnapshot().getShardList()));
1506 LOG.info("testRestoreFromSnapshot ending");
1510 public void testAddShardReplicaForNonExistentShardConfig() {
1511 new TestKit(getSystem()) {
1513 ActorRef shardManager = actorFactory
1514 .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1515 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1517 shardManager.tell(new AddShardReplica("model-inventory"), getRef());
1518 Status.Failure resp = expectMsgClass(duration("2 seconds"), Status.Failure.class);
1520 assertEquals("Failure obtained", true, resp.cause() instanceof IllegalArgumentException);
1526 public void testAddShardReplica() {
1527 LOG.info("testAddShardReplica starting");
1528 MockConfiguration mockConfig = new MockConfiguration(
1529 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
1530 .put("astronauts", Arrays.asList("member-2")).build());
1532 final String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
1533 datastoreContextBuilder.shardManagerPersistenceId(shardManagerID);
1535 // Create an ActorSystem ShardManager actor for member-1.
1536 final ActorSystem system1 = newActorSystem("Member1");
1537 Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1538 ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
1539 final TestActorRef<TestShardManager> newReplicaShardManager = TestActorRef.create(system1,
1540 newTestShardMgrBuilder(mockConfig).shardActor(mockDefaultShardActor)
1541 .cluster(new ClusterWrapperImpl(system1)).props()
1542 .withDispatcher(Dispatchers.DefaultDispatcherId()),
1545 // Create an ActorSystem ShardManager actor for member-2.
1546 final ActorSystem system2 = newActorSystem("Member2");
1547 Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1549 String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
1550 String name = ShardIdentifier.create("astronauts", MEMBER_2, "config").toString();
1551 final TestActorRef<MockRespondActor> mockShardLeaderActor = TestActorRef.create(system2,
1552 Props.create(MockRespondActor.class, AddServer.class,
1553 new AddServerReply(ServerChangeStatus.OK, memberId2))
1554 .withDispatcher(Dispatchers.DefaultDispatcherId()),
1556 final TestActorRef<TestShardManager> leaderShardManager = TestActorRef.create(system2,
1557 newTestShardMgrBuilder(mockConfig).shardActor(mockShardLeaderActor)
1558 .cluster(new ClusterWrapperImpl(system2)).props()
1559 .withDispatcher(Dispatchers.DefaultDispatcherId()),
1562 new TestKit(system1) {
1564 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1565 leaderShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1567 leaderShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1569 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
1570 leaderShardManager.tell(
1571 new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
1572 mockShardLeaderActor);
1573 leaderShardManager.tell(
1574 new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
1575 mockShardLeaderActor);
1577 newReplicaShardManager.underlyingActor().waitForMemberUp();
1578 leaderShardManager.underlyingActor().waitForMemberUp();
1580 // Have a dummy snapshot to be overwritten by the new data
1582 String[] restoredShards = { "default", "people" };
1583 ShardManagerSnapshot snapshot =
1584 new ShardManagerSnapshot(Arrays.asList(restoredShards), Collections.emptyMap());
1585 InMemorySnapshotStore.addSnapshot(shardManagerID, snapshot);
1586 Uninterruptibles.sleepUninterruptibly(2, TimeUnit.MILLISECONDS);
1588 InMemorySnapshotStore.addSnapshotSavedLatch(shardManagerID);
1589 InMemorySnapshotStore.addSnapshotDeletedLatch(shardManagerID);
1591 // construct a mock response message
1592 newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
1593 AddServer addServerMsg = MessageCollectorActor.expectFirstMatching(mockShardLeaderActor,
1595 String addServerId = "member-1-shard-astronauts-" + shardMrgIDSuffix;
1596 assertEquals("AddServer serverId", addServerId, addServerMsg.getNewServerId());
1597 expectMsgClass(duration("5 seconds"), Status.Success.class);
1599 InMemorySnapshotStore.waitForSavedSnapshot(shardManagerID, ShardManagerSnapshot.class);
1600 InMemorySnapshotStore.waitForDeletedSnapshot(shardManagerID);
1601 List<ShardManagerSnapshot> persistedSnapshots = InMemorySnapshotStore.getSnapshots(shardManagerID,
1602 ShardManagerSnapshot.class);
1603 assertEquals("Number of snapshots persisted", 1, persistedSnapshots.size());
1604 ShardManagerSnapshot shardManagerSnapshot = persistedSnapshots.get(0);
1605 assertEquals("Persisted local shards", Sets.newHashSet("default", "astronauts"),
1606 Sets.newHashSet(shardManagerSnapshot.getShardList()));
1609 LOG.info("testAddShardReplica ending");
1613 public void testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader() {
1614 LOG.info("testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader starting");
1615 new TestKit(getSystem()) {
1617 TestActorRef<TestShardManager> shardManager = actorFactory
1618 .createTestActor(newPropsShardMgrWithMockShardActor(), shardMgrID);
1620 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1621 shardManager.tell(new ActorInitialized(), mockShardActor);
1623 String leaderId = "leader-member-shard-default-" + shardMrgIDSuffix;
1624 AddServerReply addServerReply = new AddServerReply(ServerChangeStatus.ALREADY_EXISTS, null);
1625 ActorRef leaderShardActor = shardManager.underlyingActor().getContext()
1626 .actorOf(Props.create(MockRespondActor.class, AddServer.class, addServerReply), leaderId);
1628 MockClusterWrapper.sendMemberUp(shardManager, "leader-member", leaderShardActor.path().toString());
1630 String newReplicaId = "member-1-shard-default-" + shardMrgIDSuffix;
1632 new RoleChangeNotification(newReplicaId, RaftState.Candidate.name(), RaftState.Follower.name()),
1635 new ShardLeaderStateChanged(newReplicaId, leaderId, DataStoreVersions.CURRENT_VERSION),
1638 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1640 MessageCollectorActor.expectFirstMatching(leaderShardActor, AddServer.class);
1642 Failure resp = expectMsgClass(duration("5 seconds"), Failure.class);
1643 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1645 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1646 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1648 // Send message again to verify previous in progress state is
1651 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1652 resp = expectMsgClass(duration("5 seconds"), Failure.class);
1653 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1655 // Send message again with an AddServer timeout to verify the
1656 // pre-existing shard actor isn't terminated.
1659 newDatastoreContextFactory(
1660 datastoreContextBuilder.shardLeaderElectionTimeout(100, TimeUnit.MILLISECONDS).build()),
1662 leaderShardActor.tell(MockRespondActor.CLEAR_RESPONSE, ActorRef.noSender());
1663 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1664 expectMsgClass(duration("5 seconds"), Failure.class);
1666 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1667 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1671 LOG.info("testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader ending");
1675 public void testAddShardReplicaWithPreExistingLocalReplicaLeader() {
1676 LOG.info("testAddShardReplicaWithPreExistingLocalReplicaLeader starting");
1677 new TestKit(getSystem()) {
1679 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1680 ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1682 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1683 shardManager.tell(new ActorInitialized(), mockShardActor);
1684 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
1685 DataStoreVersions.CURRENT_VERSION), getRef());
1687 new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
1690 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1691 Failure resp = expectMsgClass(duration("5 seconds"), Failure.class);
1692 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1694 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1695 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1699 LOG.info("testAddShardReplicaWithPreExistingLocalReplicaLeader ending");
1703 public void testAddShardReplicaWithAddServerReplyFailure() {
1704 LOG.info("testAddShardReplicaWithAddServerReplyFailure starting");
1705 new TestKit(getSystem()) {
1707 TestKit mockShardLeaderKit = new TestKit(getSystem());
1709 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1710 .put("astronauts", Arrays.asList("member-2")).build());
1712 ActorRef mockNewReplicaShardActor = newMockShardActor(getSystem(), "astronauts", "member-1");
1713 final TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(
1714 newTestShardMgrBuilder(mockConfig).shardActor(mockNewReplicaShardActor).props()
1715 .withDispatcher(Dispatchers.DefaultDispatcherId()), shardMgrID);
1716 shardManager.underlyingActor()
1717 .setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1719 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1721 TestKit terminateWatcher = new TestKit(getSystem());
1722 terminateWatcher.watch(mockNewReplicaShardActor);
1724 shardManager.tell(new AddShardReplica("astronauts"), getRef());
1726 AddServer addServerMsg = mockShardLeaderKit.expectMsgClass(AddServer.class);
1727 assertEquals("AddServer serverId", "member-1-shard-astronauts-" + shardMrgIDSuffix,
1728 addServerMsg.getNewServerId());
1729 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.TIMEOUT, null));
1731 Failure failure = expectMsgClass(duration("5 seconds"), Failure.class);
1732 assertEquals("Failure cause", TimeoutException.class, failure.cause().getClass());
1734 shardManager.tell(new FindLocalShard("astronauts", false), getRef());
1735 expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
1737 terminateWatcher.expectTerminated(mockNewReplicaShardActor);
1739 shardManager.tell(new AddShardReplica("astronauts"), getRef());
1740 mockShardLeaderKit.expectMsgClass(AddServer.class);
1741 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.NO_LEADER, null));
1742 failure = expectMsgClass(duration("5 seconds"), Failure.class);
1743 assertEquals("Failure cause", NoShardLeaderException.class, failure.cause().getClass());
1747 LOG.info("testAddShardReplicaWithAddServerReplyFailure ending");
1751 public void testAddShardReplicaWithAlreadyInProgress() {
1752 testServerChangeWhenAlreadyInProgress("astronauts", new AddShardReplica("astronauts"),
1753 AddServer.class, new AddShardReplica("astronauts"));
1757 public void testAddShardReplicaWithFindPrimaryTimeout() {
1758 LOG.info("testAddShardReplicaWithFindPrimaryTimeout starting");
1759 datastoreContextBuilder.shardInitializationTimeout(100, TimeUnit.MILLISECONDS);
1760 new TestKit(getSystem()) {
1762 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1763 .put("astronauts", Arrays.asList("member-2")).build());
1765 final ActorRef newReplicaShardManager = actorFactory
1766 .createActor(newTestShardMgrBuilder(mockConfig).shardActor(mockShardActor).props()
1767 .withDispatcher(Dispatchers.DefaultDispatcherId()), shardMgrID);
1769 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1770 MockClusterWrapper.sendMemberUp(newReplicaShardManager, "member-2",
1771 AddressFromURIString.parse("akka://non-existent@127.0.0.1:5").toString());
1773 newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
1774 Status.Failure resp = expectMsgClass(duration("5 seconds"), Status.Failure.class);
1775 assertEquals("Failure obtained", true, resp.cause() instanceof RuntimeException);
1779 LOG.info("testAddShardReplicaWithFindPrimaryTimeout ending");
1783 public void testRemoveShardReplicaForNonExistentShard() {
1784 new TestKit(getSystem()) {
1786 ActorRef shardManager = actorFactory
1787 .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1788 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1790 shardManager.tell(new RemoveShardReplica("model-inventory", MEMBER_1), getRef());
1791 Status.Failure resp = expectMsgClass(duration("10 seconds"), Status.Failure.class);
1792 assertEquals("Failure obtained", true, resp.cause() instanceof PrimaryNotFoundException);
1801 public void testRemoveShardReplicaLocal() {
1802 new TestKit(getSystem()) {
1804 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1806 final ActorRef respondActor = actorFactory.createActor(Props.create(MockRespondActor.class,
1807 RemoveServer.class, new RemoveServerReply(ServerChangeStatus.OK, null)), memberId);
1809 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
1811 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1812 shardManager.tell(new ActorInitialized(), respondActor);
1813 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
1814 DataStoreVersions.CURRENT_VERSION), getRef());
1816 new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
1819 shardManager.tell(new RemoveShardReplica(Shard.DEFAULT_NAME, MEMBER_1), getRef());
1820 final RemoveServer removeServer = MessageCollectorActor.expectFirstMatching(respondActor,
1821 RemoveServer.class);
1822 assertEquals(ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString(),
1823 removeServer.getServerId());
1824 expectMsgClass(duration("5 seconds"), Success.class);
1830 public void testRemoveShardReplicaRemote() {
1831 MockConfiguration mockConfig = new MockConfiguration(
1832 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
1833 .put("astronauts", Arrays.asList("member-1")).build());
1835 String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
1837 // Create an ActorSystem ShardManager actor for member-1.
1838 final ActorSystem system1 = newActorSystem("Member1");
1839 Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1840 ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
1842 final TestActorRef<TestShardManager> newReplicaShardManager = TestActorRef.create(system1,
1843 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockDefaultShardActor).cluster(
1844 new ClusterWrapperImpl(system1)).props().withDispatcher(Dispatchers.DefaultDispatcherId()),
1847 // Create an ActorSystem ShardManager actor for member-2.
1848 final ActorSystem system2 = newActorSystem("Member2");
1849 Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1851 String name = ShardIdentifier.create("default", MEMBER_2, shardMrgIDSuffix).toString();
1852 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
1853 final TestActorRef<MockRespondActor> mockShardLeaderActor =
1854 TestActorRef.create(system2, Props.create(MockRespondActor.class, RemoveServer.class,
1855 new RemoveServerReply(ServerChangeStatus.OK, memberId2)), name);
1857 LOG.error("Mock Shard Leader Actor : {}", mockShardLeaderActor);
1859 final TestActorRef<TestShardManager> leaderShardManager = TestActorRef.create(system2,
1860 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockShardLeaderActor).cluster(
1861 new ClusterWrapperImpl(system2)).props().withDispatcher(Dispatchers.DefaultDispatcherId()),
1864 // Because mockShardLeaderActor is created at the top level of the actor system it has an address like so,
1865 // akka://cluster-test@127.0.0.1:2559/user/member-2-shard-default-config1
1866 // However when a shard manager has a local shard which is a follower and a leader that is remote it will
1867 // try to compute an address for the remote shard leader using the ShardPeerAddressResolver. This address will
1869 // akka://cluster-test@127.0.0.1:2559/user/shardmanager-config1/member-2-shard-default-config1
1870 // In this specific case if we did a FindPrimary for shard default from member-1 we would come up
1871 // with the address of an actor which does not exist, therefore any message sent to that actor would go to
1873 // To work around this problem we create a ForwardingActor with the right address and pass to it the
1874 // mockShardLeaderActor. The ForwardingActor simply forwards all messages to the mockShardLeaderActor and every
1875 // thing works as expected
1876 final ActorRef actorRef = leaderShardManager.underlyingActor().context()
1877 .actorOf(Props.create(ForwardingActor.class, mockShardLeaderActor),
1878 "member-2-shard-default-" + shardMrgIDSuffix);
1880 LOG.error("Forwarding actor : {}", actorRef);
1882 new TestKit(system1) {
1884 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1885 leaderShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1887 leaderShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1888 newReplicaShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1890 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
1891 leaderShardManager.tell(
1892 new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
1893 mockShardLeaderActor);
1894 leaderShardManager.tell(
1895 new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
1896 mockShardLeaderActor);
1898 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
1899 newReplicaShardManager.tell(
1900 new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class), leaderVersion),
1902 newReplicaShardManager.tell(
1903 new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
1906 newReplicaShardManager.underlyingActor().waitForMemberUp();
1907 leaderShardManager.underlyingActor().waitForMemberUp();
1909 // construct a mock response message
1910 newReplicaShardManager.tell(new RemoveShardReplica("default", MEMBER_1), getRef());
1911 RemoveServer removeServer = MessageCollectorActor.expectFirstMatching(mockShardLeaderActor,
1912 RemoveServer.class);
1913 String removeServerId = ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString();
1914 assertEquals("RemoveServer serverId", removeServerId, removeServer.getServerId());
1915 expectMsgClass(duration("5 seconds"), Status.Success.class);
1921 public void testRemoveShardReplicaWhenAnotherRemoveShardReplicaAlreadyInProgress() {
1922 testServerChangeWhenAlreadyInProgress("astronauts", new RemoveShardReplica("astronauts", MEMBER_2),
1923 RemoveServer.class, new RemoveShardReplica("astronauts", MEMBER_3));
1927 public void testRemoveShardReplicaWhenAddShardReplicaAlreadyInProgress() {
1928 testServerChangeWhenAlreadyInProgress("astronauts", new AddShardReplica("astronauts"),
1929 AddServer.class, new RemoveShardReplica("astronauts", MEMBER_2));
1933 public void testServerChangeWhenAlreadyInProgress(final String shardName, final Object firstServerChange,
1934 final Class<?> firstForwardedServerChangeClass,
1935 final Object secondServerChange) {
1936 new TestKit(getSystem()) {
1938 TestKit mockShardLeaderKit = new TestKit(getSystem());
1939 final TestKit secondRequestKit = new TestKit(getSystem());
1941 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1942 .put(shardName, Arrays.asList("member-2")).build());
1944 final TestActorRef<TestShardManager> shardManager = TestActorRef.create(getSystem(),
1945 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockShardActor)
1946 .cluster(new MockClusterWrapper()).props()
1947 .withDispatcher(Dispatchers.DefaultDispatcherId()),
1950 shardManager.underlyingActor()
1951 .setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1953 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1955 shardManager.tell(firstServerChange, getRef());
1957 mockShardLeaderKit.expectMsgClass(firstForwardedServerChangeClass);
1959 shardManager.tell(secondServerChange, secondRequestKit.getRef());
1961 secondRequestKit.expectMsgClass(duration("5 seconds"), Failure.class);
1967 public void testServerRemovedShardActorNotRunning() {
1968 LOG.info("testServerRemovedShardActorNotRunning starting");
1969 new TestKit(getSystem()) {
1971 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1972 .put("default", Arrays.asList("member-1", "member-2"))
1973 .put("astronauts", Arrays.asList("member-2"))
1974 .put("people", Arrays.asList("member-1", "member-2")).build());
1976 TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(
1977 newShardMgrProps(mockConfig).withDispatcher(Dispatchers.DefaultDispatcherId()));
1979 shardManager.underlyingActor().waitForRecoveryComplete();
1980 shardManager.tell(new FindLocalShard("people", false), getRef());
1981 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1983 shardManager.tell(new FindLocalShard("default", false), getRef());
1984 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1986 // Removed the default shard replica from member-1
1987 ShardIdentifier.Builder builder = new ShardIdentifier.Builder();
1988 ShardIdentifier shardId = builder.shardName("default").memberName(MEMBER_1).type(shardMrgIDSuffix)
1990 shardManager.tell(new ServerRemoved(shardId.toString()), getRef());
1992 shardManager.underlyingActor().verifySnapshotPersisted(Sets.newHashSet("people"));
1996 LOG.info("testServerRemovedShardActorNotRunning ending");
2000 public void testServerRemovedShardActorRunning() {
2001 LOG.info("testServerRemovedShardActorRunning starting");
2002 new TestKit(getSystem()) {
2004 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
2005 .put("default", Arrays.asList("member-1", "member-2"))
2006 .put("astronauts", Arrays.asList("member-2"))
2007 .put("people", Arrays.asList("member-1", "member-2")).build());
2009 String shardId = ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString();
2010 ActorRef shard = actorFactory.createActor(MessageCollectorActor.props(), shardId);
2012 TestActorRef<TestShardManager> shardManager = actorFactory
2013 .createTestActor(newTestShardMgrBuilder(mockConfig).addShardActor("default", shard).props()
2014 .withDispatcher(Dispatchers.DefaultDispatcherId()));
2016 shardManager.underlyingActor().waitForRecoveryComplete();
2018 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
2019 shardManager.tell(new ActorInitialized(), shard);
2021 waitForShardInitialized(shardManager, "people", this);
2022 waitForShardInitialized(shardManager, "default", this);
2024 // Removed the default shard replica from member-1
2025 shardManager.tell(new ServerRemoved(shardId), getRef());
2027 shardManager.underlyingActor().verifySnapshotPersisted(Sets.newHashSet("people"));
2029 MessageCollectorActor.expectFirstMatching(shard, Shutdown.class);
2033 LOG.info("testServerRemovedShardActorRunning ending");
2037 public void testShardPersistenceWithRestoredData() {
2038 LOG.info("testShardPersistenceWithRestoredData starting");
2039 new TestKit(getSystem()) {
2041 MockConfiguration mockConfig =
2042 new MockConfiguration(ImmutableMap.<String, List<String>>builder()
2043 .put("default", Arrays.asList("member-1", "member-2"))
2044 .put("astronauts", Arrays.asList("member-2"))
2045 .put("people", Arrays.asList("member-1", "member-2")).build());
2046 String[] restoredShards = {"default", "astronauts"};
2047 ShardManagerSnapshot snapshot =
2048 new ShardManagerSnapshot(Arrays.asList(restoredShards), Collections.emptyMap());
2049 InMemorySnapshotStore.addSnapshot("shard-manager-" + shardMrgIDSuffix, snapshot);
2051 // create shardManager to come up with restored data
2052 TestActorRef<TestShardManager> newRestoredShardManager = actorFactory.createTestActor(
2053 newShardMgrProps(mockConfig).withDispatcher(Dispatchers.DefaultDispatcherId()));
2055 newRestoredShardManager.underlyingActor().waitForRecoveryComplete();
2057 newRestoredShardManager.tell(new FindLocalShard("people", false), getRef());
2058 LocalShardNotFound notFound = expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
2059 assertEquals("for uninitialized shard", "people", notFound.getShardName());
2061 // Verify a local shard is created for the restored shards,
2062 // although we expect a NotInitializedException for the shards
2063 // as the actor initialization
2064 // message is not sent for them
2065 newRestoredShardManager.tell(new FindLocalShard("default", false), getRef());
2066 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
2068 newRestoredShardManager.tell(new FindLocalShard("astronauts", false), getRef());
2069 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
2073 LOG.info("testShardPersistenceWithRestoredData ending");
2077 public void testShutDown() throws Exception {
2078 LOG.info("testShutDown starting");
2079 new TestKit(getSystem()) {
2081 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
2082 .put("shard1", Arrays.asList("member-1")).put("shard2", Arrays.asList("member-1")).build());
2084 String shardId1 = ShardIdentifier.create("shard1", MEMBER_1, shardMrgIDSuffix).toString();
2085 ActorRef shard1 = actorFactory.createActor(MessageCollectorActor.props(), shardId1);
2087 String shardId2 = ShardIdentifier.create("shard2", MEMBER_1, shardMrgIDSuffix).toString();
2088 ActorRef shard2 = actorFactory.createActor(MessageCollectorActor.props(), shardId2);
2090 ActorRef shardManager = actorFactory.createActor(newTestShardMgrBuilder(mockConfig)
2091 .addShardActor("shard1", shard1).addShardActor("shard2", shard2).props());
2093 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
2094 shardManager.tell(new ActorInitialized(), shard1);
2095 shardManager.tell(new ActorInitialized(), shard2);
2097 FiniteDuration duration = FiniteDuration.create(5, TimeUnit.SECONDS);
2098 Future<Boolean> stopFuture = Patterns.gracefulStop(shardManager, duration, Shutdown.INSTANCE);
2100 MessageCollectorActor.expectFirstMatching(shard1, Shutdown.class);
2101 MessageCollectorActor.expectFirstMatching(shard2, Shutdown.class);
2104 Await.ready(stopFuture, FiniteDuration.create(500, TimeUnit.MILLISECONDS));
2105 fail("ShardManager actor stopped without waiting for the Shards to be stopped");
2106 } catch (TimeoutException e) {
2110 actorFactory.killActor(shard1, this);
2111 actorFactory.killActor(shard2, this);
2113 Boolean stopped = Await.result(stopFuture, duration);
2114 assertEquals("Stopped", Boolean.TRUE, stopped);
2118 LOG.info("testShutDown ending");
2122 public void testChangeServersVotingStatus() {
2123 new TestKit(getSystem()) {
2125 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
2127 ActorRef respondActor = actorFactory
2128 .createActor(Props.create(MockRespondActor.class, ChangeServersVotingStatus.class,
2129 new ServerChangeReply(ServerChangeStatus.OK, null)), memberId);
2131 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
2133 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
2134 shardManager.tell(new ActorInitialized(), respondActor);
2135 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
2136 DataStoreVersions.CURRENT_VERSION), getRef());
2138 new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
2142 new ChangeShardMembersVotingStatus("default", ImmutableMap.of("member-2", Boolean.TRUE)),
2145 ChangeServersVotingStatus actualChangeStatusMsg = MessageCollectorActor
2146 .expectFirstMatching(respondActor, ChangeServersVotingStatus.class);
2147 assertEquals("ChangeServersVotingStatus map", actualChangeStatusMsg.getServerVotingStatusMap(),
2148 ImmutableMap.of(ShardIdentifier
2149 .create("default", MemberName.forName("member-2"), shardMrgIDSuffix).toString(),
2152 expectMsgClass(duration("5 seconds"), Success.class);
2158 public void testChangeServersVotingStatusWithNoLeader() {
2159 new TestKit(getSystem()) {
2161 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
2163 ActorRef respondActor = actorFactory
2164 .createActor(Props.create(MockRespondActor.class, ChangeServersVotingStatus.class,
2165 new ServerChangeReply(ServerChangeStatus.NO_LEADER, null)), memberId);
2167 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
2169 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
2170 shardManager.tell(new ActorInitialized(), respondActor);
2171 shardManager.tell(new RoleChangeNotification(memberId, null, RaftState.Follower.name()), respondActor);
2174 new ChangeShardMembersVotingStatus("default", ImmutableMap.of("member-2", Boolean.TRUE)),
2177 MessageCollectorActor.expectFirstMatching(respondActor, ChangeServersVotingStatus.class);
2179 Status.Failure resp = expectMsgClass(duration("5 seconds"), Status.Failure.class);
2180 assertEquals("Failure resposnse", true, resp.cause() instanceof NoShardLeaderException);
2185 public static class TestShardManager extends ShardManager {
2186 private final CountDownLatch recoveryComplete = new CountDownLatch(1);
2187 private final CountDownLatch snapshotPersist = new CountDownLatch(1);
2188 private ShardManagerSnapshot snapshot;
2189 private final Map<String, ActorRef> shardActors;
2190 private final ActorRef shardActor;
2191 private CountDownLatch findPrimaryMessageReceived = new CountDownLatch(1);
2192 private CountDownLatch memberUpReceived = new CountDownLatch(1);
2193 private CountDownLatch memberRemovedReceived = new CountDownLatch(1);
2194 private CountDownLatch memberUnreachableReceived = new CountDownLatch(1);
2195 private CountDownLatch memberReachableReceived = new CountDownLatch(1);
2196 private volatile MessageInterceptor messageInterceptor;
2198 TestShardManager(final Builder builder) {
2200 shardActor = builder.shardActor;
2201 shardActors = builder.shardActors;
2205 protected void handleRecover(final Object message) throws Exception {
2207 super.handleRecover(message);
2209 if (message instanceof RecoveryCompleted) {
2210 recoveryComplete.countDown();
2215 private void countDownIfOther(final Member member, final CountDownLatch latch) {
2216 if (!getCluster().getCurrentMemberName().equals(memberToName(member))) {
2222 public void handleCommand(final Object message) throws Exception {
2224 if (messageInterceptor != null && messageInterceptor.canIntercept(message)) {
2225 getSender().tell(messageInterceptor.apply(message), getSelf());
2227 super.handleCommand(message);
2230 if (message instanceof FindPrimary) {
2231 findPrimaryMessageReceived.countDown();
2232 } else if (message instanceof ClusterEvent.MemberUp) {
2233 countDownIfOther(((ClusterEvent.MemberUp) message).member(), memberUpReceived);
2234 } else if (message instanceof ClusterEvent.MemberRemoved) {
2235 countDownIfOther(((ClusterEvent.MemberRemoved) message).member(), memberRemovedReceived);
2236 } else if (message instanceof ClusterEvent.UnreachableMember) {
2237 countDownIfOther(((ClusterEvent.UnreachableMember) message).member(), memberUnreachableReceived);
2238 } else if (message instanceof ClusterEvent.ReachableMember) {
2239 countDownIfOther(((ClusterEvent.ReachableMember) message).member(), memberReachableReceived);
2244 void setMessageInterceptor(final MessageInterceptor messageInterceptor) {
2245 this.messageInterceptor = messageInterceptor;
2248 void waitForRecoveryComplete() {
2249 assertEquals("Recovery complete", true,
2250 Uninterruptibles.awaitUninterruptibly(recoveryComplete, 5, TimeUnit.SECONDS));
2253 public void waitForMemberUp() {
2254 assertEquals("MemberUp received", true,
2255 Uninterruptibles.awaitUninterruptibly(memberUpReceived, 5, TimeUnit.SECONDS));
2256 memberUpReceived = new CountDownLatch(1);
2259 void waitForMemberRemoved() {
2260 assertEquals("MemberRemoved received", true,
2261 Uninterruptibles.awaitUninterruptibly(memberRemovedReceived, 5, TimeUnit.SECONDS));
2262 memberRemovedReceived = new CountDownLatch(1);
2265 void waitForUnreachableMember() {
2266 assertEquals("UnreachableMember received", true,
2267 Uninterruptibles.awaitUninterruptibly(memberUnreachableReceived, 5, TimeUnit.SECONDS
2269 memberUnreachableReceived = new CountDownLatch(1);
2272 void waitForReachableMember() {
2273 assertEquals("ReachableMember received", true,
2274 Uninterruptibles.awaitUninterruptibly(memberReachableReceived, 5, TimeUnit.SECONDS));
2275 memberReachableReceived = new CountDownLatch(1);
2278 void verifyFindPrimary() {
2279 assertEquals("FindPrimary received", true,
2280 Uninterruptibles.awaitUninterruptibly(findPrimaryMessageReceived, 5, TimeUnit.SECONDS));
2281 findPrimaryMessageReceived = new CountDownLatch(1);
2284 public static Builder builder(final DatastoreContext.Builder datastoreContextBuilder) {
2285 return new Builder(datastoreContextBuilder);
2288 public static class Builder extends AbstractGenericCreator<Builder, TestShardManager> {
2289 private ActorRef shardActor;
2290 private final Map<String, ActorRef> shardActors = new HashMap<>();
2292 Builder(final DatastoreContext.Builder datastoreContextBuilder) {
2293 super(TestShardManager.class);
2294 datastoreContextFactory(newDatastoreContextFactory(datastoreContextBuilder.build()));
2297 Builder shardActor(final ActorRef newShardActor) {
2298 this.shardActor = newShardActor;
2302 Builder addShardActor(final String shardName, final ActorRef actorRef) {
2303 shardActors.put(shardName, actorRef);
2309 public void saveSnapshot(final Object obj) {
2310 snapshot = (ShardManagerSnapshot) obj;
2311 snapshotPersist.countDown();
2312 super.saveSnapshot(obj);
2315 void verifySnapshotPersisted(final Set<String> shardList) {
2316 assertEquals("saveSnapshot invoked", true,
2317 Uninterruptibles.awaitUninterruptibly(snapshotPersist, 5, TimeUnit.SECONDS));
2318 assertEquals("Shard Persisted", shardList, Sets.newHashSet(snapshot.getShardList()));
2322 protected ActorRef newShardActor(final ShardInformation info) {
2323 if (shardActors.get(info.getShardName()) != null) {
2324 return shardActors.get(info.getShardName());
2327 if (shardActor != null) {
2331 return super.newShardActor(info);
2335 private abstract static class AbstractGenericCreator<T extends AbstractGenericCreator<T, ?>, C extends ShardManager>
2336 extends AbstractShardManagerCreator<T> {
2337 private final Class<C> shardManagerClass;
2339 AbstractGenericCreator(final Class<C> shardManagerClass) {
2340 this.shardManagerClass = shardManagerClass;
2341 cluster(new MockClusterWrapper()).configuration(new MockConfiguration()).waitTillReadyCountDownLatch(ready)
2342 .primaryShardInfoCache(new PrimaryShardInfoFutureCache());
2346 public Props props() {
2348 return Props.create(shardManagerClass, this);
2352 private static class GenericCreator<C extends ShardManager> extends AbstractGenericCreator<GenericCreator<C>, C> {
2353 GenericCreator(final Class<C> shardManagerClass) {
2354 super(shardManagerClass);
2358 private static class DelegatingShardManagerCreator implements Creator<ShardManager> {
2359 private static final long serialVersionUID = 1L;
2360 private final Creator<ShardManager> delegate;
2362 DelegatingShardManagerCreator(final Creator<ShardManager> delegate) {
2363 this.delegate = delegate;
2367 public ShardManager create() throws Exception {
2368 return delegate.create();
2372 interface MessageInterceptor extends Function<Object, Object> {
2373 boolean canIntercept(Object message);
2376 private static MessageInterceptor newFindPrimaryInterceptor(final ActorRef primaryActor) {
2377 return new MessageInterceptor() {
2379 public Object apply(final Object message) {
2380 return new RemotePrimaryShardFound(Serialization.serializedActorPath(primaryActor), (short) 1);
2384 public boolean canIntercept(final Object message) {
2385 return message instanceof FindPrimary;
2390 private static class MockRespondActor extends MessageCollectorActor {
2391 static final String CLEAR_RESPONSE = "clear-response";
2393 private Object responseMsg;
2394 private final Class<?> requestClass;
2396 @SuppressWarnings("unused")
2397 MockRespondActor(final Class<?> requestClass, final Object responseMsg) {
2398 this.requestClass = requestClass;
2399 this.responseMsg = responseMsg;
2403 public void onReceive(final Object message) throws Exception {
2404 if (message.equals(CLEAR_RESPONSE)) {
2407 super.onReceive(message);
2408 if (message.getClass().equals(requestClass) && responseMsg != null) {
2409 getSender().tell(responseMsg, getSelf());