2 * Copyright (c) 2014, 2015 Cisco Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
9 package org.opendaylight.controller.cluster.datastore.shardmanager;
11 import static org.junit.Assert.assertEquals;
12 import static org.junit.Assert.assertNotNull;
13 import static org.junit.Assert.assertNull;
14 import static org.junit.Assert.assertSame;
15 import static org.junit.Assert.assertTrue;
16 import static org.junit.Assert.fail;
17 import static org.mockito.Mockito.mock;
18 import static org.mockito.Mockito.never;
19 import static org.mockito.Mockito.times;
20 import static org.mockito.Mockito.verify;
22 import akka.actor.ActorRef;
23 import akka.actor.ActorSystem;
24 import akka.actor.AddressFromURIString;
25 import akka.actor.Props;
26 import akka.actor.Status;
27 import akka.actor.Status.Failure;
28 import akka.actor.Status.Success;
29 import akka.cluster.Cluster;
30 import akka.cluster.ClusterEvent;
31 import akka.cluster.Member;
32 import akka.dispatch.Dispatchers;
33 import akka.japi.Creator;
34 import akka.pattern.Patterns;
35 import akka.persistence.RecoveryCompleted;
36 import akka.serialization.Serialization;
37 import akka.testkit.JavaTestKit;
38 import akka.testkit.TestActorRef;
39 import akka.util.Timeout;
40 import com.google.common.base.Function;
41 import com.google.common.base.Stopwatch;
42 import com.google.common.collect.ImmutableMap;
43 import com.google.common.collect.ImmutableSet;
44 import com.google.common.collect.Lists;
45 import com.google.common.collect.Sets;
46 import com.google.common.util.concurrent.Uninterruptibles;
47 import com.typesafe.config.ConfigFactory;
49 import java.util.AbstractMap;
50 import java.util.Arrays;
51 import java.util.Collection;
52 import java.util.Collections;
53 import java.util.HashMap;
54 import java.util.List;
56 import java.util.Map.Entry;
58 import java.util.concurrent.CountDownLatch;
59 import java.util.concurrent.TimeUnit;
60 import java.util.concurrent.TimeoutException;
61 import java.util.stream.Collectors;
62 import org.junit.Test;
63 import org.mockito.Mockito;
64 import org.opendaylight.controller.cluster.access.concepts.MemberName;
65 import org.opendaylight.controller.cluster.datastore.AbstractShardManagerTest;
66 import org.opendaylight.controller.cluster.datastore.ClusterWrapperImpl;
67 import org.opendaylight.controller.cluster.datastore.DataStoreVersions;
68 import org.opendaylight.controller.cluster.datastore.DatastoreContext;
69 import org.opendaylight.controller.cluster.datastore.DatastoreContextFactory;
70 import org.opendaylight.controller.cluster.datastore.DistributedDataStore;
71 import org.opendaylight.controller.cluster.datastore.Shard;
72 import org.opendaylight.controller.cluster.datastore.ShardManager.SchemaContextModules;
73 import org.opendaylight.controller.cluster.datastore.config.ConfigurationImpl;
74 import org.opendaylight.controller.cluster.datastore.config.EmptyModuleShardConfigProvider;
75 import org.opendaylight.controller.cluster.datastore.config.ModuleShardConfiguration;
76 import org.opendaylight.controller.cluster.datastore.exceptions.AlreadyExistsException;
77 import org.opendaylight.controller.cluster.datastore.exceptions.NoShardLeaderException;
78 import org.opendaylight.controller.cluster.datastore.exceptions.NotInitializedException;
79 import org.opendaylight.controller.cluster.datastore.exceptions.PrimaryNotFoundException;
80 import org.opendaylight.controller.cluster.datastore.identifiers.ShardIdentifier;
81 import org.opendaylight.controller.cluster.datastore.identifiers.ShardManagerIdentifier;
82 import org.opendaylight.controller.cluster.datastore.messages.ActorInitialized;
83 import org.opendaylight.controller.cluster.datastore.messages.AddShardReplica;
84 import org.opendaylight.controller.cluster.datastore.messages.ChangeShardMembersVotingStatus;
85 import org.opendaylight.controller.cluster.datastore.messages.CreateShard;
86 import org.opendaylight.controller.cluster.datastore.messages.FindLocalShard;
87 import org.opendaylight.controller.cluster.datastore.messages.FindPrimary;
88 import org.opendaylight.controller.cluster.datastore.messages.LocalPrimaryShardFound;
89 import org.opendaylight.controller.cluster.datastore.messages.LocalShardFound;
90 import org.opendaylight.controller.cluster.datastore.messages.LocalShardNotFound;
91 import org.opendaylight.controller.cluster.datastore.messages.PeerDown;
92 import org.opendaylight.controller.cluster.datastore.messages.PeerUp;
93 import org.opendaylight.controller.cluster.datastore.messages.PrimaryShardInfo;
94 import org.opendaylight.controller.cluster.datastore.messages.RemotePrimaryShardFound;
95 import org.opendaylight.controller.cluster.datastore.messages.RemoveShardReplica;
96 import org.opendaylight.controller.cluster.datastore.messages.ShardLeaderStateChanged;
97 import org.opendaylight.controller.cluster.datastore.messages.UpdateSchemaContext;
98 import org.opendaylight.controller.cluster.datastore.persisted.DatastoreSnapshot;
99 import org.opendaylight.controller.cluster.datastore.persisted.DatastoreSnapshot.ShardSnapshot;
100 import org.opendaylight.controller.cluster.datastore.persisted.ShardManagerSnapshot;
101 import org.opendaylight.controller.cluster.datastore.utils.ForwardingActor;
102 import org.opendaylight.controller.cluster.datastore.utils.MockClusterWrapper;
103 import org.opendaylight.controller.cluster.datastore.utils.MockConfiguration;
104 import org.opendaylight.controller.cluster.datastore.utils.PrimaryShardInfoFutureCache;
105 import org.opendaylight.controller.cluster.notifications.LeaderStateChanged;
106 import org.opendaylight.controller.cluster.notifications.RegisterRoleChangeListener;
107 import org.opendaylight.controller.cluster.notifications.RoleChangeNotification;
108 import org.opendaylight.controller.cluster.raft.RaftState;
109 import org.opendaylight.controller.cluster.raft.base.messages.FollowerInitialSyncUpStatus;
110 import org.opendaylight.controller.cluster.raft.base.messages.SwitchBehavior;
111 import org.opendaylight.controller.cluster.raft.client.messages.GetSnapshot;
112 import org.opendaylight.controller.cluster.raft.client.messages.Shutdown;
113 import org.opendaylight.controller.cluster.raft.messages.AddServer;
114 import org.opendaylight.controller.cluster.raft.messages.AddServerReply;
115 import org.opendaylight.controller.cluster.raft.messages.ChangeServersVotingStatus;
116 import org.opendaylight.controller.cluster.raft.messages.RemoveServer;
117 import org.opendaylight.controller.cluster.raft.messages.RemoveServerReply;
118 import org.opendaylight.controller.cluster.raft.messages.ServerChangeReply;
119 import org.opendaylight.controller.cluster.raft.messages.ServerChangeStatus;
120 import org.opendaylight.controller.cluster.raft.messages.ServerRemoved;
121 import org.opendaylight.controller.cluster.raft.policy.DisableElectionsRaftPolicy;
122 import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal;
123 import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore;
124 import org.opendaylight.controller.cluster.raft.utils.MessageCollectorActor;
125 import org.opendaylight.controller.md.cluster.datastore.model.TestModel;
126 import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTree;
127 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
128 import org.slf4j.Logger;
129 import org.slf4j.LoggerFactory;
130 import scala.concurrent.Await;
131 import scala.concurrent.Future;
132 import scala.concurrent.duration.FiniteDuration;
134 public class ShardManagerTest extends AbstractShardManagerTest {
135 private static final Logger LOG = LoggerFactory.getLogger(ShardManagerTest.class);
136 private static final MemberName MEMBER_2 = MemberName.forName("member-2");
137 private static final MemberName MEMBER_3 = MemberName.forName("member-3");
139 private final String shardMgrID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
141 private ActorSystem newActorSystem(String config) {
142 ActorSystem system = ActorSystem.create("cluster-test", ConfigFactory.load().getConfig(config));
143 actorSystems.add(system);
147 private ActorRef newMockShardActor(ActorSystem system, String shardName, String memberName) {
148 String name = ShardIdentifier.create(shardName, MemberName.forName(memberName), "config").toString();
149 if (system == getSystem()) {
150 return actorFactory.createTestActor(Props.create(MessageCollectorActor.class), name);
153 return TestActorRef.create(system, Props.create(MessageCollectorActor.class), name);
156 private Props newShardMgrProps() {
157 return newShardMgrProps(new MockConfiguration());
160 private static DatastoreContextFactory newDatastoreContextFactory(DatastoreContext datastoreContext) {
161 DatastoreContextFactory mockFactory = mock(DatastoreContextFactory.class);
162 Mockito.doReturn(datastoreContext).when(mockFactory).getBaseDatastoreContext();
163 Mockito.doReturn(datastoreContext).when(mockFactory).getShardDatastoreContext(Mockito.anyString());
167 private TestShardManager.Builder newTestShardMgrBuilderWithMockShardActor() {
168 return newTestShardMgrBuilderWithMockShardActor(mockShardActor);
171 private TestShardManager.Builder newTestShardMgrBuilderWithMockShardActor(ActorRef shardActor) {
172 return TestShardManager.builder(datastoreContextBuilder).shardActor(shardActor)
173 .distributedDataStore(mock(DistributedDataStore.class));
177 private Props newPropsShardMgrWithMockShardActor() {
178 return newTestShardMgrBuilderWithMockShardActor().props().withDispatcher(
179 Dispatchers.DefaultDispatcherId());
182 private Props newPropsShardMgrWithMockShardActor(ActorRef shardActor) {
183 return newTestShardMgrBuilderWithMockShardActor(shardActor).props();
187 private TestShardManager newTestShardManager() {
188 return newTestShardManager(newShardMgrProps());
191 private TestShardManager newTestShardManager(Props props) {
192 TestActorRef<TestShardManager> shardManagerActor = actorFactory.createTestActor(props);
193 TestShardManager shardManager = shardManagerActor.underlyingActor();
194 shardManager.waitForRecoveryComplete();
198 private static void waitForShardInitialized(ActorRef shardManager, String shardName, JavaTestKit kit) {
199 AssertionError last = null;
200 Stopwatch sw = Stopwatch.createStarted();
201 while (sw.elapsed(TimeUnit.SECONDS) <= 5) {
203 shardManager.tell(new FindLocalShard(shardName, true), kit.getRef());
204 kit.expectMsgClass(LocalShardFound.class);
206 } catch (AssertionError e) {
210 Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
216 @SuppressWarnings("unchecked")
217 private static <T> T expectMsgClassOrFailure(Class<T> msgClass, JavaTestKit kit, String msg) {
218 Object reply = kit.expectMsgAnyClassOf(JavaTestKit.duration("5 sec"), msgClass, Failure.class);
219 if (reply instanceof Failure) {
220 throw new AssertionError(msg + " failed", ((Failure)reply).cause());
227 public void testPerShardDatastoreContext() throws Exception {
228 LOG.info("testPerShardDatastoreContext starting");
229 final DatastoreContextFactory mockFactory = newDatastoreContextFactory(
230 datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
233 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(6).build())
234 .when(mockFactory).getShardDatastoreContext("default");
237 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(7).build())
238 .when(mockFactory).getShardDatastoreContext("topology");
240 final MockConfiguration mockConfig = new MockConfiguration() {
242 public Collection<String> getMemberShardNames(MemberName memberName) {
243 return Arrays.asList("default", "topology");
247 public Collection<MemberName> getMembersFromShardName(String shardName) {
248 return members("member-1");
252 final TestActorRef<MessageCollectorActor> defaultShardActor = actorFactory.createTestActor(
253 Props.create(MessageCollectorActor.class), actorFactory.generateActorId("default"));
254 final TestActorRef<MessageCollectorActor> topologyShardActor = actorFactory.createTestActor(
255 Props.create(MessageCollectorActor.class), actorFactory.generateActorId("topology"));
257 final Map<String, Entry<ActorRef, DatastoreContext>> shardInfoMap = Collections.synchronizedMap(
258 new HashMap<String, Entry<ActorRef, DatastoreContext>>());
259 shardInfoMap.put("default", new AbstractMap.SimpleEntry<>(defaultShardActor, null));
260 shardInfoMap.put("topology", new AbstractMap.SimpleEntry<>(topologyShardActor, null));
262 final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
263 final CountDownLatch newShardActorLatch = new CountDownLatch(2);
264 class LocalShardManager extends ShardManager {
265 LocalShardManager(AbstractShardManagerCreator<?> creator) {
270 protected ActorRef newShardActor(SchemaContext schemaContext, ShardInformation info) {
271 Entry<ActorRef, DatastoreContext> entry = shardInfoMap.get(info.getShardName());
274 ref = entry.getKey();
275 entry.setValue(info.getDatastoreContext());
278 newShardActorLatch.countDown();
283 final Creator<ShardManager> creator = new Creator<ShardManager>() {
284 private static final long serialVersionUID = 1L;
286 public ShardManager create() throws Exception {
287 return new LocalShardManager(
288 new GenericCreator<>(LocalShardManager.class).datastoreContextFactory(mockFactory)
289 .primaryShardInfoCache(primaryShardInfoCache).configuration(mockConfig));
293 JavaTestKit kit = new JavaTestKit(getSystem());
295 final ActorRef shardManager = actorFactory.createActor(Props.create(
296 new DelegatingShardManagerCreator(creator)).withDispatcher(Dispatchers.DefaultDispatcherId()));
298 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), kit.getRef());
300 assertEquals("Shard actors created", true, newShardActorLatch.await(5, TimeUnit.SECONDS));
301 assertEquals("getShardElectionTimeoutFactor", 6,
302 shardInfoMap.get("default").getValue().getShardElectionTimeoutFactor());
303 assertEquals("getShardElectionTimeoutFactor", 7,
304 shardInfoMap.get("topology").getValue().getShardElectionTimeoutFactor());
306 DatastoreContextFactory newMockFactory = newDatastoreContextFactory(
307 datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
309 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(66).build())
310 .when(newMockFactory).getShardDatastoreContext("default");
313 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(77).build())
314 .when(newMockFactory).getShardDatastoreContext("topology");
316 shardManager.tell(newMockFactory, kit.getRef());
318 DatastoreContext newContext = MessageCollectorActor.expectFirstMatching(defaultShardActor,
319 DatastoreContext.class);
320 assertEquals("getShardElectionTimeoutFactor", 66, newContext.getShardElectionTimeoutFactor());
322 newContext = MessageCollectorActor.expectFirstMatching(topologyShardActor, DatastoreContext.class);
323 assertEquals("getShardElectionTimeoutFactor", 77, newContext.getShardElectionTimeoutFactor());
325 LOG.info("testPerShardDatastoreContext ending");
329 public void testOnReceiveFindPrimaryForNonExistentShard() throws Exception {
330 new JavaTestKit(getSystem()) {
332 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
334 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
336 shardManager.tell(new FindPrimary("non-existent", false), getRef());
338 expectMsgClass(duration("5 seconds"), PrimaryNotFoundException.class);
344 public void testOnReceiveFindPrimaryForLocalLeaderShard() throws Exception {
345 LOG.info("testOnReceiveFindPrimaryForLocalLeaderShard starting");
346 new JavaTestKit(getSystem()) {
348 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
350 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
352 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
353 shardManager.tell(new ActorInitialized(), mockShardActor);
355 DataTree mockDataTree = mock(DataTree.class);
356 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
357 DataStoreVersions.CURRENT_VERSION), getRef());
359 MessageCollectorActor.expectFirstMatching(mockShardActor, RegisterRoleChangeListener.class);
361 new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
364 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
366 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
367 LocalPrimaryShardFound.class);
368 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
369 primaryFound.getPrimaryPath().contains("member-1-shard-default"));
370 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
374 LOG.info("testOnReceiveFindPrimaryForLocalLeaderShard ending");
378 public void testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp() throws Exception {
379 LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp starting");
380 new JavaTestKit(getSystem()) {
382 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
384 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
385 shardManager.tell(new ActorInitialized(), mockShardActor);
387 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
388 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
390 new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
392 shardManager.tell(new LeaderStateChanged(memberId1, memberId2, DataStoreVersions.CURRENT_VERSION),
395 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
397 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
401 LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp ending");
405 public void testOnReceiveFindPrimaryForNonLocalLeaderShard() throws Exception {
406 LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShard starting");
407 new JavaTestKit(getSystem()) {
409 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
411 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
412 shardManager.tell(new ActorInitialized(), mockShardActor);
414 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
415 MockClusterWrapper.sendMemberUp(shardManager, "member-2", getRef().path().toString());
417 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
419 new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
421 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
422 shardManager.tell(new ShardLeaderStateChanged(memberId1, memberId2, leaderVersion), mockShardActor);
424 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
426 RemotePrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
427 RemotePrimaryShardFound.class);
428 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
429 primaryFound.getPrimaryPath().contains("member-2-shard-default"));
430 assertEquals("getPrimaryVersion", leaderVersion, primaryFound.getPrimaryVersion());
434 LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShard ending");
438 public void testOnReceiveFindPrimaryForUninitializedShard() throws Exception {
439 new JavaTestKit(getSystem()) {
441 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
443 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
445 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
451 public void testOnReceiveFindPrimaryForInitializedShardWithNoRole() throws Exception {
452 new JavaTestKit(getSystem()) {
454 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
456 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
457 shardManager.tell(new ActorInitialized(), mockShardActor);
459 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
461 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
467 public void testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId() throws Exception {
468 LOG.info("testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId starting");
469 new JavaTestKit(getSystem()) {
471 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
473 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
474 shardManager.tell(new ActorInitialized(), mockShardActor);
476 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
478 new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Follower.name()),
481 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
483 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
485 DataTree mockDataTree = mock(DataTree.class);
486 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
487 DataStoreVersions.CURRENT_VERSION), mockShardActor);
489 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
491 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
492 LocalPrimaryShardFound.class);
493 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
494 primaryFound.getPrimaryPath().contains("member-1-shard-default"));
495 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
499 LOG.info("testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId starting");
503 public void testOnReceiveFindPrimaryWaitForShardLeader() throws Exception {
504 LOG.info("testOnReceiveFindPrimaryWaitForShardLeader starting");
505 datastoreContextBuilder.shardInitializationTimeout(10, TimeUnit.SECONDS);
506 new JavaTestKit(getSystem()) {
508 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
510 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
512 // We're passing waitUntilInitialized = true to FindPrimary so
513 // the response should be
514 // delayed until we send ActorInitialized and
515 // RoleChangeNotification.
516 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
518 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
520 shardManager.tell(new ActorInitialized(), mockShardActor);
522 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
524 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
526 new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
529 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
531 DataTree mockDataTree = mock(DataTree.class);
532 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
533 DataStoreVersions.CURRENT_VERSION), mockShardActor);
535 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
536 LocalPrimaryShardFound.class);
537 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
538 primaryFound.getPrimaryPath().contains("member-1-shard-default"));
539 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
541 expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
545 LOG.info("testOnReceiveFindPrimaryWaitForShardLeader ending");
549 public void testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard() throws Exception {
550 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard starting");
551 new JavaTestKit(getSystem()) {
553 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
555 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
557 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
559 expectMsgClass(duration("2 seconds"), NotInitializedException.class);
561 shardManager.tell(new ActorInitialized(), mockShardActor);
563 expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
567 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard ending");
571 public void testOnReceiveFindPrimaryWaitForReadyWithCandidateShard() throws Exception {
572 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithCandidateShard starting");
573 new JavaTestKit(getSystem()) {
575 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
577 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
578 shardManager.tell(new ActorInitialized(), mockShardActor);
579 shardManager.tell(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix, null,
580 RaftState.Candidate.name()), mockShardActor);
582 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
584 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
588 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithCandidateShard ending");
592 public void testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard() throws Exception {
593 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard starting");
594 new JavaTestKit(getSystem()) {
596 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
598 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
599 shardManager.tell(new ActorInitialized(), mockShardActor);
600 shardManager.tell(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix, null,
601 RaftState.IsolatedLeader.name()), mockShardActor);
603 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
605 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
609 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard ending");
613 public void testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard() throws Exception {
614 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard starting");
615 new JavaTestKit(getSystem()) {
617 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
619 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
620 shardManager.tell(new ActorInitialized(), mockShardActor);
622 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
624 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
628 LOG.info("testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard ending");
632 public void testOnReceiveFindPrimaryForRemoteShard() throws Exception {
633 LOG.info("testOnReceiveFindPrimaryForRemoteShard starting");
634 String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
636 // Create an ActorSystem ShardManager actor for member-1.
638 final ActorSystem system1 = newActorSystem("Member1");
639 Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
641 final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
642 newTestShardMgrBuilderWithMockShardActor().cluster(
643 new ClusterWrapperImpl(system1)).props().withDispatcher(
644 Dispatchers.DefaultDispatcherId()), shardManagerID);
646 // Create an ActorSystem ShardManager actor for member-2.
648 final ActorSystem system2 = newActorSystem("Member2");
650 Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
652 final ActorRef mockShardActor2 = newMockShardActor(system2, "astronauts", "member-2");
654 MockConfiguration mockConfig2 = new MockConfiguration(
655 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
656 .put("astronauts", Arrays.asList("member-2")).build());
658 final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
659 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
660 new ClusterWrapperImpl(system2)).props().withDispatcher(
661 Dispatchers.DefaultDispatcherId()), shardManagerID);
663 new JavaTestKit(system1) {
665 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
666 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
668 shardManager2.tell(new ActorInitialized(), mockShardActor2);
670 String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
671 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
673 new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
676 new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
679 shardManager1.underlyingActor().waitForMemberUp();
680 shardManager1.tell(new FindPrimary("astronauts", false), getRef());
682 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
683 String path = found.getPrimaryPath();
684 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-astronauts-config"));
685 assertEquals("getPrimaryVersion", leaderVersion, found.getPrimaryVersion());
687 shardManager2.underlyingActor().verifyFindPrimary();
689 Cluster.get(system2).down(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
691 shardManager1.underlyingActor().waitForMemberRemoved();
693 shardManager1.tell(new FindPrimary("astronauts", false), getRef());
695 expectMsgClass(duration("5 seconds"), PrimaryNotFoundException.class);
699 LOG.info("testOnReceiveFindPrimaryForRemoteShard ending");
703 public void testShardAvailabilityOnChangeOfMemberReachability() throws Exception {
704 LOG.info("testShardAvailabilityOnChangeOfMemberReachability starting");
705 String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
707 // Create an ActorSystem ShardManager actor for member-1.
709 final ActorSystem system1 = newActorSystem("Member1");
710 Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
712 final ActorRef mockShardActor1 = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
714 final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
715 newTestShardMgrBuilder().shardActor(mockShardActor1).cluster(
716 new ClusterWrapperImpl(system1)).props().withDispatcher(
717 Dispatchers.DefaultDispatcherId()), shardManagerID);
719 // Create an ActorSystem ShardManager actor for member-2.
721 final ActorSystem system2 = newActorSystem("Member2");
723 Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
725 final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
727 MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
728 .put("default", Arrays.asList("member-1", "member-2")).build());
730 final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
731 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
732 new ClusterWrapperImpl(system2)).props().withDispatcher(
733 Dispatchers.DefaultDispatcherId()), shardManagerID);
735 new JavaTestKit(system1) {
737 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
738 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
739 shardManager1.tell(new ActorInitialized(), mockShardActor1);
740 shardManager2.tell(new ActorInitialized(), mockShardActor2);
742 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
743 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
744 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class),
745 DataStoreVersions.CURRENT_VERSION), mockShardActor1);
747 new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
749 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class),
750 DataStoreVersions.CURRENT_VERSION), mockShardActor2);
752 new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
754 shardManager1.underlyingActor().waitForMemberUp();
756 shardManager1.tell(new FindPrimary("default", true), getRef());
758 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
759 String path = found.getPrimaryPath();
760 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
762 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
763 "akka://cluster-test@127.0.0.1:2558"), getRef());
765 shardManager1.underlyingActor().waitForUnreachableMember();
767 PeerDown peerDown = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
768 assertEquals("getMemberName", MEMBER_2, peerDown.getMemberName());
769 MessageCollectorActor.clearMessages(mockShardActor1);
772 MockClusterWrapper.createMemberRemoved("member-2", "akka://cluster-test@127.0.0.1:2558"),
775 MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
777 shardManager1.tell(new FindPrimary("default", true), getRef());
779 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
782 MockClusterWrapper.createReachableMember("member-2", "akka://cluster-test@127.0.0.1:2558"),
785 shardManager1.underlyingActor().waitForReachableMember();
787 PeerUp peerUp = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
788 assertEquals("getMemberName", MEMBER_2, peerUp.getMemberName());
789 MessageCollectorActor.clearMessages(mockShardActor1);
791 shardManager1.tell(new FindPrimary("default", true), getRef());
793 RemotePrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
794 String path1 = found1.getPrimaryPath();
795 assertTrue("Unexpected primary path " + path1, path1.contains("member-2-shard-default-config"));
798 MockClusterWrapper.createMemberUp("member-2", "akka://cluster-test@127.0.0.1:2558"),
801 MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
803 // Test FindPrimary wait succeeds after reachable member event.
805 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
806 "akka://cluster-test@127.0.0.1:2558"), getRef());
807 shardManager1.underlyingActor().waitForUnreachableMember();
809 shardManager1.tell(new FindPrimary("default", true), getRef());
812 MockClusterWrapper.createReachableMember("member-2", "akka://cluster-test@127.0.0.1:2558"),
815 RemotePrimaryShardFound found2 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
816 String path2 = found2.getPrimaryPath();
817 assertTrue("Unexpected primary path " + path2, path2.contains("member-2-shard-default-config"));
821 LOG.info("testShardAvailabilityOnChangeOfMemberReachability ending");
825 public void testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange() throws Exception {
826 LOG.info("testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange starting");
827 String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
829 // Create an ActorSystem ShardManager actor for member-1.
831 final ActorSystem system1 = newActorSystem("Member1");
832 Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
834 final ActorRef mockShardActor1 = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
836 final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
837 final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
838 newTestShardMgrBuilder().shardActor(mockShardActor1).cluster(new ClusterWrapperImpl(system1))
839 .primaryShardInfoCache(primaryShardInfoCache).props()
840 .withDispatcher(Dispatchers.DefaultDispatcherId()),
843 // Create an ActorSystem ShardManager actor for member-2.
845 final ActorSystem system2 = newActorSystem("Member2");
847 Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
849 final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
851 MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
852 .put("default", Arrays.asList("member-1", "member-2")).build());
854 final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
855 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
856 new ClusterWrapperImpl(system2)).props().withDispatcher(
857 Dispatchers.DefaultDispatcherId()), shardManagerID);
859 new JavaTestKit(system1) {
861 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
862 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
863 shardManager1.tell(new ActorInitialized(), mockShardActor1);
864 shardManager2.tell(new ActorInitialized(), mockShardActor2);
866 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
867 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
868 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class),
869 DataStoreVersions.CURRENT_VERSION), mockShardActor1);
871 new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
873 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class),
874 DataStoreVersions.CURRENT_VERSION), mockShardActor2);
876 new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
878 shardManager1.underlyingActor().waitForMemberUp();
880 shardManager1.tell(new FindPrimary("default", true), getRef());
882 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
883 String path = found.getPrimaryPath();
884 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
886 primaryShardInfoCache.putSuccessful("default", new PrimaryShardInfo(
887 system1.actorSelection(mockShardActor1.path()), DataStoreVersions.CURRENT_VERSION));
889 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
890 "akka://cluster-test@127.0.0.1:2558"), getRef());
892 shardManager1.underlyingActor().waitForUnreachableMember();
894 shardManager1.tell(new FindPrimary("default", true), getRef());
896 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
898 assertNull("Expected primaryShardInfoCache entry removed",
899 primaryShardInfoCache.getIfPresent("default"));
901 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId1, mock(DataTree.class),
902 DataStoreVersions.CURRENT_VERSION), mockShardActor1);
904 new RoleChangeNotification(memberId1, RaftState.Follower.name(), RaftState.Leader.name()),
907 shardManager1.tell(new FindPrimary("default", true), getRef());
909 LocalPrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
910 String path1 = found1.getPrimaryPath();
911 assertTrue("Unexpected primary path " + path1, path1.contains("member-1-shard-default-config"));
916 LOG.info("testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange ending");
921 public void testOnReceiveFindLocalShardForNonExistentShard() throws Exception {
922 new JavaTestKit(getSystem()) {
924 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
926 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
928 shardManager.tell(new FindLocalShard("non-existent", false), getRef());
930 LocalShardNotFound notFound = expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
932 assertEquals("getShardName", "non-existent", notFound.getShardName());
938 public void testOnReceiveFindLocalShardForExistentShard() throws Exception {
939 new JavaTestKit(getSystem()) {
941 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
943 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
944 shardManager.tell(new ActorInitialized(), mockShardActor);
946 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
948 LocalShardFound found = expectMsgClass(duration("5 seconds"), LocalShardFound.class);
950 assertTrue("Found path contains " + found.getPath().path().toString(),
951 found.getPath().path().toString().contains("member-1-shard-default-config"));
957 public void testOnReceiveFindLocalShardForNotInitializedShard() throws Exception {
958 new JavaTestKit(getSystem()) {
960 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
962 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
964 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
970 public void testOnReceiveFindLocalShardWaitForShardInitialized() throws Exception {
971 LOG.info("testOnReceiveFindLocalShardWaitForShardInitialized starting");
972 new JavaTestKit(getSystem()) {
974 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
976 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
978 // We're passing waitUntilInitialized = true to FindLocalShard
979 // so the response should be
980 // delayed until we send ActorInitialized.
981 Future<Object> future = Patterns.ask(shardManager, new FindLocalShard(Shard.DEFAULT_NAME, true),
982 new Timeout(5, TimeUnit.SECONDS));
984 shardManager.tell(new ActorInitialized(), mockShardActor);
986 Object resp = Await.result(future, duration("5 seconds"));
987 assertTrue("Expected: LocalShardFound, Actual: " + resp, resp instanceof LocalShardFound);
991 LOG.info("testOnReceiveFindLocalShardWaitForShardInitialized starting");
995 public void testOnRecoveryJournalIsCleaned() {
996 String persistenceID = "shard-manager-" + shardMrgIDSuffix;
997 InMemoryJournal.addEntry(persistenceID, 1L, new SchemaContextModules(ImmutableSet.of("foo")));
998 InMemoryJournal.addEntry(persistenceID, 2L, new SchemaContextModules(ImmutableSet.of("bar")));
999 InMemoryJournal.addDeleteMessagesCompleteLatch(persistenceID);
1001 newTestShardManager();
1003 InMemoryJournal.waitForDeleteMessagesComplete(persistenceID);
1005 // Journal entries up to the last one should've been deleted
1006 Map<Long, Object> journal = InMemoryJournal.get(persistenceID);
1007 synchronized (journal) {
1008 assertEquals("Journal size", 0, journal.size());
1013 public void testRoleChangeNotificationAndShardLeaderStateChangedReleaseReady() throws Exception {
1014 TestShardManager shardManager = newTestShardManager();
1016 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1017 shardManager.onReceiveCommand(new RoleChangeNotification(
1018 memberId, RaftState.Candidate.name(), RaftState.Leader.name()));
1020 verify(ready, never()).countDown();
1022 shardManager.onReceiveCommand(new ShardLeaderStateChanged(memberId, memberId,
1023 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1025 verify(ready, times(1)).countDown();
1029 public void testRoleChangeNotificationToFollowerWithShardLeaderStateChangedReleaseReady() throws Exception {
1030 new JavaTestKit(getSystem()) {
1032 TestShardManager shardManager = newTestShardManager();
1034 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1035 shardManager.onReceiveCommand(new RoleChangeNotification(memberId, null, RaftState.Follower.name()));
1037 verify(ready, never()).countDown();
1040 .onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
1042 shardManager.onReceiveCommand(
1043 new ShardLeaderStateChanged(memberId, "member-2-shard-default-" + shardMrgIDSuffix,
1044 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1046 verify(ready, times(1)).countDown();
1052 public void testReadyCountDownForMemberUpAfterLeaderStateChanged() throws Exception {
1053 new JavaTestKit(getSystem()) {
1055 TestShardManager shardManager = newTestShardManager();
1057 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1058 shardManager.onReceiveCommand(new RoleChangeNotification(memberId, null, RaftState.Follower.name()));
1060 verify(ready, never()).countDown();
1062 shardManager.onReceiveCommand(
1063 new ShardLeaderStateChanged(memberId, "member-2-shard-default-" + shardMrgIDSuffix,
1064 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1067 .onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
1069 verify(ready, times(1)).countDown();
1075 public void testRoleChangeNotificationDoNothingForUnknownShard() throws Exception {
1076 TestShardManager shardManager = newTestShardManager();
1078 shardManager.onReceiveCommand(new RoleChangeNotification(
1079 "unknown", RaftState.Candidate.name(), RaftState.Leader.name()));
1081 verify(ready, never()).countDown();
1085 public void testByDefaultSyncStatusIsFalse() throws Exception {
1086 TestShardManager shardManager = newTestShardManager();
1088 assertEquals(false, shardManager.getMBean().getSyncStatus());
1092 public void testWhenShardIsLeaderSyncStatusIsTrue() throws Exception {
1093 TestShardManager shardManager = newTestShardManager();
1095 shardManager.onReceiveCommand(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix,
1096 RaftState.Follower.name(), RaftState.Leader.name()));
1098 assertEquals(true, shardManager.getMBean().getSyncStatus());
1102 public void testWhenShardIsCandidateSyncStatusIsFalse() throws Exception {
1103 TestShardManager shardManager = newTestShardManager();
1105 String shardId = "member-1-shard-default-" + shardMrgIDSuffix;
1106 shardManager.onReceiveCommand(new RoleChangeNotification(shardId,
1107 RaftState.Follower.name(), RaftState.Candidate.name()));
1109 assertEquals(false, shardManager.getMBean().getSyncStatus());
1111 // Send a FollowerInitialSyncStatus with status = true for the replica whose current state is candidate
1112 shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(
1115 assertEquals(false, shardManager.getMBean().getSyncStatus());
1119 public void testWhenShardIsFollowerSyncStatusDependsOnFollowerInitialSyncStatus() throws Exception {
1120 TestShardManager shardManager = newTestShardManager();
1122 String shardId = "member-1-shard-default-" + shardMrgIDSuffix;
1123 shardManager.onReceiveCommand(new RoleChangeNotification(shardId,
1124 RaftState.Candidate.name(), RaftState.Follower.name()));
1126 // Initially will be false
1127 assertEquals(false, shardManager.getMBean().getSyncStatus());
1129 // Send status true will make sync status true
1130 shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(true, shardId));
1132 assertEquals(true, shardManager.getMBean().getSyncStatus());
1134 // Send status false will make sync status false
1135 shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(false, shardId));
1137 assertEquals(false, shardManager.getMBean().getSyncStatus());
1142 public void testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards() throws Exception {
1143 LOG.info("testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards starting");
1144 TestShardManager shardManager = newTestShardManager(newShardMgrProps(new MockConfiguration() {
1146 public List<String> getMemberShardNames(MemberName memberName) {
1147 return Arrays.asList("default", "astronauts");
1151 // Initially will be false
1152 assertEquals(false, shardManager.getMBean().getSyncStatus());
1154 // Make default shard leader
1155 String defaultShardId = "member-1-shard-default-" + shardMrgIDSuffix;
1156 shardManager.onReceiveCommand(new RoleChangeNotification(defaultShardId,
1157 RaftState.Follower.name(), RaftState.Leader.name()));
1159 // default = Leader, astronauts is unknown so sync status remains false
1160 assertEquals(false, shardManager.getMBean().getSyncStatus());
1162 // Make astronauts shard leader as well
1163 String astronautsShardId = "member-1-shard-astronauts-" + shardMrgIDSuffix;
1164 shardManager.onReceiveCommand(new RoleChangeNotification(astronautsShardId,
1165 RaftState.Follower.name(), RaftState.Leader.name()));
1167 // Now sync status should be true
1168 assertEquals(true, shardManager.getMBean().getSyncStatus());
1170 // Make astronauts a Follower
1171 shardManager.onReceiveCommand(new RoleChangeNotification(astronautsShardId,
1172 RaftState.Leader.name(), RaftState.Follower.name()));
1174 // Sync status is not true
1175 assertEquals(false, shardManager.getMBean().getSyncStatus());
1177 // Make the astronauts follower sync status true
1178 shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(true, astronautsShardId));
1180 // Sync status is now true
1181 assertEquals(true, shardManager.getMBean().getSyncStatus());
1183 LOG.info("testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards ending");
1187 public void testOnReceiveSwitchShardBehavior() throws Exception {
1188 new JavaTestKit(getSystem()) {
1190 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1192 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1193 shardManager.tell(new ActorInitialized(), mockShardActor);
1195 shardManager.tell(new SwitchShardBehavior(mockShardName, RaftState.Leader, 1000), getRef());
1197 SwitchBehavior switchBehavior = MessageCollectorActor.expectFirstMatching(mockShardActor,
1198 SwitchBehavior.class);
1200 assertEquals(RaftState.Leader, switchBehavior.getNewState());
1201 assertEquals(1000, switchBehavior.getNewTerm());
1206 private static List<MemberName> members(String... names) {
1207 return Arrays.asList(names).stream().map(MemberName::forName).collect(Collectors.toList());
1211 public void testOnCreateShard() {
1212 LOG.info("testOnCreateShard starting");
1213 new JavaTestKit(getSystem()) {
1215 datastoreContextBuilder.shardInitializationTimeout(1, TimeUnit.MINUTES).persistent(true);
1217 ActorRef shardManager = actorFactory
1218 .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1220 SchemaContext schemaContext = TestModel.createTestContext();
1221 shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
1223 DatastoreContext datastoreContext = DatastoreContext.newBuilder().shardElectionTimeoutFactor(100)
1224 .persistent(false).build();
1225 Shard.Builder shardBuilder = Shard.builder();
1227 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1228 "foo", null, members("member-1", "member-5", "member-6"));
1229 shardManager.tell(new CreateShard(config, shardBuilder, datastoreContext), getRef());
1231 expectMsgClass(duration("5 seconds"), Success.class);
1233 shardManager.tell(new FindLocalShard("foo", true), getRef());
1235 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1237 assertEquals("isRecoveryApplicable", false, shardBuilder.getDatastoreContext().isPersistent());
1238 assertTrue("Epxected ShardPeerAddressResolver", shardBuilder.getDatastoreContext().getShardRaftConfig()
1239 .getPeerAddressResolver() instanceof ShardPeerAddressResolver);
1240 assertEquals("peerMembers", Sets.newHashSet(
1241 ShardIdentifier.create("foo", MemberName.forName("member-5"), shardMrgIDSuffix).toString(),
1242 ShardIdentifier.create("foo", MemberName.forName("member-6"), shardMrgIDSuffix).toString()),
1243 shardBuilder.getPeerAddresses().keySet());
1244 assertEquals("ShardIdentifier", ShardIdentifier.create("foo", MEMBER_1, shardMrgIDSuffix),
1245 shardBuilder.getId());
1246 assertSame("schemaContext", schemaContext, shardBuilder.getSchemaContext());
1248 // Send CreateShard with same name - should return Success with
1251 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1253 Success success = expectMsgClass(duration("5 seconds"), Success.class);
1254 assertNotNull("Success status is null", success.status());
1258 LOG.info("testOnCreateShard ending");
1262 public void testOnCreateShardWithLocalMemberNotInShardConfig() {
1263 LOG.info("testOnCreateShardWithLocalMemberNotInShardConfig starting");
1264 new JavaTestKit(getSystem()) {
1266 datastoreContextBuilder.shardInitializationTimeout(1, TimeUnit.MINUTES).persistent(true);
1268 ActorRef shardManager = actorFactory
1269 .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1271 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1273 Shard.Builder shardBuilder = Shard.builder();
1274 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1275 "foo", null, members("member-5", "member-6"));
1277 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1278 expectMsgClass(duration("5 seconds"), Success.class);
1280 shardManager.tell(new FindLocalShard("foo", true), getRef());
1281 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1283 assertEquals("peerMembers size", 0, shardBuilder.getPeerAddresses().size());
1284 assertEquals("schemaContext", DisableElectionsRaftPolicy.class.getName(), shardBuilder
1285 .getDatastoreContext().getShardRaftConfig().getCustomRaftPolicyImplementationClass());
1289 LOG.info("testOnCreateShardWithLocalMemberNotInShardConfig ending");
1293 public void testOnCreateShardWithNoInitialSchemaContext() {
1294 LOG.info("testOnCreateShardWithNoInitialSchemaContext starting");
1295 new JavaTestKit(getSystem()) {
1297 ActorRef shardManager = actorFactory
1298 .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1300 Shard.Builder shardBuilder = Shard.builder();
1302 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1303 "foo", null, members("member-1"));
1304 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1306 expectMsgClass(duration("5 seconds"), Success.class);
1308 SchemaContext schemaContext = TestModel.createTestContext();
1309 shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
1311 shardManager.tell(new FindLocalShard("foo", true), getRef());
1313 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1315 assertSame("schemaContext", schemaContext, shardBuilder.getSchemaContext());
1316 assertNotNull("schemaContext is null", shardBuilder.getDatastoreContext());
1320 LOG.info("testOnCreateShardWithNoInitialSchemaContext ending");
1324 public void testGetSnapshot() throws Exception {
1325 LOG.info("testGetSnapshot starting");
1326 JavaTestKit kit = new JavaTestKit(getSystem());
1328 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1329 .put("shard1", Arrays.asList("member-1")).put("shard2", Arrays.asList("member-1"))
1330 .put("astronauts", Collections.<String>emptyList()).build());
1332 TestActorRef<TestShardManager> shardManager = actorFactory
1333 .createTestActor(newShardMgrProps(mockConfig).withDispatcher(Dispatchers.DefaultDispatcherId()));
1335 shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1336 Failure failure = kit.expectMsgClass(Failure.class);
1337 assertEquals("Failure cause type", IllegalStateException.class, failure.cause().getClass());
1339 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1341 waitForShardInitialized(shardManager, "shard1", kit);
1342 waitForShardInitialized(shardManager, "shard2", kit);
1344 shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1346 DatastoreSnapshot datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1348 assertEquals("getType", shardMrgIDSuffix, datastoreSnapshot.getType());
1349 assertNull("Expected null ShardManagerSnapshot", datastoreSnapshot.getShardManagerSnapshot());
1351 Function<ShardSnapshot, String> shardNameTransformer = s -> s.getName();
1353 assertEquals("Shard names", Sets.newHashSet("shard1", "shard2"), Sets.newHashSet(
1354 Lists.transform(datastoreSnapshot.getShardSnapshots(), shardNameTransformer)));
1356 // Add a new replica
1358 JavaTestKit mockShardLeaderKit = new JavaTestKit(getSystem());
1360 TestShardManager shardManagerInstance = shardManager.underlyingActor();
1361 shardManagerInstance.setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1363 shardManager.tell(new AddShardReplica("astronauts"), kit.getRef());
1364 mockShardLeaderKit.expectMsgClass(AddServer.class);
1365 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.OK, ""));
1366 kit.expectMsgClass(Status.Success.class);
1367 waitForShardInitialized(shardManager, "astronauts", kit);
1369 // Send another GetSnapshot and verify
1371 shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1372 datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1374 assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"), Sets.newHashSet(
1375 Lists.transform(datastoreSnapshot.getShardSnapshots(), shardNameTransformer)));
1377 ShardManagerSnapshot snapshot = datastoreSnapshot.getShardManagerSnapshot();
1378 assertNotNull("Expected ShardManagerSnapshot", snapshot);
1379 assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"),
1380 Sets.newHashSet(snapshot.getShardList()));
1382 LOG.info("testGetSnapshot ending");
1386 public void testRestoreFromSnapshot() throws Exception {
1387 LOG.info("testRestoreFromSnapshot starting");
1389 datastoreContextBuilder.shardInitializationTimeout(3, TimeUnit.SECONDS);
1391 JavaTestKit kit = new JavaTestKit(getSystem());
1393 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1394 .put("shard1", Collections.<String>emptyList()).put("shard2", Collections.<String>emptyList())
1395 .put("astronauts", Collections.<String>emptyList()).build());
1397 ShardManagerSnapshot snapshot =
1398 new ShardManagerSnapshot(Arrays.asList("shard1", "shard2", "astronauts"), Collections.emptyMap());
1399 DatastoreSnapshot restoreFromSnapshot = new DatastoreSnapshot(shardMrgIDSuffix, snapshot,
1400 Collections.<ShardSnapshot>emptyList());
1401 TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(newTestShardMgrBuilder(mockConfig)
1402 .restoreFromSnapshot(restoreFromSnapshot).props().withDispatcher(Dispatchers.DefaultDispatcherId()));
1404 shardManager.underlyingActor().waitForRecoveryComplete();
1406 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1408 waitForShardInitialized(shardManager, "shard1", kit);
1409 waitForShardInitialized(shardManager, "shard2", kit);
1410 waitForShardInitialized(shardManager, "astronauts", kit);
1412 shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1414 DatastoreSnapshot datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1416 assertEquals("getType", shardMrgIDSuffix, datastoreSnapshot.getType());
1418 assertNotNull("Expected ShardManagerSnapshot", datastoreSnapshot.getShardManagerSnapshot());
1419 assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"),
1420 Sets.newHashSet(datastoreSnapshot.getShardManagerSnapshot().getShardList()));
1422 LOG.info("testRestoreFromSnapshot ending");
1426 public void testAddShardReplicaForNonExistentShardConfig() throws Exception {
1427 new JavaTestKit(getSystem()) {
1429 ActorRef shardManager = actorFactory
1430 .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1432 shardManager.tell(new AddShardReplica("model-inventory"), getRef());
1433 Status.Failure resp = expectMsgClass(duration("2 seconds"), Status.Failure.class);
1435 assertEquals("Failure obtained", true, resp.cause() instanceof IllegalArgumentException);
1441 public void testAddShardReplica() throws Exception {
1442 LOG.info("testAddShardReplica starting");
1443 MockConfiguration mockConfig = new MockConfiguration(
1444 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
1445 .put("astronauts", Arrays.asList("member-2")).build());
1447 final String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
1448 datastoreContextBuilder.shardManagerPersistenceId(shardManagerID);
1450 // Create an ActorSystem ShardManager actor for member-1.
1451 final ActorSystem system1 = newActorSystem("Member1");
1452 Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1453 ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
1454 final TestActorRef<TestShardManager> newReplicaShardManager = TestActorRef.create(system1,
1455 newTestShardMgrBuilder(mockConfig).shardActor(mockDefaultShardActor)
1456 .cluster(new ClusterWrapperImpl(system1)).props()
1457 .withDispatcher(Dispatchers.DefaultDispatcherId()),
1460 // Create an ActorSystem ShardManager actor for member-2.
1461 final ActorSystem system2 = newActorSystem("Member2");
1462 Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1464 String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
1465 String name = ShardIdentifier.create("astronauts", MEMBER_2, "config").toString();
1466 final TestActorRef<MockRespondActor> mockShardLeaderActor = TestActorRef.create(system2,
1467 Props.create(MockRespondActor.class, AddServer.class,
1468 new AddServerReply(ServerChangeStatus.OK, memberId2))
1469 .withDispatcher(Dispatchers.DefaultDispatcherId()),
1471 final TestActorRef<TestShardManager> leaderShardManager = TestActorRef.create(system2,
1472 newTestShardMgrBuilder(mockConfig).shardActor(mockShardLeaderActor)
1473 .cluster(new ClusterWrapperImpl(system2)).props()
1474 .withDispatcher(Dispatchers.DefaultDispatcherId()),
1477 new JavaTestKit(system1) {
1479 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1480 leaderShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1482 leaderShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1484 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
1485 leaderShardManager.tell(
1486 new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
1487 mockShardLeaderActor);
1488 leaderShardManager.tell(
1489 new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
1490 mockShardLeaderActor);
1492 newReplicaShardManager.underlyingActor().waitForMemberUp();
1493 leaderShardManager.underlyingActor().waitForMemberUp();
1495 // Have a dummy snapshot to be overwritten by the new data
1497 String[] restoredShards = { "default", "people" };
1498 ShardManagerSnapshot snapshot =
1499 new ShardManagerSnapshot(Arrays.asList(restoredShards), Collections.emptyMap());
1500 InMemorySnapshotStore.addSnapshot(shardManagerID, snapshot);
1501 Uninterruptibles.sleepUninterruptibly(2, TimeUnit.MILLISECONDS);
1503 InMemorySnapshotStore.addSnapshotSavedLatch(shardManagerID);
1504 InMemorySnapshotStore.addSnapshotDeletedLatch(shardManagerID);
1506 // construct a mock response message
1507 newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
1508 AddServer addServerMsg = MessageCollectorActor.expectFirstMatching(mockShardLeaderActor,
1510 String addServerId = "member-1-shard-astronauts-" + shardMrgIDSuffix;
1511 assertEquals("AddServer serverId", addServerId, addServerMsg.getNewServerId());
1512 expectMsgClass(duration("5 seconds"), Status.Success.class);
1514 InMemorySnapshotStore.waitForSavedSnapshot(shardManagerID, ShardManagerSnapshot.class);
1515 InMemorySnapshotStore.waitForDeletedSnapshot(shardManagerID);
1516 List<ShardManagerSnapshot> persistedSnapshots = InMemorySnapshotStore.getSnapshots(shardManagerID,
1517 ShardManagerSnapshot.class);
1518 assertEquals("Number of snapshots persisted", 1, persistedSnapshots.size());
1519 ShardManagerSnapshot shardManagerSnapshot = persistedSnapshots.get(0);
1520 assertEquals("Persisted local shards", Sets.newHashSet("default", "astronauts"),
1521 Sets.newHashSet(shardManagerSnapshot.getShardList()));
1524 LOG.info("testAddShardReplica ending");
1528 public void testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader() throws Exception {
1529 LOG.info("testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader starting");
1530 new JavaTestKit(getSystem()) {
1532 TestActorRef<TestShardManager> shardManager = actorFactory
1533 .createTestActor(newPropsShardMgrWithMockShardActor(), shardMgrID);
1535 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1536 shardManager.tell(new ActorInitialized(), mockShardActor);
1538 String leaderId = "leader-member-shard-default-" + shardMrgIDSuffix;
1539 AddServerReply addServerReply = new AddServerReply(ServerChangeStatus.ALREADY_EXISTS, null);
1540 ActorRef leaderShardActor = shardManager.underlyingActor().getContext()
1541 .actorOf(Props.create(MockRespondActor.class, AddServer.class, addServerReply), leaderId);
1543 MockClusterWrapper.sendMemberUp(shardManager, "leader-member", leaderShardActor.path().toString());
1545 String newReplicaId = "member-1-shard-default-" + shardMrgIDSuffix;
1547 new RoleChangeNotification(newReplicaId, RaftState.Candidate.name(), RaftState.Follower.name()),
1550 new ShardLeaderStateChanged(newReplicaId, leaderId, DataStoreVersions.CURRENT_VERSION),
1553 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1555 MessageCollectorActor.expectFirstMatching(leaderShardActor, AddServer.class);
1557 Failure resp = expectMsgClass(duration("5 seconds"), Failure.class);
1558 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1560 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1561 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1563 // Send message again to verify previous in progress state is
1566 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1567 resp = expectMsgClass(duration("5 seconds"), Failure.class);
1568 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1570 // Send message again with an AddServer timeout to verify the
1571 // pre-existing shard actor isn't terminated.
1574 newDatastoreContextFactory(
1575 datastoreContextBuilder.shardLeaderElectionTimeout(100, TimeUnit.MILLISECONDS).build()),
1577 leaderShardActor.tell(MockRespondActor.CLEAR_RESPONSE, ActorRef.noSender());
1578 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1579 expectMsgClass(duration("5 seconds"), Failure.class);
1581 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1582 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1586 LOG.info("testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader ending");
1590 public void testAddShardReplicaWithPreExistingLocalReplicaLeader() throws Exception {
1591 LOG.info("testAddShardReplicaWithPreExistingLocalReplicaLeader starting");
1592 new JavaTestKit(getSystem()) {
1594 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1595 ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1597 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1598 shardManager.tell(new ActorInitialized(), mockShardActor);
1599 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
1600 DataStoreVersions.CURRENT_VERSION), getRef());
1602 new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
1605 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1606 Failure resp = expectMsgClass(duration("5 seconds"), Failure.class);
1607 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1609 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1610 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1614 LOG.info("testAddShardReplicaWithPreExistingLocalReplicaLeader ending");
1618 public void testAddShardReplicaWithAddServerReplyFailure() throws Exception {
1619 LOG.info("testAddShardReplicaWithAddServerReplyFailure starting");
1620 new JavaTestKit(getSystem()) {
1622 JavaTestKit mockShardLeaderKit = new JavaTestKit(getSystem());
1624 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1625 .put("astronauts", Arrays.asList("member-2")).build());
1627 ActorRef mockNewReplicaShardActor = newMockShardActor(getSystem(), "astronauts", "member-1");
1628 final TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(
1629 newTestShardMgrBuilder(mockConfig).shardActor(mockNewReplicaShardActor).props(), shardMgrID);
1630 shardManager.underlyingActor()
1631 .setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1633 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1635 JavaTestKit terminateWatcher = new JavaTestKit(getSystem());
1636 terminateWatcher.watch(mockNewReplicaShardActor);
1638 shardManager.tell(new AddShardReplica("astronauts"), getRef());
1640 AddServer addServerMsg = mockShardLeaderKit.expectMsgClass(AddServer.class);
1641 assertEquals("AddServer serverId", "member-1-shard-astronauts-" + shardMrgIDSuffix,
1642 addServerMsg.getNewServerId());
1643 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.TIMEOUT, null));
1645 Failure failure = expectMsgClass(duration("5 seconds"), Failure.class);
1646 assertEquals("Failure cause", TimeoutException.class, failure.cause().getClass());
1648 shardManager.tell(new FindLocalShard("astronauts", false), getRef());
1649 expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
1651 terminateWatcher.expectTerminated(mockNewReplicaShardActor);
1653 shardManager.tell(new AddShardReplica("astronauts"), getRef());
1654 mockShardLeaderKit.expectMsgClass(AddServer.class);
1655 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.NO_LEADER, null));
1656 failure = expectMsgClass(duration("5 seconds"), Failure.class);
1657 assertEquals("Failure cause", NoShardLeaderException.class, failure.cause().getClass());
1661 LOG.info("testAddShardReplicaWithAddServerReplyFailure ending");
1665 public void testAddShardReplicaWithAlreadyInProgress() throws Exception {
1666 testServerChangeWhenAlreadyInProgress("astronauts", new AddShardReplica("astronauts"),
1667 AddServer.class, new AddShardReplica("astronauts"));
1671 public void testAddShardReplicaWithFindPrimaryTimeout() throws Exception {
1672 LOG.info("testAddShardReplicaWithFindPrimaryTimeout starting");
1673 datastoreContextBuilder.shardInitializationTimeout(100, TimeUnit.MILLISECONDS);
1674 new JavaTestKit(getSystem()) {
1676 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1677 .put("astronauts", Arrays.asList("member-2")).build());
1679 final ActorRef newReplicaShardManager = actorFactory
1680 .createActor(newTestShardMgrBuilder(mockConfig).shardActor(mockShardActor).props(), shardMgrID);
1682 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1683 MockClusterWrapper.sendMemberUp(newReplicaShardManager, "member-2",
1684 AddressFromURIString.parse("akka://non-existent@127.0.0.1:5").toString());
1686 newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
1687 Status.Failure resp = expectMsgClass(duration("5 seconds"), Status.Failure.class);
1688 assertEquals("Failure obtained", true, resp.cause() instanceof RuntimeException);
1692 LOG.info("testAddShardReplicaWithFindPrimaryTimeout ending");
1696 public void testRemoveShardReplicaForNonExistentShard() throws Exception {
1697 new JavaTestKit(getSystem()) {
1699 ActorRef shardManager = actorFactory
1700 .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1702 shardManager.tell(new RemoveShardReplica("model-inventory", MEMBER_1), getRef());
1703 Status.Failure resp = expectMsgClass(duration("10 seconds"), Status.Failure.class);
1704 assertEquals("Failure obtained", true, resp.cause() instanceof PrimaryNotFoundException);
1713 public void testRemoveShardReplicaLocal() throws Exception {
1714 new JavaTestKit(getSystem()) {
1716 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1718 final TestActorRef<MockRespondActor> respondActor = actorFactory
1719 .createTestActor(Props.create(MockRespondActor.class, RemoveServer.class,
1720 new RemoveServerReply(ServerChangeStatus.OK, null)), memberId);
1722 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
1724 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1725 shardManager.tell(new ActorInitialized(), respondActor);
1726 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
1727 DataStoreVersions.CURRENT_VERSION), getRef());
1729 new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
1732 shardManager.tell(new RemoveShardReplica(Shard.DEFAULT_NAME, MEMBER_1), getRef());
1733 final RemoveServer removeServer = MessageCollectorActor.expectFirstMatching(respondActor,
1734 RemoveServer.class);
1735 assertEquals(ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString(),
1736 removeServer.getServerId());
1737 expectMsgClass(duration("5 seconds"), Success.class);
1743 public void testRemoveShardReplicaRemote() throws Exception {
1744 MockConfiguration mockConfig = new MockConfiguration(
1745 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
1746 .put("astronauts", Arrays.asList("member-1")).build());
1748 String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
1750 // Create an ActorSystem ShardManager actor for member-1.
1751 final ActorSystem system1 = newActorSystem("Member1");
1752 Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1753 ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
1755 final TestActorRef<TestShardManager> newReplicaShardManager = TestActorRef.create(system1,
1756 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockDefaultShardActor).cluster(
1757 new ClusterWrapperImpl(system1)).props().withDispatcher(Dispatchers.DefaultDispatcherId()),
1760 // Create an ActorSystem ShardManager actor for member-2.
1761 final ActorSystem system2 = newActorSystem("Member2");
1762 Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1764 String name = ShardIdentifier.create("default", MEMBER_2, shardMrgIDSuffix).toString();
1765 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
1766 final TestActorRef<MockRespondActor> mockShardLeaderActor =
1767 TestActorRef.create(system2, Props.create(MockRespondActor.class, RemoveServer.class,
1768 new RemoveServerReply(ServerChangeStatus.OK, memberId2)), name);
1770 LOG.error("Mock Shard Leader Actor : {}", mockShardLeaderActor);
1772 final TestActorRef<TestShardManager> leaderShardManager = TestActorRef.create(system2,
1773 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockShardLeaderActor).cluster(
1774 new ClusterWrapperImpl(system2)).props().withDispatcher(Dispatchers.DefaultDispatcherId()),
1777 // Because mockShardLeaderActor is created at the top level of the actor system it has an address like so,
1778 // akka://cluster-test@127.0.0.1:2559/user/member-2-shard-default-config1
1779 // However when a shard manager has a local shard which is a follower and a leader that is remote it will
1780 // try to compute an address for the remote shard leader using the ShardPeerAddressResolver. This address will
1782 // akka://cluster-test@127.0.0.1:2559/user/shardmanager-config1/member-2-shard-default-config1
1783 // In this specific case if we did a FindPrimary for shard default from member-1 we would come up
1784 // with the address of an actor which does not exist, therefore any message sent to that actor would go to
1786 // To work around this problem we create a ForwardingActor with the right address and pass to it the
1787 // mockShardLeaderActor. The ForwardingActor simply forwards all messages to the mockShardLeaderActor and every
1788 // thing works as expected
1789 final ActorRef actorRef = leaderShardManager.underlyingActor().context()
1790 .actorOf(Props.create(ForwardingActor.class, mockShardLeaderActor),
1791 "member-2-shard-default-" + shardMrgIDSuffix);
1793 LOG.error("Forwarding actor : {}", actorRef);
1795 new JavaTestKit(system1) {
1797 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1798 leaderShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1800 leaderShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1801 newReplicaShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1803 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
1804 leaderShardManager.tell(
1805 new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
1806 mockShardLeaderActor);
1807 leaderShardManager.tell(
1808 new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
1809 mockShardLeaderActor);
1811 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
1812 newReplicaShardManager.tell(
1813 new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class), leaderVersion),
1815 newReplicaShardManager.tell(
1816 new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
1819 newReplicaShardManager.underlyingActor().waitForMemberUp();
1820 leaderShardManager.underlyingActor().waitForMemberUp();
1822 // construct a mock response message
1823 newReplicaShardManager.tell(new RemoveShardReplica("default", MEMBER_1), getRef());
1824 RemoveServer removeServer = MessageCollectorActor.expectFirstMatching(mockShardLeaderActor,
1825 RemoveServer.class);
1826 String removeServerId = ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString();
1827 assertEquals("RemoveServer serverId", removeServerId, removeServer.getServerId());
1828 expectMsgClass(duration("5 seconds"), Status.Success.class);
1834 public void testRemoveShardReplicaWhenAnotherRemoveShardReplicaAlreadyInProgress() throws Exception {
1835 testServerChangeWhenAlreadyInProgress("astronauts", new RemoveShardReplica("astronauts", MEMBER_2),
1836 RemoveServer.class, new RemoveShardReplica("astronauts", MEMBER_3));
1840 public void testRemoveShardReplicaWhenAddShardReplicaAlreadyInProgress() throws Exception {
1841 testServerChangeWhenAlreadyInProgress("astronauts", new AddShardReplica("astronauts"),
1842 AddServer.class, new RemoveShardReplica("astronauts", MEMBER_2));
1846 public void testServerChangeWhenAlreadyInProgress(final String shardName, final Object firstServerChange,
1847 final Class<?> firstForwardedServerChangeClass,
1848 final Object secondServerChange) throws Exception {
1849 new JavaTestKit(getSystem()) {
1851 JavaTestKit mockShardLeaderKit = new JavaTestKit(getSystem());
1852 final JavaTestKit secondRequestKit = new JavaTestKit(getSystem());
1854 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1855 .put(shardName, Arrays.asList("member-2")).build());
1857 final TestActorRef<TestShardManager> shardManager = TestActorRef.create(getSystem(),
1858 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockShardActor)
1859 .cluster(new MockClusterWrapper()).props()
1860 .withDispatcher(Dispatchers.DefaultDispatcherId()),
1863 shardManager.underlyingActor()
1864 .setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1866 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1868 shardManager.tell(firstServerChange, getRef());
1870 mockShardLeaderKit.expectMsgClass(firstForwardedServerChangeClass);
1872 shardManager.tell(secondServerChange, secondRequestKit.getRef());
1874 secondRequestKit.expectMsgClass(duration("5 seconds"), Failure.class);
1880 public void testServerRemovedShardActorNotRunning() throws Exception {
1881 LOG.info("testServerRemovedShardActorNotRunning starting");
1882 new JavaTestKit(getSystem()) {
1884 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1885 .put("default", Arrays.asList("member-1", "member-2"))
1886 .put("astronauts", Arrays.asList("member-2"))
1887 .put("people", Arrays.asList("member-1", "member-2")).build());
1889 TestActorRef<TestShardManager> shardManager = actorFactory
1890 .createTestActor(newShardMgrProps(mockConfig));
1892 shardManager.underlyingActor().waitForRecoveryComplete();
1893 shardManager.tell(new FindLocalShard("people", false), getRef());
1894 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1896 shardManager.tell(new FindLocalShard("default", false), getRef());
1897 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1899 // Removed the default shard replica from member-1
1900 ShardIdentifier.Builder builder = new ShardIdentifier.Builder();
1901 ShardIdentifier shardId = builder.shardName("default").memberName(MEMBER_1).type(shardMrgIDSuffix)
1903 shardManager.tell(new ServerRemoved(shardId.toString()), getRef());
1905 shardManager.underlyingActor().verifySnapshotPersisted(Sets.newHashSet("people"));
1909 LOG.info("testServerRemovedShardActorNotRunning ending");
1913 public void testServerRemovedShardActorRunning() throws Exception {
1914 LOG.info("testServerRemovedShardActorRunning starting");
1915 new JavaTestKit(getSystem()) {
1917 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1918 .put("default", Arrays.asList("member-1", "member-2"))
1919 .put("astronauts", Arrays.asList("member-2"))
1920 .put("people", Arrays.asList("member-1", "member-2")).build());
1922 String shardId = ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString();
1923 TestActorRef<MessageCollectorActor> shard = actorFactory.createTestActor(MessageCollectorActor.props(),
1926 TestActorRef<TestShardManager> shardManager = actorFactory
1927 .createTestActor(newTestShardMgrBuilder(mockConfig).addShardActor("default", shard).props());
1929 shardManager.underlyingActor().waitForRecoveryComplete();
1931 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1932 shardManager.tell(new ActorInitialized(), shard);
1934 waitForShardInitialized(shardManager, "people", this);
1935 waitForShardInitialized(shardManager, "default", this);
1937 // Removed the default shard replica from member-1
1938 shardManager.tell(new ServerRemoved(shardId), getRef());
1940 shardManager.underlyingActor().verifySnapshotPersisted(Sets.newHashSet("people"));
1942 MessageCollectorActor.expectFirstMatching(shard, Shutdown.class);
1946 LOG.info("testServerRemovedShardActorRunning ending");
1950 public void testShardPersistenceWithRestoredData() throws Exception {
1951 LOG.info("testShardPersistenceWithRestoredData starting");
1952 new JavaTestKit(getSystem()) {
1954 MockConfiguration mockConfig =
1955 new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1956 .put("default", Arrays.asList("member-1", "member-2"))
1957 .put("astronauts", Arrays.asList("member-2"))
1958 .put("people", Arrays.asList("member-1", "member-2")).build());
1959 String[] restoredShards = {"default", "astronauts"};
1960 ShardManagerSnapshot snapshot =
1961 new ShardManagerSnapshot(Arrays.asList(restoredShards), Collections.emptyMap());
1962 InMemorySnapshotStore.addSnapshot("shard-manager-" + shardMrgIDSuffix, snapshot);
1964 // create shardManager to come up with restored data
1965 TestActorRef<TestShardManager> newRestoredShardManager = actorFactory
1966 .createTestActor(newShardMgrProps(mockConfig));
1968 newRestoredShardManager.underlyingActor().waitForRecoveryComplete();
1970 newRestoredShardManager.tell(new FindLocalShard("people", false), getRef());
1971 LocalShardNotFound notFound = expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
1972 assertEquals("for uninitialized shard", "people", notFound.getShardName());
1974 // Verify a local shard is created for the restored shards,
1975 // although we expect a NotInitializedException for the shards
1976 // as the actor initialization
1977 // message is not sent for them
1978 newRestoredShardManager.tell(new FindLocalShard("default", false), getRef());
1979 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1981 newRestoredShardManager.tell(new FindLocalShard("astronauts", false), getRef());
1982 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1986 LOG.info("testShardPersistenceWithRestoredData ending");
1990 public void testShutDown() throws Exception {
1991 LOG.info("testShutDown starting");
1992 new JavaTestKit(getSystem()) {
1994 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1995 .put("shard1", Arrays.asList("member-1")).put("shard2", Arrays.asList("member-1")).build());
1997 String shardId1 = ShardIdentifier.create("shard1", MEMBER_1, shardMrgIDSuffix).toString();
1998 TestActorRef<MessageCollectorActor> shard1 = actorFactory.createTestActor(
1999 MessageCollectorActor.props().withDispatcher(Dispatchers.DefaultDispatcherId()), shardId1);
2001 String shardId2 = ShardIdentifier.create("shard2", MEMBER_1, shardMrgIDSuffix).toString();
2002 TestActorRef<MessageCollectorActor> shard2 = actorFactory.createTestActor(
2003 MessageCollectorActor.props().withDispatcher(Dispatchers.DefaultDispatcherId()), shardId2);
2005 TestActorRef<TestShardManager> shardManager = actorFactory
2006 .createTestActor(newTestShardMgrBuilder(mockConfig).addShardActor("shard1", shard1)
2007 .addShardActor("shard2", shard2).props()
2008 .withDispatcher(Dispatchers.DefaultDispatcherId()));
2010 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
2011 shardManager.tell(new ActorInitialized(), shard1);
2012 shardManager.tell(new ActorInitialized(), shard2);
2014 FiniteDuration duration = FiniteDuration.create(5, TimeUnit.SECONDS);
2015 Future<Boolean> stopFuture = Patterns.gracefulStop(shardManager, duration, Shutdown.INSTANCE);
2017 MessageCollectorActor.expectFirstMatching(shard1, Shutdown.class);
2018 MessageCollectorActor.expectFirstMatching(shard2, Shutdown.class);
2021 Await.ready(stopFuture, FiniteDuration.create(500, TimeUnit.MILLISECONDS));
2022 fail("ShardManager actor stopped without waiting for the Shards to be stopped");
2023 } catch (TimeoutException e) {
2027 actorFactory.killActor(shard1, this);
2028 actorFactory.killActor(shard2, this);
2030 Boolean stopped = Await.result(stopFuture, duration);
2031 assertEquals("Stopped", Boolean.TRUE, stopped);
2035 LOG.info("testShutDown ending");
2039 public void testChangeServersVotingStatus() throws Exception {
2040 new JavaTestKit(getSystem()) {
2042 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
2044 TestActorRef<MockRespondActor> respondActor = actorFactory
2045 .createTestActor(Props.create(MockRespondActor.class, ChangeServersVotingStatus.class,
2046 new ServerChangeReply(ServerChangeStatus.OK, null)), memberId);
2048 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
2050 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
2051 shardManager.tell(new ActorInitialized(), respondActor);
2052 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
2053 DataStoreVersions.CURRENT_VERSION), getRef());
2055 new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
2059 new ChangeShardMembersVotingStatus("default", ImmutableMap.of("member-2", Boolean.TRUE)),
2062 ChangeServersVotingStatus actualChangeStatusMsg = MessageCollectorActor
2063 .expectFirstMatching(respondActor, ChangeServersVotingStatus.class);
2064 assertEquals("ChangeServersVotingStatus map", actualChangeStatusMsg.getServerVotingStatusMap(),
2065 ImmutableMap.of(ShardIdentifier
2066 .create("default", MemberName.forName("member-2"), shardMrgIDSuffix).toString(),
2069 expectMsgClass(duration("5 seconds"), Success.class);
2075 public void testChangeServersVotingStatusWithNoLeader() throws Exception {
2076 new JavaTestKit(getSystem()) {
2078 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
2080 TestActorRef<MockRespondActor> respondActor = actorFactory
2081 .createTestActor(Props.create(MockRespondActor.class, ChangeServersVotingStatus.class,
2082 new ServerChangeReply(ServerChangeStatus.NO_LEADER, null)), memberId);
2084 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
2086 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
2087 shardManager.tell(new ActorInitialized(), respondActor);
2088 shardManager.tell(new RoleChangeNotification(memberId, null, RaftState.Follower.name()), respondActor);
2091 new ChangeShardMembersVotingStatus("default", ImmutableMap.of("member-2", Boolean.TRUE)),
2094 MessageCollectorActor.expectFirstMatching(respondActor, ChangeServersVotingStatus.class);
2096 Status.Failure resp = expectMsgClass(duration("5 seconds"), Status.Failure.class);
2097 assertEquals("Failure resposnse", true, resp.cause() instanceof NoShardLeaderException);
2102 public static class TestShardManager extends ShardManager {
2103 private final CountDownLatch recoveryComplete = new CountDownLatch(1);
2104 private final CountDownLatch snapshotPersist = new CountDownLatch(1);
2105 private ShardManagerSnapshot snapshot;
2106 private final Map<String, ActorRef> shardActors;
2107 private final ActorRef shardActor;
2108 private CountDownLatch findPrimaryMessageReceived = new CountDownLatch(1);
2109 private CountDownLatch memberUpReceived = new CountDownLatch(1);
2110 private CountDownLatch memberRemovedReceived = new CountDownLatch(1);
2111 private CountDownLatch memberUnreachableReceived = new CountDownLatch(1);
2112 private CountDownLatch memberReachableReceived = new CountDownLatch(1);
2113 private volatile MessageInterceptor messageInterceptor;
2115 private TestShardManager(Builder builder) {
2117 shardActor = builder.shardActor;
2118 shardActors = builder.shardActors;
2122 protected void handleRecover(Object message) throws Exception {
2124 super.handleRecover(message);
2126 if (message instanceof RecoveryCompleted) {
2127 recoveryComplete.countDown();
2132 private void countDownIfOther(final Member member, CountDownLatch latch) {
2133 if (!getCluster().getCurrentMemberName().equals(memberToName(member))) {
2139 public void handleCommand(Object message) throws Exception {
2141 if (messageInterceptor != null && messageInterceptor.canIntercept(message)) {
2142 getSender().tell(messageInterceptor.apply(message), getSelf());
2144 super.handleCommand(message);
2147 if (message instanceof FindPrimary) {
2148 findPrimaryMessageReceived.countDown();
2149 } else if (message instanceof ClusterEvent.MemberUp) {
2150 countDownIfOther(((ClusterEvent.MemberUp) message).member(), memberUpReceived);
2151 } else if (message instanceof ClusterEvent.MemberRemoved) {
2152 countDownIfOther(((ClusterEvent.MemberRemoved) message).member(), memberRemovedReceived);
2153 } else if (message instanceof ClusterEvent.UnreachableMember) {
2154 countDownIfOther(((ClusterEvent.UnreachableMember) message).member(), memberUnreachableReceived);
2155 } else if (message instanceof ClusterEvent.ReachableMember) {
2156 countDownIfOther(((ClusterEvent.ReachableMember) message).member(), memberReachableReceived);
2161 void setMessageInterceptor(MessageInterceptor messageInterceptor) {
2162 this.messageInterceptor = messageInterceptor;
2165 void waitForRecoveryComplete() {
2166 assertEquals("Recovery complete", true,
2167 Uninterruptibles.awaitUninterruptibly(recoveryComplete, 5, TimeUnit.SECONDS));
2170 public void waitForMemberUp() {
2171 assertEquals("MemberUp received", true,
2172 Uninterruptibles.awaitUninterruptibly(memberUpReceived, 5, TimeUnit.SECONDS));
2173 memberUpReceived = new CountDownLatch(1);
2176 void waitForMemberRemoved() {
2177 assertEquals("MemberRemoved received", true,
2178 Uninterruptibles.awaitUninterruptibly(memberRemovedReceived, 5, TimeUnit.SECONDS));
2179 memberRemovedReceived = new CountDownLatch(1);
2182 void waitForUnreachableMember() {
2183 assertEquals("UnreachableMember received", true,
2184 Uninterruptibles.awaitUninterruptibly(memberUnreachableReceived, 5, TimeUnit.SECONDS
2186 memberUnreachableReceived = new CountDownLatch(1);
2189 void waitForReachableMember() {
2190 assertEquals("ReachableMember received", true,
2191 Uninterruptibles.awaitUninterruptibly(memberReachableReceived, 5, TimeUnit.SECONDS));
2192 memberReachableReceived = new CountDownLatch(1);
2195 void verifyFindPrimary() {
2196 assertEquals("FindPrimary received", true,
2197 Uninterruptibles.awaitUninterruptibly(findPrimaryMessageReceived, 5, TimeUnit.SECONDS));
2198 findPrimaryMessageReceived = new CountDownLatch(1);
2201 public static Builder builder(DatastoreContext.Builder datastoreContextBuilder) {
2202 return new Builder(datastoreContextBuilder);
2205 public static class Builder extends AbstractGenericCreator<Builder, TestShardManager> {
2206 private ActorRef shardActor;
2207 private final Map<String, ActorRef> shardActors = new HashMap<>();
2209 Builder(DatastoreContext.Builder datastoreContextBuilder) {
2210 super(TestShardManager.class);
2211 datastoreContextFactory(newDatastoreContextFactory(datastoreContextBuilder.build()));
2214 Builder shardActor(ActorRef newShardActor) {
2215 this.shardActor = newShardActor;
2219 Builder addShardActor(String shardName, ActorRef actorRef) {
2220 shardActors.put(shardName, actorRef);
2226 public void saveSnapshot(Object obj) {
2227 snapshot = (ShardManagerSnapshot) obj;
2228 snapshotPersist.countDown();
2229 super.saveSnapshot(obj);
2232 void verifySnapshotPersisted(Set<String> shardList) {
2233 assertEquals("saveSnapshot invoked", true,
2234 Uninterruptibles.awaitUninterruptibly(snapshotPersist, 5, TimeUnit.SECONDS));
2235 assertEquals("Shard Persisted", shardList, Sets.newHashSet(snapshot.getShardList()));
2239 protected ActorRef newShardActor(SchemaContext schemaContext, ShardInformation info) {
2240 if (shardActors.get(info.getShardName()) != null) {
2241 return shardActors.get(info.getShardName());
2244 if (shardActor != null) {
2248 return super.newShardActor(schemaContext, info);
2252 private abstract static class AbstractGenericCreator<T extends AbstractGenericCreator<T, ?>, C extends ShardManager>
2253 extends AbstractShardManagerCreator<T> {
2254 private final Class<C> shardManagerClass;
2256 AbstractGenericCreator(Class<C> shardManagerClass) {
2257 this.shardManagerClass = shardManagerClass;
2258 cluster(new MockClusterWrapper()).configuration(new MockConfiguration()).waitTillReadyCountDownLatch(ready)
2259 .primaryShardInfoCache(new PrimaryShardInfoFutureCache());
2263 public Props props() {
2265 return Props.create(shardManagerClass, this);
2269 private static class GenericCreator<C extends ShardManager> extends AbstractGenericCreator<GenericCreator<C>, C> {
2270 GenericCreator(Class<C> shardManagerClass) {
2271 super(shardManagerClass);
2275 private static class DelegatingShardManagerCreator implements Creator<ShardManager> {
2276 private static final long serialVersionUID = 1L;
2277 private final Creator<ShardManager> delegate;
2279 DelegatingShardManagerCreator(Creator<ShardManager> delegate) {
2280 this.delegate = delegate;
2284 public ShardManager create() throws Exception {
2285 return delegate.create();
2289 interface MessageInterceptor extends Function<Object, Object> {
2290 boolean canIntercept(Object message);
2293 private static MessageInterceptor newFindPrimaryInterceptor(final ActorRef primaryActor) {
2294 return new MessageInterceptor() {
2296 public Object apply(Object message) {
2297 return new RemotePrimaryShardFound(Serialization.serializedActorPath(primaryActor), (short) 1);
2301 public boolean canIntercept(Object message) {
2302 return message instanceof FindPrimary;
2307 private static class MockRespondActor extends MessageCollectorActor {
2308 static final String CLEAR_RESPONSE = "clear-response";
2310 private Object responseMsg;
2311 private final Class<?> requestClass;
2313 @SuppressWarnings("unused")
2314 MockRespondActor(Class<?> requestClass, Object responseMsg) {
2315 this.requestClass = requestClass;
2316 this.responseMsg = responseMsg;
2320 public void onReceive(Object message) throws Exception {
2321 if (message.equals(CLEAR_RESPONSE)) {
2324 super.onReceive(message);
2325 if (message.getClass().equals(requestClass) && responseMsg != null) {
2326 getSender().tell(responseMsg, getSelf());