opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/shardmanager/ShardManagerTest.java

   1 /*
   2  * Copyright (c) 2014, 2015 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.controller.cluster.datastore.shardmanager;
  10
  11 import static org.junit.Assert.assertEquals;
  12 import static org.junit.Assert.assertNotNull;
  13 import static org.junit.Assert.assertNull;
  14 import static org.junit.Assert.assertSame;
  15 import static org.junit.Assert.assertTrue;
  16 import static org.junit.Assert.fail;
  17 import static org.mockito.Mockito.mock;
  18 import static org.mockito.Mockito.never;
  19 import static org.mockito.Mockito.times;
  20 import static org.mockito.Mockito.verify;
  21
  22 import akka.actor.ActorRef;
  23 import akka.actor.ActorSystem;
  24 import akka.actor.AddressFromURIString;
  25 import akka.actor.Props;
  26 import akka.actor.Status;
  27 import akka.actor.Status.Failure;
  28 import akka.actor.Status.Success;
  29 import akka.cluster.Cluster;
  30 import akka.cluster.ClusterEvent;
  31 import akka.cluster.Member;
  32 import akka.dispatch.Dispatchers;
  33 import akka.japi.Creator;
  34 import akka.pattern.Patterns;
  35 import akka.persistence.RecoveryCompleted;
  36 import akka.serialization.Serialization;
  37 import akka.testkit.JavaTestKit;
  38 import akka.testkit.TestActorRef;
  39 import akka.util.Timeout;
  40 import com.google.common.base.Function;
  41 import com.google.common.base.Stopwatch;
  42 import com.google.common.collect.ImmutableMap;
  43 import com.google.common.collect.Lists;
  44 import com.google.common.collect.Sets;
  45 import com.google.common.util.concurrent.Uninterruptibles;
  46 import java.net.URI;
  47 import java.util.AbstractMap;
  48 import java.util.Arrays;
  49 import java.util.Collection;
  50 import java.util.Collections;
  51 import java.util.HashMap;
  52 import java.util.List;
  53 import java.util.Map;
  54 import java.util.Map.Entry;
  55 import java.util.Set;
  56 import java.util.concurrent.CountDownLatch;
  57 import java.util.concurrent.TimeUnit;
  58 import java.util.concurrent.TimeoutException;
  59 import java.util.stream.Collectors;
  60 import org.junit.Test;
  61 import org.mockito.Mockito;
  62 import org.opendaylight.controller.cluster.access.concepts.MemberName;
  63 import org.opendaylight.controller.cluster.datastore.AbstractShardManagerTest;
  64 import org.opendaylight.controller.cluster.datastore.ClusterWrapperImpl;
  65 import org.opendaylight.controller.cluster.datastore.DataStoreVersions;
  66 import org.opendaylight.controller.cluster.datastore.DatastoreContext;
  67 import org.opendaylight.controller.cluster.datastore.DatastoreContextFactory;
  68 import org.opendaylight.controller.cluster.datastore.DistributedDataStore;
  69 import org.opendaylight.controller.cluster.datastore.Shard;
  70 import org.opendaylight.controller.cluster.datastore.config.ConfigurationImpl;
  71 import org.opendaylight.controller.cluster.datastore.config.EmptyModuleShardConfigProvider;
  72 import org.opendaylight.controller.cluster.datastore.config.ModuleShardConfiguration;
  73 import org.opendaylight.controller.cluster.datastore.exceptions.AlreadyExistsException;
  74 import org.opendaylight.controller.cluster.datastore.exceptions.NoShardLeaderException;
  75 import org.opendaylight.controller.cluster.datastore.exceptions.NotInitializedException;
  76 import org.opendaylight.controller.cluster.datastore.exceptions.PrimaryNotFoundException;
  77 import org.opendaylight.controller.cluster.datastore.identifiers.ShardIdentifier;
  78 import org.opendaylight.controller.cluster.datastore.identifiers.ShardManagerIdentifier;
  79 import org.opendaylight.controller.cluster.datastore.messages.ActorInitialized;
  80 import org.opendaylight.controller.cluster.datastore.messages.AddShardReplica;
  81 import org.opendaylight.controller.cluster.datastore.messages.ChangeShardMembersVotingStatus;
  82 import org.opendaylight.controller.cluster.datastore.messages.CreateShard;
  83 import org.opendaylight.controller.cluster.datastore.messages.FindLocalShard;
  84 import org.opendaylight.controller.cluster.datastore.messages.FindPrimary;
  85 import org.opendaylight.controller.cluster.datastore.messages.LocalPrimaryShardFound;
  86 import org.opendaylight.controller.cluster.datastore.messages.LocalShardFound;
  87 import org.opendaylight.controller.cluster.datastore.messages.LocalShardNotFound;
  88 import org.opendaylight.controller.cluster.datastore.messages.PeerDown;
  89 import org.opendaylight.controller.cluster.datastore.messages.PeerUp;
  90 import org.opendaylight.controller.cluster.datastore.messages.PrimaryShardInfo;
  91 import org.opendaylight.controller.cluster.datastore.messages.RemotePrimaryShardFound;
  92 import org.opendaylight.controller.cluster.datastore.messages.RemoveShardReplica;
  93 import org.opendaylight.controller.cluster.datastore.messages.ShardLeaderStateChanged;
  94 import org.opendaylight.controller.cluster.datastore.messages.UpdateSchemaContext;
  95 import org.opendaylight.controller.cluster.datastore.persisted.DatastoreSnapshot;
  96 import org.opendaylight.controller.cluster.datastore.persisted.DatastoreSnapshot.ShardSnapshot;
  97 import org.opendaylight.controller.cluster.datastore.persisted.ShardManagerSnapshot;
  98 import org.opendaylight.controller.cluster.datastore.utils.ForwardingActor;
  99 import org.opendaylight.controller.cluster.datastore.utils.MockClusterWrapper;
 100 import org.opendaylight.controller.cluster.datastore.utils.MockConfiguration;
 101 import org.opendaylight.controller.cluster.datastore.utils.PrimaryShardInfoFutureCache;
 102 import org.opendaylight.controller.cluster.notifications.LeaderStateChanged;
 103 import org.opendaylight.controller.cluster.notifications.RegisterRoleChangeListener;
 104 import org.opendaylight.controller.cluster.notifications.RoleChangeNotification;
 105 import org.opendaylight.controller.cluster.raft.RaftState;
 106 import org.opendaylight.controller.cluster.raft.base.messages.FollowerInitialSyncUpStatus;
 107 import org.opendaylight.controller.cluster.raft.base.messages.SwitchBehavior;
 108 import org.opendaylight.controller.cluster.raft.client.messages.GetSnapshot;
 109 import org.opendaylight.controller.cluster.raft.client.messages.Shutdown;
 110 import org.opendaylight.controller.cluster.raft.messages.AddServer;
 111 import org.opendaylight.controller.cluster.raft.messages.AddServerReply;
 112 import org.opendaylight.controller.cluster.raft.messages.ChangeServersVotingStatus;
 113 import org.opendaylight.controller.cluster.raft.messages.RemoveServer;
 114 import org.opendaylight.controller.cluster.raft.messages.RemoveServerReply;
 115 import org.opendaylight.controller.cluster.raft.messages.ServerChangeReply;
 116 import org.opendaylight.controller.cluster.raft.messages.ServerChangeStatus;
 117 import org.opendaylight.controller.cluster.raft.messages.ServerRemoved;
 118 import org.opendaylight.controller.cluster.raft.policy.DisableElectionsRaftPolicy;
 119 import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore;
 120 import org.opendaylight.controller.cluster.raft.utils.MessageCollectorActor;
 121 import org.opendaylight.controller.md.cluster.datastore.model.TestModel;
 122 import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTree;
 123 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
 124 import org.slf4j.Logger;
 125 import org.slf4j.LoggerFactory;
 126 import scala.concurrent.Await;
 127 import scala.concurrent.Future;
 128 import scala.concurrent.duration.FiniteDuration;
 129
 130 public class ShardManagerTest extends AbstractShardManagerTest {
 131     private static final Logger LOG = LoggerFactory.getLogger(ShardManagerTest.class);
 132     private static final MemberName MEMBER_2 = MemberName.forName("member-2");
 133     private static final MemberName MEMBER_3 = MemberName.forName("member-3");
 134
 135     private final String shardMgrID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
 136
 137     private ActorSystem newActorSystem(final String config) {
 138         return newActorSystem("cluster-test", config);
 139     }
 140
 141     private ActorRef newMockShardActor(final ActorSystem system, final String shardName, final String memberName) {
 142         String name = ShardIdentifier.create(shardName, MemberName.forName(memberName), "config").toString();
 143         if (system == getSystem()) {
 144             return actorFactory.createActor(MessageCollectorActor.props(), name);
 145         }
 146
 147         return system.actorOf(MessageCollectorActor.props(), name);
 148     }
 149
 150     private Props newShardMgrProps() {
 151         return newShardMgrProps(new MockConfiguration());
 152     }
 153
 154     private static DatastoreContextFactory newDatastoreContextFactory(final DatastoreContext datastoreContext) {
 155         DatastoreContextFactory mockFactory = mock(DatastoreContextFactory.class);
 156         Mockito.doReturn(datastoreContext).when(mockFactory).getBaseDatastoreContext();
 157         Mockito.doReturn(datastoreContext).when(mockFactory).getShardDatastoreContext(Mockito.anyString());
 158         return mockFactory;
 159     }
 160
 161     private TestShardManager.Builder newTestShardMgrBuilderWithMockShardActor() {
 162         return newTestShardMgrBuilderWithMockShardActor(mockShardActor);
 163     }
 164
 165     private TestShardManager.Builder newTestShardMgrBuilderWithMockShardActor(final ActorRef shardActor) {
 166         return TestShardManager.builder(datastoreContextBuilder).shardActor(shardActor)
 167                 .distributedDataStore(mock(DistributedDataStore.class));
 168     }
 169
 170
 171     private Props newPropsShardMgrWithMockShardActor() {
 172         return newTestShardMgrBuilderWithMockShardActor().props().withDispatcher(
 173                 Dispatchers.DefaultDispatcherId());
 174     }
 175
 176     private Props newPropsShardMgrWithMockShardActor(final ActorRef shardActor) {
 177         return newTestShardMgrBuilderWithMockShardActor(shardActor).props()
 178                 .withDispatcher(Dispatchers.DefaultDispatcherId());
 179     }
 180
 181
 182     private TestShardManager newTestShardManager() {
 183         return newTestShardManager(newShardMgrProps());
 184     }
 185
 186     private TestShardManager newTestShardManager(final Props props) {
 187         TestActorRef<TestShardManager> shardManagerActor = actorFactory.createTestActor(props);
 188         TestShardManager shardManager = shardManagerActor.underlyingActor();
 189         shardManager.waitForRecoveryComplete();
 190         return shardManager;
 191     }
 192
 193     private static void waitForShardInitialized(final ActorRef shardManager, final String shardName,
 194             final JavaTestKit kit) {
 195         AssertionError last = null;
 196         Stopwatch sw = Stopwatch.createStarted();
 197         while (sw.elapsed(TimeUnit.SECONDS) <= 5) {
 198             try {
 199                 shardManager.tell(new FindLocalShard(shardName, true), kit.getRef());
 200                 kit.expectMsgClass(LocalShardFound.class);
 201                 return;
 202             } catch (AssertionError e) {
 203                 last = e;
 204             }
 205
 206             Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
 207         }
 208
 209         throw last;
 210     }
 211
 212     @SuppressWarnings("unchecked")
 213     private static <T> T expectMsgClassOrFailure(final Class<T> msgClass, final JavaTestKit kit, final String msg) {
 214         Object reply = kit.expectMsgAnyClassOf(JavaTestKit.duration("5 sec"), msgClass, Failure.class);
 215         if (reply instanceof Failure) {
 216             throw new AssertionError(msg + " failed", ((Failure)reply).cause());
 217         }
 218
 219         return (T)reply;
 220     }
 221
 222     @Test
 223     public void testPerShardDatastoreContext() throws Exception {
 224         LOG.info("testPerShardDatastoreContext starting");
 225         final DatastoreContextFactory mockFactory = newDatastoreContextFactory(
 226                 datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
 227
 228         Mockito.doReturn(
 229                 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(6).build())
 230                 .when(mockFactory).getShardDatastoreContext("default");
 231
 232         Mockito.doReturn(
 233                 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(7).build())
 234                 .when(mockFactory).getShardDatastoreContext("topology");
 235
 236         final MockConfiguration mockConfig = new MockConfiguration() {
 237             @Override
 238             public Collection<String> getMemberShardNames(final MemberName memberName) {
 239                 return Arrays.asList("default", "topology");
 240             }
 241
 242             @Override
 243             public Collection<MemberName> getMembersFromShardName(final String shardName) {
 244                 return members("member-1");
 245             }
 246         };
 247
 248         final ActorRef defaultShardActor = actorFactory.createActor(
 249                 MessageCollectorActor.props(), actorFactory.generateActorId("default"));
 250         final ActorRef topologyShardActor = actorFactory.createActor(
 251                 MessageCollectorActor.props(), actorFactory.generateActorId("topology"));
 252
 253         final Map<String, Entry<ActorRef, DatastoreContext>> shardInfoMap = Collections.synchronizedMap(
 254                 new HashMap<String, Entry<ActorRef, DatastoreContext>>());
 255         shardInfoMap.put("default", new AbstractMap.SimpleEntry<>(defaultShardActor, null));
 256         shardInfoMap.put("topology", new AbstractMap.SimpleEntry<>(topologyShardActor, null));
 257
 258         final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
 259         final CountDownLatch newShardActorLatch = new CountDownLatch(2);
 260         class LocalShardManager extends ShardManager {
 261             LocalShardManager(final AbstractShardManagerCreator<?> creator) {
 262                 super(creator);
 263             }
 264
 265             @Override
 266             protected ActorRef newShardActor(final ShardInformation info) {
 267                 Entry<ActorRef, DatastoreContext> entry = shardInfoMap.get(info.getShardName());
 268                 ActorRef ref = null;
 269                 if (entry != null) {
 270                     ref = entry.getKey();
 271                     entry.setValue(info.getDatastoreContext());
 272                 }
 273
 274                 newShardActorLatch.countDown();
 275                 return ref;
 276             }
 277         }
 278
 279         final Creator<ShardManager> creator = new Creator<ShardManager>() {
 280             private static final long serialVersionUID = 1L;
 281             @Override
 282             public ShardManager create() throws Exception {
 283                 return new LocalShardManager(
 284                         new GenericCreator<>(LocalShardManager.class).datastoreContextFactory(mockFactory)
 285                                 .primaryShardInfoCache(primaryShardInfoCache).configuration(mockConfig));
 286             }
 287         };
 288
 289         JavaTestKit kit = new JavaTestKit(getSystem());
 290
 291         final ActorRef shardManager = actorFactory.createActor(Props.create(
 292                 new DelegatingShardManagerCreator(creator)).withDispatcher(Dispatchers.DefaultDispatcherId()));
 293
 294         shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), kit.getRef());
 295
 296         assertEquals("Shard actors created", true, newShardActorLatch.await(5, TimeUnit.SECONDS));
 297         assertEquals("getShardElectionTimeoutFactor", 6,
 298                 shardInfoMap.get("default").getValue().getShardElectionTimeoutFactor());
 299         assertEquals("getShardElectionTimeoutFactor", 7,
 300                 shardInfoMap.get("topology").getValue().getShardElectionTimeoutFactor());
 301
 302         DatastoreContextFactory newMockFactory = newDatastoreContextFactory(
 303                 datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
 304         Mockito.doReturn(
 305                 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(66).build())
 306                 .when(newMockFactory).getShardDatastoreContext("default");
 307
 308         Mockito.doReturn(
 309                 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(77).build())
 310                 .when(newMockFactory).getShardDatastoreContext("topology");
 311
 312         shardManager.tell(newMockFactory, kit.getRef());
 313
 314         DatastoreContext newContext = MessageCollectorActor.expectFirstMatching(defaultShardActor,
 315                 DatastoreContext.class);
 316         assertEquals("getShardElectionTimeoutFactor", 66, newContext.getShardElectionTimeoutFactor());
 317
 318         newContext = MessageCollectorActor.expectFirstMatching(topologyShardActor, DatastoreContext.class);
 319         assertEquals("getShardElectionTimeoutFactor", 77, newContext.getShardElectionTimeoutFactor());
 320
 321         LOG.info("testPerShardDatastoreContext ending");
 322     }
 323
 324     @Test
 325     public void testOnReceiveFindPrimaryForNonExistentShard() throws Exception {
 326         new JavaTestKit(getSystem()) {
 327             {
 328                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 329
 330                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 331
 332                 shardManager.tell(new FindPrimary("non-existent", false), getRef());
 333
 334                 expectMsgClass(duration("5 seconds"), PrimaryNotFoundException.class);
 335             }
 336         };
 337     }
 338
 339     @Test
 340     public void testOnReceiveFindPrimaryForLocalLeaderShard() throws Exception {
 341         LOG.info("testOnReceiveFindPrimaryForLocalLeaderShard starting");
 342         new JavaTestKit(getSystem()) {
 343             {
 344                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
 345
 346                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 347
 348                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 349                 shardManager.tell(new ActorInitialized(), mockShardActor);
 350
 351                 DataTree mockDataTree = mock(DataTree.class);
 352                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
 353                         DataStoreVersions.CURRENT_VERSION), getRef());
 354
 355                 MessageCollectorActor.expectFirstMatching(mockShardActor, RegisterRoleChangeListener.class);
 356                 shardManager.tell(
 357                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
 358                         mockShardActor);
 359
 360                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 361
 362                 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
 363                         LocalPrimaryShardFound.class);
 364                 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
 365                         primaryFound.getPrimaryPath().contains("member-1-shard-default"));
 366                 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
 367             }
 368         };
 369
 370         LOG.info("testOnReceiveFindPrimaryForLocalLeaderShard ending");
 371     }
 372
 373     @Test
 374     public void testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp() throws Exception {
 375         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp starting");
 376         new JavaTestKit(getSystem()) {
 377             {
 378                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 379
 380                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 381                 shardManager.tell(new ActorInitialized(), mockShardActor);
 382
 383                 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
 384                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
 385                 shardManager.tell(
 386                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
 387                         mockShardActor);
 388                 shardManager.tell(new LeaderStateChanged(memberId1, memberId2, DataStoreVersions.CURRENT_VERSION),
 389                         mockShardActor);
 390
 391                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 392
 393                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 394             }
 395         };
 396
 397         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp ending");
 398     }
 399
 400     @Test
 401     public void testOnReceiveFindPrimaryForNonLocalLeaderShard() throws Exception {
 402         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShard starting");
 403         new JavaTestKit(getSystem()) {
 404             {
 405                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 406
 407                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 408                 shardManager.tell(new ActorInitialized(), mockShardActor);
 409
 410                 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
 411                 MockClusterWrapper.sendMemberUp(shardManager, "member-2", getRef().path().toString());
 412
 413                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
 414                 shardManager.tell(
 415                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
 416                         mockShardActor);
 417                 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
 418                 shardManager.tell(new ShardLeaderStateChanged(memberId1, memberId2, leaderVersion), mockShardActor);
 419
 420                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 421
 422                 RemotePrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
 423                         RemotePrimaryShardFound.class);
 424                 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
 425                         primaryFound.getPrimaryPath().contains("member-2-shard-default"));
 426                 assertEquals("getPrimaryVersion", leaderVersion, primaryFound.getPrimaryVersion());
 427             }
 428         };
 429
 430         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShard ending");
 431     }
 432
 433     @Test
 434     public void testOnReceiveFindPrimaryForUninitializedShard() throws Exception {
 435         new JavaTestKit(getSystem()) {
 436             {
 437                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 438
 439                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 440
 441                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
 442             }
 443         };
 444     }
 445
 446     @Test
 447     public void testOnReceiveFindPrimaryForInitializedShardWithNoRole() throws Exception {
 448         new JavaTestKit(getSystem()) {
 449             {
 450                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 451
 452                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 453                 shardManager.tell(new ActorInitialized(), mockShardActor);
 454
 455                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 456
 457                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 458             }
 459         };
 460     }
 461
 462     @Test
 463     public void testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId() throws Exception {
 464         LOG.info("testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId starting");
 465         new JavaTestKit(getSystem()) {
 466             {
 467                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 468
 469                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 470                 shardManager.tell(new ActorInitialized(), mockShardActor);
 471
 472                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
 473                 shardManager.tell(
 474                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Follower.name()),
 475                         mockShardActor);
 476
 477                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 478
 479                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 480
 481                 DataTree mockDataTree = mock(DataTree.class);
 482                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
 483                         DataStoreVersions.CURRENT_VERSION), mockShardActor);
 484
 485                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 486
 487                 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
 488                         LocalPrimaryShardFound.class);
 489                 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
 490                         primaryFound.getPrimaryPath().contains("member-1-shard-default"));
 491                 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
 492             }
 493         };
 494
 495         LOG.info("testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId starting");
 496     }
 497
 498     @Test
 499     public void testOnReceiveFindPrimaryWaitForShardLeader() throws Exception {
 500         LOG.info("testOnReceiveFindPrimaryWaitForShardLeader starting");
 501         datastoreContextBuilder.shardInitializationTimeout(10, TimeUnit.SECONDS);
 502         new JavaTestKit(getSystem()) {
 503             {
 504                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 505
 506                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 507
 508                 // We're passing waitUntilInitialized = true to FindPrimary so
 509                 // the response should be
 510                 // delayed until we send ActorInitialized and
 511                 // RoleChangeNotification.
 512                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
 513
 514                 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
 515
 516                 shardManager.tell(new ActorInitialized(), mockShardActor);
 517
 518                 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
 519
 520                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
 521                 shardManager.tell(
 522                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
 523                         mockShardActor);
 524
 525                 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
 526
 527                 DataTree mockDataTree = mock(DataTree.class);
 528                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
 529                         DataStoreVersions.CURRENT_VERSION), mockShardActor);
 530
 531                 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
 532                         LocalPrimaryShardFound.class);
 533                 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
 534                         primaryFound.getPrimaryPath().contains("member-1-shard-default"));
 535                 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
 536
 537                 expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
 538             }
 539         };
 540
 541         LOG.info("testOnReceiveFindPrimaryWaitForShardLeader ending");
 542     }
 543
 544     @Test
 545     public void testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard() throws Exception {
 546         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard starting");
 547         new JavaTestKit(getSystem()) {
 548             {
 549                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 550
 551                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 552
 553                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
 554
 555                 expectMsgClass(duration("2 seconds"), NotInitializedException.class);
 556
 557                 shardManager.tell(new ActorInitialized(), mockShardActor);
 558
 559                 expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
 560             }
 561         };
 562
 563         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard ending");
 564     }
 565
 566     @Test
 567     public void testOnReceiveFindPrimaryWaitForReadyWithCandidateShard() throws Exception {
 568         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithCandidateShard starting");
 569         new JavaTestKit(getSystem()) {
 570             {
 571                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 572
 573                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 574                 shardManager.tell(new ActorInitialized(), mockShardActor);
 575                 shardManager.tell(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix, null,
 576                         RaftState.Candidate.name()), mockShardActor);
 577
 578                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
 579
 580                 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
 581             }
 582         };
 583
 584         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithCandidateShard ending");
 585     }
 586
 587     @Test
 588     public void testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard() throws Exception {
 589         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard starting");
 590         new JavaTestKit(getSystem()) {
 591             {
 592                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 593
 594                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 595                 shardManager.tell(new ActorInitialized(), mockShardActor);
 596                 shardManager.tell(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix, null,
 597                         RaftState.IsolatedLeader.name()), mockShardActor);
 598
 599                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
 600
 601                 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
 602             }
 603         };
 604
 605         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard ending");
 606     }
 607
 608     @Test
 609     public void testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard() throws Exception {
 610         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard starting");
 611         new JavaTestKit(getSystem()) {
 612             {
 613                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 614
 615                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 616                 shardManager.tell(new ActorInitialized(), mockShardActor);
 617
 618                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
 619
 620                 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
 621             }
 622         };
 623
 624         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard ending");
 625     }
 626
 627     @Test
 628     public void testOnReceiveFindPrimaryForRemoteShard() throws Exception {
 629         LOG.info("testOnReceiveFindPrimaryForRemoteShard starting");
 630         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
 631
 632         // Create an ActorSystem ShardManager actor for member-1.
 633
 634         final ActorSystem system1 = newActorSystem("Member1");
 635         Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
 636
 637         final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
 638                 newTestShardMgrBuilderWithMockShardActor().cluster(
 639                         new ClusterWrapperImpl(system1)).props().withDispatcher(
 640                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
 641
 642         // Create an ActorSystem ShardManager actor for member-2.
 643
 644         final ActorSystem system2 = newActorSystem("Member2");
 645
 646         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
 647
 648         final ActorRef mockShardActor2 = newMockShardActor(system2, "astronauts", "member-2");
 649
 650         MockConfiguration mockConfig2 = new MockConfiguration(
 651                 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
 652                         .put("astronauts", Arrays.asList("member-2")).build());
 653
 654         final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
 655                 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
 656                         new ClusterWrapperImpl(system2)).props().withDispatcher(
 657                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
 658
 659         new JavaTestKit(system1) {
 660             {
 661                 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 662                 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 663
 664                 shardManager2.tell(new ActorInitialized(), mockShardActor2);
 665
 666                 String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
 667                 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
 668                 shardManager2.tell(
 669                         new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
 670                         mockShardActor2);
 671                 shardManager2.tell(
 672                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
 673                         mockShardActor2);
 674
 675                 shardManager1.underlyingActor().waitForMemberUp();
 676                 shardManager1.tell(new FindPrimary("astronauts", false), getRef());
 677
 678                 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 679                 String path = found.getPrimaryPath();
 680                 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-astronauts-config"));
 681                 assertEquals("getPrimaryVersion", leaderVersion, found.getPrimaryVersion());
 682
 683                 shardManager2.underlyingActor().verifyFindPrimary();
 684
 685                 Cluster.get(system2).down(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
 686
 687                 shardManager1.underlyingActor().waitForMemberRemoved();
 688
 689                 shardManager1.tell(new FindPrimary("astronauts", false), getRef());
 690
 691                 expectMsgClass(duration("5 seconds"), PrimaryNotFoundException.class);
 692             }
 693         };
 694
 695         LOG.info("testOnReceiveFindPrimaryForRemoteShard ending");
 696     }
 697
 698     @Test
 699     public void testShardAvailabilityOnChangeOfMemberReachability() throws Exception {
 700         LOG.info("testShardAvailabilityOnChangeOfMemberReachability starting");
 701         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
 702
 703         // Create an ActorSystem ShardManager actor for member-1.
 704
 705         final ActorSystem system1 = newActorSystem("Member1");
 706         Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
 707
 708         final ActorRef mockShardActor1 = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
 709
 710         final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
 711                 newTestShardMgrBuilder().shardActor(mockShardActor1).cluster(
 712                         new ClusterWrapperImpl(system1)).props().withDispatcher(
 713                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
 714
 715         // Create an ActorSystem ShardManager actor for member-2.
 716
 717         final ActorSystem system2 = newActorSystem("Member2");
 718
 719         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
 720
 721         final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
 722
 723         MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
 724                 .put("default", Arrays.asList("member-1", "member-2")).build());
 725
 726         final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
 727                 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
 728                         new ClusterWrapperImpl(system2)).props().withDispatcher(
 729                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
 730
 731         new JavaTestKit(system1) {
 732             {
 733                 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 734                 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 735                 shardManager1.tell(new ActorInitialized(), mockShardActor1);
 736                 shardManager2.tell(new ActorInitialized(), mockShardActor2);
 737
 738                 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
 739                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
 740                 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class),
 741                         DataStoreVersions.CURRENT_VERSION), mockShardActor1);
 742                 shardManager1.tell(
 743                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
 744                         mockShardActor1);
 745                 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class),
 746                         DataStoreVersions.CURRENT_VERSION), mockShardActor2);
 747                 shardManager2.tell(
 748                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
 749                         mockShardActor2);
 750                 shardManager1.underlyingActor().waitForMemberUp();
 751
 752                 shardManager1.tell(new FindPrimary("default", true), getRef());
 753
 754                 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 755                 String path = found.getPrimaryPath();
 756                 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
 757
 758                 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
 759                         "akka://cluster-test@127.0.0.1:2558"), getRef());
 760
 761                 shardManager1.underlyingActor().waitForUnreachableMember();
 762
 763                 PeerDown peerDown = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
 764                 assertEquals("getMemberName", MEMBER_2, peerDown.getMemberName());
 765                 MessageCollectorActor.clearMessages(mockShardActor1);
 766
 767                 shardManager1.tell(
 768                         MockClusterWrapper.createMemberRemoved("member-2", "akka://cluster-test@127.0.0.1:2558"),
 769                         getRef());
 770
 771                 MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
 772
 773                 shardManager1.tell(new FindPrimary("default", true), getRef());
 774
 775                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 776
 777                 shardManager1.tell(
 778                         MockClusterWrapper.createReachableMember("member-2", "akka://cluster-test@127.0.0.1:2558"),
 779                         getRef());
 780
 781                 shardManager1.underlyingActor().waitForReachableMember();
 782
 783                 PeerUp peerUp = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
 784                 assertEquals("getMemberName", MEMBER_2, peerUp.getMemberName());
 785                 MessageCollectorActor.clearMessages(mockShardActor1);
 786
 787                 shardManager1.tell(new FindPrimary("default", true), getRef());
 788
 789                 RemotePrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 790                 String path1 = found1.getPrimaryPath();
 791                 assertTrue("Unexpected primary path " + path1, path1.contains("member-2-shard-default-config"));
 792
 793                 shardManager1.tell(
 794                         MockClusterWrapper.createMemberUp("member-2", "akka://cluster-test@127.0.0.1:2558"),
 795                         getRef());
 796
 797                 MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
 798
 799                 // Test FindPrimary wait succeeds after reachable member event.
 800
 801                 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
 802                         "akka://cluster-test@127.0.0.1:2558"), getRef());
 803                 shardManager1.underlyingActor().waitForUnreachableMember();
 804
 805                 shardManager1.tell(new FindPrimary("default", true), getRef());
 806
 807                 shardManager1.tell(
 808                         MockClusterWrapper.createReachableMember("member-2", "akka://cluster-test@127.0.0.1:2558"),
 809                         getRef());
 810
 811                 RemotePrimaryShardFound found2 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 812                 String path2 = found2.getPrimaryPath();
 813                 assertTrue("Unexpected primary path " + path2, path2.contains("member-2-shard-default-config"));
 814             }
 815         };
 816
 817         LOG.info("testShardAvailabilityOnChangeOfMemberReachability ending");
 818     }
 819
 820     @Test
 821     public void testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange() throws Exception {
 822         LOG.info("testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange starting");
 823         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
 824
 825         // Create an ActorSystem ShardManager actor for member-1.
 826
 827         final ActorSystem system1 = newActorSystem("Member1");
 828         Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
 829
 830         final ActorRef mockShardActor1 = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
 831
 832         final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
 833         final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
 834                 newTestShardMgrBuilder().shardActor(mockShardActor1).cluster(new ClusterWrapperImpl(system1))
 835                         .primaryShardInfoCache(primaryShardInfoCache).props()
 836                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
 837                 shardManagerID);
 838
 839         // Create an ActorSystem ShardManager actor for member-2.
 840
 841         final ActorSystem system2 = newActorSystem("Member2");
 842
 843         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
 844
 845         final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
 846
 847         MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
 848                 .put("default", Arrays.asList("member-1", "member-2")).build());
 849
 850         final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
 851                 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
 852                         new ClusterWrapperImpl(system2)).props().withDispatcher(
 853                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
 854
 855         new JavaTestKit(system1) {
 856             {
 857                 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 858                 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 859                 shardManager1.tell(new ActorInitialized(), mockShardActor1);
 860                 shardManager2.tell(new ActorInitialized(), mockShardActor2);
 861
 862                 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
 863                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
 864                 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class),
 865                         DataStoreVersions.CURRENT_VERSION), mockShardActor1);
 866                 shardManager1.tell(
 867                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
 868                         mockShardActor1);
 869                 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class),
 870                         DataStoreVersions.CURRENT_VERSION), mockShardActor2);
 871                 shardManager2.tell(
 872                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
 873                         mockShardActor2);
 874                 shardManager1.underlyingActor().waitForMemberUp();
 875
 876                 shardManager1.tell(new FindPrimary("default", true), getRef());
 877
 878                 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 879                 String path = found.getPrimaryPath();
 880                 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
 881
 882                 primaryShardInfoCache.putSuccessful("default", new PrimaryShardInfo(
 883                         system1.actorSelection(mockShardActor1.path()), DataStoreVersions.CURRENT_VERSION));
 884
 885                 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
 886                         "akka://cluster-test@127.0.0.1:2558"), getRef());
 887
 888                 shardManager1.underlyingActor().waitForUnreachableMember();
 889
 890                 shardManager1.tell(new FindPrimary("default", true), getRef());
 891
 892                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 893
 894                 assertNull("Expected primaryShardInfoCache entry removed",
 895                         primaryShardInfoCache.getIfPresent("default"));
 896
 897                 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId1, mock(DataTree.class),
 898                         DataStoreVersions.CURRENT_VERSION), mockShardActor1);
 899                 shardManager1.tell(
 900                         new RoleChangeNotification(memberId1, RaftState.Follower.name(), RaftState.Leader.name()),
 901                         mockShardActor1);
 902
 903                 shardManager1.tell(new FindPrimary("default", true), getRef());
 904
 905                 LocalPrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
 906                 String path1 = found1.getPrimaryPath();
 907                 assertTrue("Unexpected primary path " + path1, path1.contains("member-1-shard-default-config"));
 908
 909             }
 910         };
 911
 912         LOG.info("testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange ending");
 913     }
 914
 915
 916     @Test
 917     public void testOnReceiveFindLocalShardForNonExistentShard() throws Exception {
 918         new JavaTestKit(getSystem()) {
 919             {
 920                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 921
 922                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 923
 924                 shardManager.tell(new FindLocalShard("non-existent", false), getRef());
 925
 926                 LocalShardNotFound notFound = expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
 927
 928                 assertEquals("getShardName", "non-existent", notFound.getShardName());
 929             }
 930         };
 931     }
 932
 933     @Test
 934     public void testOnReceiveFindLocalShardForExistentShard() throws Exception {
 935         new JavaTestKit(getSystem()) {
 936             {
 937                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 938
 939                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 940                 shardManager.tell(new ActorInitialized(), mockShardActor);
 941
 942                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
 943
 944                 LocalShardFound found = expectMsgClass(duration("5 seconds"), LocalShardFound.class);
 945
 946                 assertTrue("Found path contains " + found.getPath().path().toString(),
 947                         found.getPath().path().toString().contains("member-1-shard-default-config"));
 948             }
 949         };
 950     }
 951
 952     @Test
 953     public void testOnReceiveFindLocalShardForNotInitializedShard() throws Exception {
 954         new JavaTestKit(getSystem()) {
 955             {
 956                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 957
 958                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
 959
 960                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
 961             }
 962         };
 963     }
 964
 965     @Test
 966     public void testOnReceiveFindLocalShardWaitForShardInitialized() throws Exception {
 967         LOG.info("testOnReceiveFindLocalShardWaitForShardInitialized starting");
 968         new JavaTestKit(getSystem()) {
 969             {
 970                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 971
 972                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 973
 974                 // We're passing waitUntilInitialized = true to FindLocalShard
 975                 // so the response should be
 976                 // delayed until we send ActorInitialized.
 977                 Future<Object> future = Patterns.ask(shardManager, new FindLocalShard(Shard.DEFAULT_NAME, true),
 978                         new Timeout(5, TimeUnit.SECONDS));
 979
 980                 shardManager.tell(new ActorInitialized(), mockShardActor);
 981
 982                 Object resp = Await.result(future, duration("5 seconds"));
 983                 assertTrue("Expected: LocalShardFound, Actual: " + resp, resp instanceof LocalShardFound);
 984             }
 985         };
 986
 987         LOG.info("testOnReceiveFindLocalShardWaitForShardInitialized starting");
 988     }
 989
 990     @Test
 991     public void testRoleChangeNotificationAndShardLeaderStateChangedReleaseReady() throws Exception {
 992         TestShardManager shardManager = newTestShardManager();
 993
 994         String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
 995         shardManager.onReceiveCommand(new RoleChangeNotification(
 996                 memberId, RaftState.Candidate.name(), RaftState.Leader.name()));
 997
 998         verify(ready, never()).countDown();
 999
1000         shardManager.onReceiveCommand(new ShardLeaderStateChanged(memberId, memberId,
1001                 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1002
1003         verify(ready, times(1)).countDown();
1004     }
1005
1006     @Test
1007     public void testRoleChangeNotificationToFollowerWithShardLeaderStateChangedReleaseReady() throws Exception {
1008         new JavaTestKit(getSystem()) {
1009             {
1010                 TestShardManager shardManager = newTestShardManager();
1011
1012                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1013                 shardManager.onReceiveCommand(new RoleChangeNotification(memberId, null, RaftState.Follower.name()));
1014
1015                 verify(ready, never()).countDown();
1016
1017                 shardManager
1018                         .onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
1019
1020                 shardManager.onReceiveCommand(
1021                         new ShardLeaderStateChanged(memberId, "member-2-shard-default-" + shardMrgIDSuffix,
1022                                 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1023
1024                 verify(ready, times(1)).countDown();
1025             }
1026         };
1027     }
1028
1029     @Test
1030     public void testReadyCountDownForMemberUpAfterLeaderStateChanged() throws Exception {
1031         new JavaTestKit(getSystem()) {
1032             {
1033                 TestShardManager shardManager = newTestShardManager();
1034
1035                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1036                 shardManager.onReceiveCommand(new RoleChangeNotification(memberId, null, RaftState.Follower.name()));
1037
1038                 verify(ready, never()).countDown();
1039
1040                 shardManager.onReceiveCommand(
1041                         new ShardLeaderStateChanged(memberId, "member-2-shard-default-" + shardMrgIDSuffix,
1042                                 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1043
1044                 shardManager
1045                         .onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
1046
1047                 verify(ready, times(1)).countDown();
1048             }
1049         };
1050     }
1051
1052     @Test
1053     public void testRoleChangeNotificationDoNothingForUnknownShard() throws Exception {
1054         TestShardManager shardManager = newTestShardManager();
1055
1056         shardManager.onReceiveCommand(new RoleChangeNotification(
1057                 "unknown", RaftState.Candidate.name(), RaftState.Leader.name()));
1058
1059         verify(ready, never()).countDown();
1060     }
1061
1062     @Test
1063     public void testByDefaultSyncStatusIsFalse() throws Exception {
1064         TestShardManager shardManager = newTestShardManager();
1065
1066         assertEquals(false, shardManager.getMBean().getSyncStatus());
1067     }
1068
1069     @Test
1070     public void testWhenShardIsLeaderSyncStatusIsTrue() throws Exception {
1071         TestShardManager shardManager = newTestShardManager();
1072
1073         shardManager.onReceiveCommand(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix,
1074                 RaftState.Follower.name(), RaftState.Leader.name()));
1075
1076         assertEquals(true, shardManager.getMBean().getSyncStatus());
1077     }
1078
1079     @Test
1080     public void testWhenShardIsCandidateSyncStatusIsFalse() throws Exception {
1081         TestShardManager shardManager = newTestShardManager();
1082
1083         String shardId = "member-1-shard-default-" + shardMrgIDSuffix;
1084         shardManager.onReceiveCommand(new RoleChangeNotification(shardId,
1085                 RaftState.Follower.name(), RaftState.Candidate.name()));
1086
1087         assertEquals(false, shardManager.getMBean().getSyncStatus());
1088
1089         // Send a FollowerInitialSyncStatus with status = true for the replica whose current state is candidate
1090         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(
1091                 true, shardId));
1092
1093         assertEquals(false, shardManager.getMBean().getSyncStatus());
1094     }
1095
1096     @Test
1097     public void testWhenShardIsFollowerSyncStatusDependsOnFollowerInitialSyncStatus() throws Exception {
1098         TestShardManager shardManager = newTestShardManager();
1099
1100         String shardId = "member-1-shard-default-" + shardMrgIDSuffix;
1101         shardManager.onReceiveCommand(new RoleChangeNotification(shardId,
1102                 RaftState.Candidate.name(), RaftState.Follower.name()));
1103
1104         // Initially will be false
1105         assertEquals(false, shardManager.getMBean().getSyncStatus());
1106
1107         // Send status true will make sync status true
1108         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(true, shardId));
1109
1110         assertEquals(true, shardManager.getMBean().getSyncStatus());
1111
1112         // Send status false will make sync status false
1113         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(false, shardId));
1114
1115         assertEquals(false, shardManager.getMBean().getSyncStatus());
1116
1117     }
1118
1119     @Test
1120     public void testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards() throws Exception {
1121         LOG.info("testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards starting");
1122         TestShardManager shardManager = newTestShardManager(newShardMgrProps(new MockConfiguration() {
1123             @Override
1124             public List<String> getMemberShardNames(final MemberName memberName) {
1125                 return Arrays.asList("default", "astronauts");
1126             }
1127         }));
1128
1129         // Initially will be false
1130         assertEquals(false, shardManager.getMBean().getSyncStatus());
1131
1132         // Make default shard leader
1133         String defaultShardId = "member-1-shard-default-" + shardMrgIDSuffix;
1134         shardManager.onReceiveCommand(new RoleChangeNotification(defaultShardId,
1135                 RaftState.Follower.name(), RaftState.Leader.name()));
1136
1137         // default = Leader, astronauts is unknown so sync status remains false
1138         assertEquals(false, shardManager.getMBean().getSyncStatus());
1139
1140         // Make astronauts shard leader as well
1141         String astronautsShardId = "member-1-shard-astronauts-" + shardMrgIDSuffix;
1142         shardManager.onReceiveCommand(new RoleChangeNotification(astronautsShardId,
1143                 RaftState.Follower.name(), RaftState.Leader.name()));
1144
1145         // Now sync status should be true
1146         assertEquals(true, shardManager.getMBean().getSyncStatus());
1147
1148         // Make astronauts a Follower
1149         shardManager.onReceiveCommand(new RoleChangeNotification(astronautsShardId,
1150                 RaftState.Leader.name(), RaftState.Follower.name()));
1151
1152         // Sync status is not true
1153         assertEquals(false, shardManager.getMBean().getSyncStatus());
1154
1155         // Make the astronauts follower sync status true
1156         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(true, astronautsShardId));
1157
1158         // Sync status is now true
1159         assertEquals(true, shardManager.getMBean().getSyncStatus());
1160
1161         LOG.info("testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards ending");
1162     }
1163
1164     @Test
1165     public void testOnReceiveSwitchShardBehavior() throws Exception {
1166         new JavaTestKit(getSystem()) {
1167             {
1168                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1169
1170                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1171                 shardManager.tell(new ActorInitialized(), mockShardActor);
1172
1173                 shardManager.tell(new SwitchShardBehavior(mockShardName, RaftState.Leader, 1000), getRef());
1174
1175                 SwitchBehavior switchBehavior = MessageCollectorActor.expectFirstMatching(mockShardActor,
1176                         SwitchBehavior.class);
1177
1178                 assertEquals(RaftState.Leader, switchBehavior.getNewState());
1179                 assertEquals(1000, switchBehavior.getNewTerm());
1180             }
1181         };
1182     }
1183
1184     private static List<MemberName> members(final String... names) {
1185         return Arrays.asList(names).stream().map(MemberName::forName).collect(Collectors.toList());
1186     }
1187
1188     @Test
1189     public void testOnCreateShard() {
1190         LOG.info("testOnCreateShard starting");
1191         new JavaTestKit(getSystem()) {
1192             {
1193                 datastoreContextBuilder.shardInitializationTimeout(1, TimeUnit.MINUTES).persistent(true);
1194
1195                 ActorRef shardManager = actorFactory
1196                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1197                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1198
1199                 SchemaContext schemaContext = TestModel.createTestContext();
1200                 shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
1201
1202                 DatastoreContext datastoreContext = DatastoreContext.newBuilder().shardElectionTimeoutFactor(100)
1203                         .persistent(false).build();
1204                 Shard.Builder shardBuilder = Shard.builder();
1205
1206                 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1207                         "foo", null, members("member-1", "member-5", "member-6"));
1208                 shardManager.tell(new CreateShard(config, shardBuilder, datastoreContext), getRef());
1209
1210                 expectMsgClass(duration("5 seconds"), Success.class);
1211
1212                 shardManager.tell(new FindLocalShard("foo", true), getRef());
1213
1214                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1215
1216                 assertEquals("isRecoveryApplicable", false, shardBuilder.getDatastoreContext().isPersistent());
1217                 assertTrue("Epxected ShardPeerAddressResolver", shardBuilder.getDatastoreContext().getShardRaftConfig()
1218                         .getPeerAddressResolver() instanceof ShardPeerAddressResolver);
1219                 assertEquals("peerMembers", Sets.newHashSet(
1220                         ShardIdentifier.create("foo", MemberName.forName("member-5"), shardMrgIDSuffix).toString(),
1221                         ShardIdentifier.create("foo", MemberName.forName("member-6"), shardMrgIDSuffix).toString()),
1222                         shardBuilder.getPeerAddresses().keySet());
1223                 assertEquals("ShardIdentifier", ShardIdentifier.create("foo", MEMBER_1, shardMrgIDSuffix),
1224                         shardBuilder.getId());
1225                 assertSame("schemaContext", schemaContext, shardBuilder.getSchemaContext());
1226
1227                 // Send CreateShard with same name - should return Success with
1228                 // a message.
1229
1230                 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1231
1232                 Success success = expectMsgClass(duration("5 seconds"), Success.class);
1233                 assertNotNull("Success status is null", success.status());
1234             }
1235         };
1236
1237         LOG.info("testOnCreateShard ending");
1238     }
1239
1240     @Test
1241     public void testOnCreateShardWithLocalMemberNotInShardConfig() {
1242         LOG.info("testOnCreateShardWithLocalMemberNotInShardConfig starting");
1243         new JavaTestKit(getSystem()) {
1244             {
1245                 datastoreContextBuilder.shardInitializationTimeout(1, TimeUnit.MINUTES).persistent(true);
1246
1247                 ActorRef shardManager = actorFactory
1248                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1249                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1250
1251                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1252
1253                 Shard.Builder shardBuilder = Shard.builder();
1254                 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1255                         "foo", null, members("member-5", "member-6"));
1256
1257                 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1258                 expectMsgClass(duration("5 seconds"), Success.class);
1259
1260                 shardManager.tell(new FindLocalShard("foo", true), getRef());
1261                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1262
1263                 assertEquals("peerMembers size", 0, shardBuilder.getPeerAddresses().size());
1264                 assertEquals("schemaContext", DisableElectionsRaftPolicy.class.getName(), shardBuilder
1265                         .getDatastoreContext().getShardRaftConfig().getCustomRaftPolicyImplementationClass());
1266             }
1267         };
1268
1269         LOG.info("testOnCreateShardWithLocalMemberNotInShardConfig ending");
1270     }
1271
1272     @Test
1273     public void testOnCreateShardWithNoInitialSchemaContext() {
1274         LOG.info("testOnCreateShardWithNoInitialSchemaContext starting");
1275         new JavaTestKit(getSystem()) {
1276             {
1277                 ActorRef shardManager = actorFactory
1278                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1279                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1280
1281                 Shard.Builder shardBuilder = Shard.builder();
1282
1283                 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1284                         "foo", null, members("member-1"));
1285                 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1286
1287                 expectMsgClass(duration("5 seconds"), Success.class);
1288
1289                 SchemaContext schemaContext = TestModel.createTestContext();
1290                 shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
1291
1292                 shardManager.tell(new FindLocalShard("foo", true), getRef());
1293
1294                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1295
1296                 assertSame("schemaContext", schemaContext, shardBuilder.getSchemaContext());
1297                 assertNotNull("schemaContext is null", shardBuilder.getDatastoreContext());
1298             }
1299         };
1300
1301         LOG.info("testOnCreateShardWithNoInitialSchemaContext ending");
1302     }
1303
1304     @Test
1305     public void testGetSnapshot() throws Exception {
1306         LOG.info("testGetSnapshot starting");
1307         JavaTestKit kit = new JavaTestKit(getSystem());
1308
1309         MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1310                 .put("shard1", Arrays.asList("member-1")).put("shard2", Arrays.asList("member-1"))
1311                 .put("astronauts", Collections.<String>emptyList()).build());
1312
1313         TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(newShardMgrProps(mockConfig)
1314                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1315
1316         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1317         Failure failure = kit.expectMsgClass(Failure.class);
1318         assertEquals("Failure cause type", IllegalStateException.class, failure.cause().getClass());
1319
1320         shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1321
1322         waitForShardInitialized(shardManager, "shard1", kit);
1323         waitForShardInitialized(shardManager, "shard2", kit);
1324
1325         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1326
1327         DatastoreSnapshot datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1328
1329         assertEquals("getType", shardMrgIDSuffix, datastoreSnapshot.getType());
1330         assertNull("Expected null ShardManagerSnapshot", datastoreSnapshot.getShardManagerSnapshot());
1331
1332         Function<ShardSnapshot, String> shardNameTransformer = ShardSnapshot::getName;
1333
1334         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2"), Sets.newHashSet(
1335                 Lists.transform(datastoreSnapshot.getShardSnapshots(), shardNameTransformer)));
1336
1337         // Add a new replica
1338
1339         JavaTestKit mockShardLeaderKit = new JavaTestKit(getSystem());
1340
1341         TestShardManager shardManagerInstance = shardManager.underlyingActor();
1342         shardManagerInstance.setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1343
1344         shardManager.tell(new AddShardReplica("astronauts"), kit.getRef());
1345         mockShardLeaderKit.expectMsgClass(AddServer.class);
1346         mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.OK, ""));
1347         kit.expectMsgClass(Status.Success.class);
1348         waitForShardInitialized(shardManager, "astronauts", kit);
1349
1350         // Send another GetSnapshot and verify
1351
1352         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1353         datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1354
1355         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"), Sets.newHashSet(
1356                 Lists.transform(datastoreSnapshot.getShardSnapshots(), shardNameTransformer)));
1357
1358         ShardManagerSnapshot snapshot = datastoreSnapshot.getShardManagerSnapshot();
1359         assertNotNull("Expected ShardManagerSnapshot", snapshot);
1360         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"),
1361                 Sets.newHashSet(snapshot.getShardList()));
1362
1363         LOG.info("testGetSnapshot ending");
1364     }
1365
1366     @Test
1367     public void testRestoreFromSnapshot() throws Exception {
1368         LOG.info("testRestoreFromSnapshot starting");
1369
1370         datastoreContextBuilder.shardInitializationTimeout(3, TimeUnit.SECONDS);
1371
1372         JavaTestKit kit = new JavaTestKit(getSystem());
1373
1374         MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1375                 .put("shard1", Collections.<String>emptyList()).put("shard2", Collections.<String>emptyList())
1376                 .put("astronauts", Collections.<String>emptyList()).build());
1377
1378         ShardManagerSnapshot snapshot =
1379                 new ShardManagerSnapshot(Arrays.asList("shard1", "shard2", "astronauts"), Collections.emptyMap());
1380         DatastoreSnapshot restoreFromSnapshot = new DatastoreSnapshot(shardMrgIDSuffix, snapshot,
1381                 Collections.<ShardSnapshot>emptyList());
1382         TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(newTestShardMgrBuilder(mockConfig)
1383                 .restoreFromSnapshot(restoreFromSnapshot).props().withDispatcher(Dispatchers.DefaultDispatcherId()));
1384
1385         shardManager.underlyingActor().waitForRecoveryComplete();
1386
1387         shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1388
1389         waitForShardInitialized(shardManager, "shard1", kit);
1390         waitForShardInitialized(shardManager, "shard2", kit);
1391         waitForShardInitialized(shardManager, "astronauts", kit);
1392
1393         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1394
1395         DatastoreSnapshot datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1396
1397         assertEquals("getType", shardMrgIDSuffix, datastoreSnapshot.getType());
1398
1399         assertNotNull("Expected ShardManagerSnapshot", datastoreSnapshot.getShardManagerSnapshot());
1400         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"),
1401                 Sets.newHashSet(datastoreSnapshot.getShardManagerSnapshot().getShardList()));
1402
1403         LOG.info("testRestoreFromSnapshot ending");
1404     }
1405
1406     @Test
1407     public void testAddShardReplicaForNonExistentShardConfig() throws Exception {
1408         new JavaTestKit(getSystem()) {
1409             {
1410                 ActorRef shardManager = actorFactory
1411                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1412                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1413
1414                 shardManager.tell(new AddShardReplica("model-inventory"), getRef());
1415                 Status.Failure resp = expectMsgClass(duration("2 seconds"), Status.Failure.class);
1416
1417                 assertEquals("Failure obtained", true, resp.cause() instanceof IllegalArgumentException);
1418             }
1419         };
1420     }
1421
1422     @Test
1423     public void testAddShardReplica() throws Exception {
1424         LOG.info("testAddShardReplica starting");
1425         MockConfiguration mockConfig = new MockConfiguration(
1426                 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
1427                         .put("astronauts", Arrays.asList("member-2")).build());
1428
1429         final String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
1430         datastoreContextBuilder.shardManagerPersistenceId(shardManagerID);
1431
1432         // Create an ActorSystem ShardManager actor for member-1.
1433         final ActorSystem system1 = newActorSystem("Member1");
1434         Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1435         ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
1436         final TestActorRef<TestShardManager> newReplicaShardManager = TestActorRef.create(system1,
1437                 newTestShardMgrBuilder(mockConfig).shardActor(mockDefaultShardActor)
1438                         .cluster(new ClusterWrapperImpl(system1)).props()
1439                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
1440                 shardManagerID);
1441
1442         // Create an ActorSystem ShardManager actor for member-2.
1443         final ActorSystem system2 = newActorSystem("Member2");
1444         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1445
1446         String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
1447         String name = ShardIdentifier.create("astronauts", MEMBER_2, "config").toString();
1448         final TestActorRef<MockRespondActor> mockShardLeaderActor = TestActorRef.create(system2,
1449                 Props.create(MockRespondActor.class, AddServer.class,
1450                         new AddServerReply(ServerChangeStatus.OK, memberId2))
1451                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
1452                 name);
1453         final TestActorRef<TestShardManager> leaderShardManager = TestActorRef.create(system2,
1454                 newTestShardMgrBuilder(mockConfig).shardActor(mockShardLeaderActor)
1455                         .cluster(new ClusterWrapperImpl(system2)).props()
1456                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
1457                 shardManagerID);
1458
1459         new JavaTestKit(system1) {
1460             {
1461                 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1462                 leaderShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1463
1464                 leaderShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1465
1466                 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
1467                 leaderShardManager.tell(
1468                         new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
1469                         mockShardLeaderActor);
1470                 leaderShardManager.tell(
1471                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
1472                         mockShardLeaderActor);
1473
1474                 newReplicaShardManager.underlyingActor().waitForMemberUp();
1475                 leaderShardManager.underlyingActor().waitForMemberUp();
1476
1477                 // Have a dummy snapshot to be overwritten by the new data
1478                 // persisted.
1479                 String[] restoredShards = { "default", "people" };
1480                 ShardManagerSnapshot snapshot =
1481                         new ShardManagerSnapshot(Arrays.asList(restoredShards), Collections.emptyMap());
1482                 InMemorySnapshotStore.addSnapshot(shardManagerID, snapshot);
1483                 Uninterruptibles.sleepUninterruptibly(2, TimeUnit.MILLISECONDS);
1484
1485                 InMemorySnapshotStore.addSnapshotSavedLatch(shardManagerID);
1486                 InMemorySnapshotStore.addSnapshotDeletedLatch(shardManagerID);
1487
1488                 // construct a mock response message
1489                 newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
1490                 AddServer addServerMsg = MessageCollectorActor.expectFirstMatching(mockShardLeaderActor,
1491                         AddServer.class);
1492                 String addServerId = "member-1-shard-astronauts-" + shardMrgIDSuffix;
1493                 assertEquals("AddServer serverId", addServerId, addServerMsg.getNewServerId());
1494                 expectMsgClass(duration("5 seconds"), Status.Success.class);
1495
1496                 InMemorySnapshotStore.waitForSavedSnapshot(shardManagerID, ShardManagerSnapshot.class);
1497                 InMemorySnapshotStore.waitForDeletedSnapshot(shardManagerID);
1498                 List<ShardManagerSnapshot> persistedSnapshots = InMemorySnapshotStore.getSnapshots(shardManagerID,
1499                         ShardManagerSnapshot.class);
1500                 assertEquals("Number of snapshots persisted", 1, persistedSnapshots.size());
1501                 ShardManagerSnapshot shardManagerSnapshot = persistedSnapshots.get(0);
1502                 assertEquals("Persisted local shards", Sets.newHashSet("default", "astronauts"),
1503                         Sets.newHashSet(shardManagerSnapshot.getShardList()));
1504             }
1505         };
1506         LOG.info("testAddShardReplica ending");
1507     }
1508
1509     @Test
1510     public void testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader() throws Exception {
1511         LOG.info("testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader starting");
1512         new JavaTestKit(getSystem()) {
1513             {
1514                 TestActorRef<TestShardManager> shardManager = actorFactory
1515                         .createTestActor(newPropsShardMgrWithMockShardActor(), shardMgrID);
1516
1517                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1518                 shardManager.tell(new ActorInitialized(), mockShardActor);
1519
1520                 String leaderId = "leader-member-shard-default-" + shardMrgIDSuffix;
1521                 AddServerReply addServerReply = new AddServerReply(ServerChangeStatus.ALREADY_EXISTS, null);
1522                 ActorRef leaderShardActor = shardManager.underlyingActor().getContext()
1523                         .actorOf(Props.create(MockRespondActor.class, AddServer.class, addServerReply), leaderId);
1524
1525                 MockClusterWrapper.sendMemberUp(shardManager, "leader-member", leaderShardActor.path().toString());
1526
1527                 String newReplicaId = "member-1-shard-default-" + shardMrgIDSuffix;
1528                 shardManager.tell(
1529                         new RoleChangeNotification(newReplicaId, RaftState.Candidate.name(), RaftState.Follower.name()),
1530                         mockShardActor);
1531                 shardManager.tell(
1532                         new ShardLeaderStateChanged(newReplicaId, leaderId, DataStoreVersions.CURRENT_VERSION),
1533                         mockShardActor);
1534
1535                 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1536
1537                 MessageCollectorActor.expectFirstMatching(leaderShardActor, AddServer.class);
1538
1539                 Failure resp = expectMsgClass(duration("5 seconds"), Failure.class);
1540                 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1541
1542                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1543                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1544
1545                 // Send message again to verify previous in progress state is
1546                 // cleared
1547
1548                 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1549                 resp = expectMsgClass(duration("5 seconds"), Failure.class);
1550                 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1551
1552                 // Send message again with an AddServer timeout to verify the
1553                 // pre-existing shard actor isn't terminated.
1554
1555                 shardManager.tell(
1556                         newDatastoreContextFactory(
1557                                 datastoreContextBuilder.shardLeaderElectionTimeout(100, TimeUnit.MILLISECONDS).build()),
1558                         getRef());
1559                 leaderShardActor.tell(MockRespondActor.CLEAR_RESPONSE, ActorRef.noSender());
1560                 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1561                 expectMsgClass(duration("5 seconds"), Failure.class);
1562
1563                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1564                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1565             }
1566         };
1567
1568         LOG.info("testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader ending");
1569     }
1570
1571     @Test
1572     public void testAddShardReplicaWithPreExistingLocalReplicaLeader() throws Exception {
1573         LOG.info("testAddShardReplicaWithPreExistingLocalReplicaLeader starting");
1574         new JavaTestKit(getSystem()) {
1575             {
1576                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1577                 ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1578
1579                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1580                 shardManager.tell(new ActorInitialized(), mockShardActor);
1581                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
1582                         DataStoreVersions.CURRENT_VERSION), getRef());
1583                 shardManager.tell(
1584                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
1585                         mockShardActor);
1586
1587                 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1588                 Failure resp = expectMsgClass(duration("5 seconds"), Failure.class);
1589                 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1590
1591                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1592                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1593             }
1594         };
1595
1596         LOG.info("testAddShardReplicaWithPreExistingLocalReplicaLeader ending");
1597     }
1598
1599     @Test
1600     public void testAddShardReplicaWithAddServerReplyFailure() throws Exception {
1601         LOG.info("testAddShardReplicaWithAddServerReplyFailure starting");
1602         new JavaTestKit(getSystem()) {
1603             {
1604                 JavaTestKit mockShardLeaderKit = new JavaTestKit(getSystem());
1605
1606                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1607                         .put("astronauts", Arrays.asList("member-2")).build());
1608
1609                 ActorRef mockNewReplicaShardActor = newMockShardActor(getSystem(), "astronauts", "member-1");
1610                 final TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(
1611                         newTestShardMgrBuilder(mockConfig).shardActor(mockNewReplicaShardActor).props()
1612                             .withDispatcher(Dispatchers.DefaultDispatcherId()), shardMgrID);
1613                 shardManager.underlyingActor()
1614                         .setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1615
1616                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1617
1618                 JavaTestKit terminateWatcher = new JavaTestKit(getSystem());
1619                 terminateWatcher.watch(mockNewReplicaShardActor);
1620
1621                 shardManager.tell(new AddShardReplica("astronauts"), getRef());
1622
1623                 AddServer addServerMsg = mockShardLeaderKit.expectMsgClass(AddServer.class);
1624                 assertEquals("AddServer serverId", "member-1-shard-astronauts-" + shardMrgIDSuffix,
1625                         addServerMsg.getNewServerId());
1626                 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.TIMEOUT, null));
1627
1628                 Failure failure = expectMsgClass(duration("5 seconds"), Failure.class);
1629                 assertEquals("Failure cause", TimeoutException.class, failure.cause().getClass());
1630
1631                 shardManager.tell(new FindLocalShard("astronauts", false), getRef());
1632                 expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
1633
1634                 terminateWatcher.expectTerminated(mockNewReplicaShardActor);
1635
1636                 shardManager.tell(new AddShardReplica("astronauts"), getRef());
1637                 mockShardLeaderKit.expectMsgClass(AddServer.class);
1638                 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.NO_LEADER, null));
1639                 failure = expectMsgClass(duration("5 seconds"), Failure.class);
1640                 assertEquals("Failure cause", NoShardLeaderException.class, failure.cause().getClass());
1641             }
1642         };
1643
1644         LOG.info("testAddShardReplicaWithAddServerReplyFailure ending");
1645     }
1646
1647     @Test
1648     public void testAddShardReplicaWithAlreadyInProgress() throws Exception {
1649         testServerChangeWhenAlreadyInProgress("astronauts", new AddShardReplica("astronauts"),
1650                 AddServer.class, new AddShardReplica("astronauts"));
1651     }
1652
1653     @Test
1654     public void testAddShardReplicaWithFindPrimaryTimeout() throws Exception {
1655         LOG.info("testAddShardReplicaWithFindPrimaryTimeout starting");
1656         datastoreContextBuilder.shardInitializationTimeout(100, TimeUnit.MILLISECONDS);
1657         new JavaTestKit(getSystem()) {
1658             {
1659                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1660                         .put("astronauts", Arrays.asList("member-2")).build());
1661
1662                 final ActorRef newReplicaShardManager = actorFactory
1663                         .createActor(newTestShardMgrBuilder(mockConfig).shardActor(mockShardActor).props()
1664                                 .withDispatcher(Dispatchers.DefaultDispatcherId()), shardMgrID);
1665
1666                 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1667                 MockClusterWrapper.sendMemberUp(newReplicaShardManager, "member-2",
1668                         AddressFromURIString.parse("akka://non-existent@127.0.0.1:5").toString());
1669
1670                 newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
1671                 Status.Failure resp = expectMsgClass(duration("5 seconds"), Status.Failure.class);
1672                 assertEquals("Failure obtained", true, resp.cause() instanceof RuntimeException);
1673             }
1674         };
1675
1676         LOG.info("testAddShardReplicaWithFindPrimaryTimeout ending");
1677     }
1678
1679     @Test
1680     public void testRemoveShardReplicaForNonExistentShard() throws Exception {
1681         new JavaTestKit(getSystem()) {
1682             {
1683                 ActorRef shardManager = actorFactory
1684                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1685                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1686
1687                 shardManager.tell(new RemoveShardReplica("model-inventory", MEMBER_1), getRef());
1688                 Status.Failure resp = expectMsgClass(duration("10 seconds"), Status.Failure.class);
1689                 assertEquals("Failure obtained", true, resp.cause() instanceof PrimaryNotFoundException);
1690             }
1691         };
1692     }
1693
1694     @Test
1695     /**
1696      * Primary is Local
1697      */
1698     public void testRemoveShardReplicaLocal() throws Exception {
1699         new JavaTestKit(getSystem()) {
1700             {
1701                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1702
1703                 final ActorRef respondActor = actorFactory.createActor(Props.create(MockRespondActor.class,
1704                         RemoveServer.class, new RemoveServerReply(ServerChangeStatus.OK, null)), memberId);
1705
1706                 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
1707
1708                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1709                 shardManager.tell(new ActorInitialized(), respondActor);
1710                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
1711                         DataStoreVersions.CURRENT_VERSION), getRef());
1712                 shardManager.tell(
1713                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
1714                         respondActor);
1715
1716                 shardManager.tell(new RemoveShardReplica(Shard.DEFAULT_NAME, MEMBER_1), getRef());
1717                 final RemoveServer removeServer = MessageCollectorActor.expectFirstMatching(respondActor,
1718                         RemoveServer.class);
1719                 assertEquals(ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString(),
1720                         removeServer.getServerId());
1721                 expectMsgClass(duration("5 seconds"), Success.class);
1722             }
1723         };
1724     }
1725
1726     @Test
1727     public void testRemoveShardReplicaRemote() throws Exception {
1728         MockConfiguration mockConfig = new MockConfiguration(
1729                 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
1730                         .put("astronauts", Arrays.asList("member-1")).build());
1731
1732         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
1733
1734         // Create an ActorSystem ShardManager actor for member-1.
1735         final ActorSystem system1 = newActorSystem("Member1");
1736         Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1737         ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
1738
1739         final TestActorRef<TestShardManager> newReplicaShardManager = TestActorRef.create(system1,
1740                 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockDefaultShardActor).cluster(
1741                         new ClusterWrapperImpl(system1)).props().withDispatcher(Dispatchers.DefaultDispatcherId()),
1742                 shardManagerID);
1743
1744         // Create an ActorSystem ShardManager actor for member-2.
1745         final ActorSystem system2 = newActorSystem("Member2");
1746         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1747
1748         String name = ShardIdentifier.create("default", MEMBER_2, shardMrgIDSuffix).toString();
1749         String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
1750         final TestActorRef<MockRespondActor> mockShardLeaderActor =
1751                 TestActorRef.create(system2, Props.create(MockRespondActor.class, RemoveServer.class,
1752                         new RemoveServerReply(ServerChangeStatus.OK, memberId2)), name);
1753
1754         LOG.error("Mock Shard Leader Actor : {}", mockShardLeaderActor);
1755
1756         final TestActorRef<TestShardManager> leaderShardManager = TestActorRef.create(system2,
1757                 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockShardLeaderActor).cluster(
1758                         new ClusterWrapperImpl(system2)).props().withDispatcher(Dispatchers.DefaultDispatcherId()),
1759                 shardManagerID);
1760
1761         // Because mockShardLeaderActor is created at the top level of the actor system it has an address like so,
1762         //    akka://cluster-test@127.0.0.1:2559/user/member-2-shard-default-config1
1763         // However when a shard manager has a local shard which is a follower and a leader that is remote it will
1764         // try to compute an address for the remote shard leader using the ShardPeerAddressResolver. This address will
1765         // look like so,
1766         //    akka://cluster-test@127.0.0.1:2559/user/shardmanager-config1/member-2-shard-default-config1
1767         // In this specific case if we did a FindPrimary for shard default from member-1 we would come up
1768         // with the address of an actor which does not exist, therefore any message sent to that actor would go to
1769         // dead letters.
1770         // To work around this problem we create a ForwardingActor with the right address and pass to it the
1771         // mockShardLeaderActor. The ForwardingActor simply forwards all messages to the mockShardLeaderActor and every
1772         // thing works as expected
1773         final ActorRef actorRef = leaderShardManager.underlyingActor().context()
1774                 .actorOf(Props.create(ForwardingActor.class, mockShardLeaderActor),
1775                         "member-2-shard-default-" + shardMrgIDSuffix);
1776
1777         LOG.error("Forwarding actor : {}", actorRef);
1778
1779         new JavaTestKit(system1) {
1780             {
1781                 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1782                 leaderShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1783
1784                 leaderShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1785                 newReplicaShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1786
1787                 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
1788                 leaderShardManager.tell(
1789                         new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
1790                         mockShardLeaderActor);
1791                 leaderShardManager.tell(
1792                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
1793                         mockShardLeaderActor);
1794
1795                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
1796                 newReplicaShardManager.tell(
1797                         new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class), leaderVersion),
1798                         mockShardActor);
1799                 newReplicaShardManager.tell(
1800                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
1801                         mockShardActor);
1802
1803                 newReplicaShardManager.underlyingActor().waitForMemberUp();
1804                 leaderShardManager.underlyingActor().waitForMemberUp();
1805
1806                 // construct a mock response message
1807                 newReplicaShardManager.tell(new RemoveShardReplica("default", MEMBER_1), getRef());
1808                 RemoveServer removeServer = MessageCollectorActor.expectFirstMatching(mockShardLeaderActor,
1809                         RemoveServer.class);
1810                 String removeServerId = ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString();
1811                 assertEquals("RemoveServer serverId", removeServerId, removeServer.getServerId());
1812                 expectMsgClass(duration("5 seconds"), Status.Success.class);
1813             }
1814         };
1815     }
1816
1817     @Test
1818     public void testRemoveShardReplicaWhenAnotherRemoveShardReplicaAlreadyInProgress() throws Exception {
1819         testServerChangeWhenAlreadyInProgress("astronauts", new RemoveShardReplica("astronauts", MEMBER_2),
1820                 RemoveServer.class, new RemoveShardReplica("astronauts", MEMBER_3));
1821     }
1822
1823     @Test
1824     public void testRemoveShardReplicaWhenAddShardReplicaAlreadyInProgress() throws Exception {
1825         testServerChangeWhenAlreadyInProgress("astronauts", new AddShardReplica("astronauts"),
1826                 AddServer.class, new RemoveShardReplica("astronauts", MEMBER_2));
1827     }
1828
1829
1830     public void testServerChangeWhenAlreadyInProgress(final String shardName, final Object firstServerChange,
1831                                                       final Class<?> firstForwardedServerChangeClass,
1832                                                       final Object secondServerChange) throws Exception {
1833         new JavaTestKit(getSystem()) {
1834             {
1835                 JavaTestKit mockShardLeaderKit = new JavaTestKit(getSystem());
1836                 final JavaTestKit secondRequestKit = new JavaTestKit(getSystem());
1837
1838                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1839                         .put(shardName, Arrays.asList("member-2")).build());
1840
1841                 final TestActorRef<TestShardManager> shardManager = TestActorRef.create(getSystem(),
1842                         newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockShardActor)
1843                                 .cluster(new MockClusterWrapper()).props()
1844                                 .withDispatcher(Dispatchers.DefaultDispatcherId()),
1845                         shardMgrID);
1846
1847                 shardManager.underlyingActor()
1848                         .setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1849
1850                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1851
1852                 shardManager.tell(firstServerChange, getRef());
1853
1854                 mockShardLeaderKit.expectMsgClass(firstForwardedServerChangeClass);
1855
1856                 shardManager.tell(secondServerChange, secondRequestKit.getRef());
1857
1858                 secondRequestKit.expectMsgClass(duration("5 seconds"), Failure.class);
1859             }
1860         };
1861     }
1862
1863     @Test
1864     public void testServerRemovedShardActorNotRunning() throws Exception {
1865         LOG.info("testServerRemovedShardActorNotRunning starting");
1866         new JavaTestKit(getSystem()) {
1867             {
1868                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1869                         .put("default", Arrays.asList("member-1", "member-2"))
1870                         .put("astronauts", Arrays.asList("member-2"))
1871                         .put("people", Arrays.asList("member-1", "member-2")).build());
1872
1873                 TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(
1874                         newShardMgrProps(mockConfig).withDispatcher(Dispatchers.DefaultDispatcherId()));
1875
1876                 shardManager.underlyingActor().waitForRecoveryComplete();
1877                 shardManager.tell(new FindLocalShard("people", false), getRef());
1878                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1879
1880                 shardManager.tell(new FindLocalShard("default", false), getRef());
1881                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1882
1883                 // Removed the default shard replica from member-1
1884                 ShardIdentifier.Builder builder = new ShardIdentifier.Builder();
1885                 ShardIdentifier shardId = builder.shardName("default").memberName(MEMBER_1).type(shardMrgIDSuffix)
1886                         .build();
1887                 shardManager.tell(new ServerRemoved(shardId.toString()), getRef());
1888
1889                 shardManager.underlyingActor().verifySnapshotPersisted(Sets.newHashSet("people"));
1890             }
1891         };
1892
1893         LOG.info("testServerRemovedShardActorNotRunning ending");
1894     }
1895
1896     @Test
1897     public void testServerRemovedShardActorRunning() throws Exception {
1898         LOG.info("testServerRemovedShardActorRunning starting");
1899         new JavaTestKit(getSystem()) {
1900             {
1901                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1902                         .put("default", Arrays.asList("member-1", "member-2"))
1903                         .put("astronauts", Arrays.asList("member-2"))
1904                         .put("people", Arrays.asList("member-1", "member-2")).build());
1905
1906                 String shardId = ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString();
1907                 ActorRef shard = actorFactory.createActor(MessageCollectorActor.props(), shardId);
1908
1909                 TestActorRef<TestShardManager> shardManager = actorFactory
1910                         .createTestActor(newTestShardMgrBuilder(mockConfig).addShardActor("default", shard).props()
1911                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1912
1913                 shardManager.underlyingActor().waitForRecoveryComplete();
1914
1915                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1916                 shardManager.tell(new ActorInitialized(), shard);
1917
1918                 waitForShardInitialized(shardManager, "people", this);
1919                 waitForShardInitialized(shardManager, "default", this);
1920
1921                 // Removed the default shard replica from member-1
1922                 shardManager.tell(new ServerRemoved(shardId), getRef());
1923
1924                 shardManager.underlyingActor().verifySnapshotPersisted(Sets.newHashSet("people"));
1925
1926                 MessageCollectorActor.expectFirstMatching(shard, Shutdown.class);
1927             }
1928         };
1929
1930         LOG.info("testServerRemovedShardActorRunning ending");
1931     }
1932
1933     @Test
1934     public void testShardPersistenceWithRestoredData() throws Exception {
1935         LOG.info("testShardPersistenceWithRestoredData starting");
1936         new JavaTestKit(getSystem()) {
1937             {
1938                 MockConfiguration mockConfig =
1939                     new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1940                             .put("default", Arrays.asList("member-1", "member-2"))
1941                             .put("astronauts", Arrays.asList("member-2"))
1942                             .put("people", Arrays.asList("member-1", "member-2")).build());
1943                 String[] restoredShards = {"default", "astronauts"};
1944                 ShardManagerSnapshot snapshot =
1945                         new ShardManagerSnapshot(Arrays.asList(restoredShards), Collections.emptyMap());
1946                 InMemorySnapshotStore.addSnapshot("shard-manager-" + shardMrgIDSuffix, snapshot);
1947
1948                 // create shardManager to come up with restored data
1949                 TestActorRef<TestShardManager> newRestoredShardManager = actorFactory.createTestActor(
1950                         newShardMgrProps(mockConfig).withDispatcher(Dispatchers.DefaultDispatcherId()));
1951
1952                 newRestoredShardManager.underlyingActor().waitForRecoveryComplete();
1953
1954                 newRestoredShardManager.tell(new FindLocalShard("people", false), getRef());
1955                 LocalShardNotFound notFound = expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
1956                 assertEquals("for uninitialized shard", "people", notFound.getShardName());
1957
1958                 // Verify a local shard is created for the restored shards,
1959                 // although we expect a NotInitializedException for the shards
1960                 // as the actor initialization
1961                 // message is not sent for them
1962                 newRestoredShardManager.tell(new FindLocalShard("default", false), getRef());
1963                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1964
1965                 newRestoredShardManager.tell(new FindLocalShard("astronauts", false), getRef());
1966                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1967             }
1968         };
1969
1970         LOG.info("testShardPersistenceWithRestoredData ending");
1971     }
1972
1973     @Test
1974     public void testShutDown() throws Exception {
1975         LOG.info("testShutDown starting");
1976         new JavaTestKit(getSystem()) {
1977             {
1978                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1979                         .put("shard1", Arrays.asList("member-1")).put("shard2", Arrays.asList("member-1")).build());
1980
1981                 String shardId1 = ShardIdentifier.create("shard1", MEMBER_1, shardMrgIDSuffix).toString();
1982                 ActorRef shard1 = actorFactory.createActor(MessageCollectorActor.props(), shardId1);
1983
1984                 String shardId2 = ShardIdentifier.create("shard2", MEMBER_1, shardMrgIDSuffix).toString();
1985                 ActorRef shard2 = actorFactory.createActor(MessageCollectorActor.props(), shardId2);
1986
1987                 ActorRef shardManager = actorFactory.createActor(newTestShardMgrBuilder(mockConfig)
1988                         .addShardActor("shard1", shard1).addShardActor("shard2", shard2).props());
1989
1990                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1991                 shardManager.tell(new ActorInitialized(), shard1);
1992                 shardManager.tell(new ActorInitialized(), shard2);
1993
1994                 FiniteDuration duration = FiniteDuration.create(5, TimeUnit.SECONDS);
1995                 Future<Boolean> stopFuture = Patterns.gracefulStop(shardManager, duration, Shutdown.INSTANCE);
1996
1997                 MessageCollectorActor.expectFirstMatching(shard1, Shutdown.class);
1998                 MessageCollectorActor.expectFirstMatching(shard2, Shutdown.class);
1999
2000                 try {
2001                     Await.ready(stopFuture, FiniteDuration.create(500, TimeUnit.MILLISECONDS));
2002                     fail("ShardManager actor stopped without waiting for the Shards to be stopped");
2003                 } catch (TimeoutException e) {
2004                     // expected
2005                 }
2006
2007                 actorFactory.killActor(shard1, this);
2008                 actorFactory.killActor(shard2, this);
2009
2010                 Boolean stopped = Await.result(stopFuture, duration);
2011                 assertEquals("Stopped", Boolean.TRUE, stopped);
2012             }
2013         };
2014
2015         LOG.info("testShutDown ending");
2016     }
2017
2018     @Test
2019     public void testChangeServersVotingStatus() throws Exception {
2020         new JavaTestKit(getSystem()) {
2021             {
2022                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
2023
2024                 ActorRef respondActor = actorFactory
2025                         .createActor(Props.create(MockRespondActor.class, ChangeServersVotingStatus.class,
2026                                 new ServerChangeReply(ServerChangeStatus.OK, null)), memberId);
2027
2028                 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
2029
2030                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
2031                 shardManager.tell(new ActorInitialized(), respondActor);
2032                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
2033                         DataStoreVersions.CURRENT_VERSION), getRef());
2034                 shardManager.tell(
2035                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
2036                         respondActor);
2037
2038                 shardManager.tell(
2039                         new ChangeShardMembersVotingStatus("default", ImmutableMap.of("member-2", Boolean.TRUE)),
2040                         getRef());
2041
2042                 ChangeServersVotingStatus actualChangeStatusMsg = MessageCollectorActor
2043                         .expectFirstMatching(respondActor, ChangeServersVotingStatus.class);
2044                 assertEquals("ChangeServersVotingStatus map", actualChangeStatusMsg.getServerVotingStatusMap(),
2045                         ImmutableMap.of(ShardIdentifier
2046                                 .create("default", MemberName.forName("member-2"), shardMrgIDSuffix).toString(),
2047                                 Boolean.TRUE));
2048
2049                 expectMsgClass(duration("5 seconds"), Success.class);
2050             }
2051         };
2052     }
2053
2054     @Test
2055     public void testChangeServersVotingStatusWithNoLeader() throws Exception {
2056         new JavaTestKit(getSystem()) {
2057             {
2058                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
2059
2060                 ActorRef respondActor = actorFactory
2061                         .createActor(Props.create(MockRespondActor.class, ChangeServersVotingStatus.class,
2062                                 new ServerChangeReply(ServerChangeStatus.NO_LEADER, null)), memberId);
2063
2064                 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
2065
2066                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
2067                 shardManager.tell(new ActorInitialized(), respondActor);
2068                 shardManager.tell(new RoleChangeNotification(memberId, null, RaftState.Follower.name()), respondActor);
2069
2070                 shardManager.tell(
2071                         new ChangeShardMembersVotingStatus("default", ImmutableMap.of("member-2", Boolean.TRUE)),
2072                         getRef());
2073
2074                 MessageCollectorActor.expectFirstMatching(respondActor, ChangeServersVotingStatus.class);
2075
2076                 Status.Failure resp = expectMsgClass(duration("5 seconds"), Status.Failure.class);
2077                 assertEquals("Failure resposnse", true, resp.cause() instanceof NoShardLeaderException);
2078             }
2079         };
2080     }
2081
2082     public static class TestShardManager extends ShardManager {
2083         private final CountDownLatch recoveryComplete = new CountDownLatch(1);
2084         private final CountDownLatch snapshotPersist = new CountDownLatch(1);
2085         private ShardManagerSnapshot snapshot;
2086         private final Map<String, ActorRef> shardActors;
2087         private final ActorRef shardActor;
2088         private CountDownLatch findPrimaryMessageReceived = new CountDownLatch(1);
2089         private CountDownLatch memberUpReceived = new CountDownLatch(1);
2090         private CountDownLatch memberRemovedReceived = new CountDownLatch(1);
2091         private CountDownLatch memberUnreachableReceived = new CountDownLatch(1);
2092         private CountDownLatch memberReachableReceived = new CountDownLatch(1);
2093         private volatile MessageInterceptor messageInterceptor;
2094
2095         private TestShardManager(final Builder builder) {
2096             super(builder);
2097             shardActor = builder.shardActor;
2098             shardActors = builder.shardActors;
2099         }
2100
2101         @Override
2102         protected void handleRecover(final Object message) throws Exception {
2103             try {
2104                 super.handleRecover(message);
2105             } finally {
2106                 if (message instanceof RecoveryCompleted) {
2107                     recoveryComplete.countDown();
2108                 }
2109             }
2110         }
2111
2112         private void countDownIfOther(final Member member, final CountDownLatch latch) {
2113             if (!getCluster().getCurrentMemberName().equals(memberToName(member))) {
2114                 latch.countDown();
2115             }
2116         }
2117
2118         @Override
2119         public void handleCommand(final Object message) throws Exception {
2120             try {
2121                 if (messageInterceptor != null && messageInterceptor.canIntercept(message)) {
2122                     getSender().tell(messageInterceptor.apply(message), getSelf());
2123                 } else {
2124                     super.handleCommand(message);
2125                 }
2126             } finally {
2127                 if (message instanceof FindPrimary) {
2128                     findPrimaryMessageReceived.countDown();
2129                 } else if (message instanceof ClusterEvent.MemberUp) {
2130                     countDownIfOther(((ClusterEvent.MemberUp) message).member(), memberUpReceived);
2131                 } else if (message instanceof ClusterEvent.MemberRemoved) {
2132                     countDownIfOther(((ClusterEvent.MemberRemoved) message).member(), memberRemovedReceived);
2133                 } else if (message instanceof ClusterEvent.UnreachableMember) {
2134                     countDownIfOther(((ClusterEvent.UnreachableMember) message).member(), memberUnreachableReceived);
2135                 } else if (message instanceof ClusterEvent.ReachableMember) {
2136                     countDownIfOther(((ClusterEvent.ReachableMember) message).member(), memberReachableReceived);
2137                 }
2138             }
2139         }
2140
2141         void setMessageInterceptor(final MessageInterceptor messageInterceptor) {
2142             this.messageInterceptor = messageInterceptor;
2143         }
2144
2145         void waitForRecoveryComplete() {
2146             assertEquals("Recovery complete", true,
2147                     Uninterruptibles.awaitUninterruptibly(recoveryComplete, 5, TimeUnit.SECONDS));
2148         }
2149
2150         public void waitForMemberUp() {
2151             assertEquals("MemberUp received", true,
2152                     Uninterruptibles.awaitUninterruptibly(memberUpReceived, 5, TimeUnit.SECONDS));
2153             memberUpReceived = new CountDownLatch(1);
2154         }
2155
2156         void waitForMemberRemoved() {
2157             assertEquals("MemberRemoved received", true,
2158                     Uninterruptibles.awaitUninterruptibly(memberRemovedReceived, 5, TimeUnit.SECONDS));
2159             memberRemovedReceived = new CountDownLatch(1);
2160         }
2161
2162         void waitForUnreachableMember() {
2163             assertEquals("UnreachableMember received", true,
2164                 Uninterruptibles.awaitUninterruptibly(memberUnreachableReceived, 5, TimeUnit.SECONDS
2165                 ));
2166             memberUnreachableReceived = new CountDownLatch(1);
2167         }
2168
2169         void waitForReachableMember() {
2170             assertEquals("ReachableMember received", true,
2171                 Uninterruptibles.awaitUninterruptibly(memberReachableReceived, 5, TimeUnit.SECONDS));
2172             memberReachableReceived = new CountDownLatch(1);
2173         }
2174
2175         void verifyFindPrimary() {
2176             assertEquals("FindPrimary received", true,
2177                     Uninterruptibles.awaitUninterruptibly(findPrimaryMessageReceived, 5, TimeUnit.SECONDS));
2178             findPrimaryMessageReceived = new CountDownLatch(1);
2179         }
2180
2181         public static Builder builder(final DatastoreContext.Builder datastoreContextBuilder) {
2182             return new Builder(datastoreContextBuilder);
2183         }
2184
2185         public static class Builder extends AbstractGenericCreator<Builder, TestShardManager> {
2186             private ActorRef shardActor;
2187             private final Map<String, ActorRef> shardActors = new HashMap<>();
2188
2189             Builder(final DatastoreContext.Builder datastoreContextBuilder) {
2190                 super(TestShardManager.class);
2191                 datastoreContextFactory(newDatastoreContextFactory(datastoreContextBuilder.build()));
2192             }
2193
2194             Builder shardActor(final ActorRef newShardActor) {
2195                 this.shardActor = newShardActor;
2196                 return this;
2197             }
2198
2199             Builder addShardActor(final String shardName, final ActorRef actorRef) {
2200                 shardActors.put(shardName, actorRef);
2201                 return this;
2202             }
2203         }
2204
2205         @Override
2206         public void saveSnapshot(final Object obj) {
2207             snapshot = (ShardManagerSnapshot) obj;
2208             snapshotPersist.countDown();
2209             super.saveSnapshot(obj);
2210         }
2211
2212         void verifySnapshotPersisted(final Set<String> shardList) {
2213             assertEquals("saveSnapshot invoked", true,
2214                     Uninterruptibles.awaitUninterruptibly(snapshotPersist, 5, TimeUnit.SECONDS));
2215             assertEquals("Shard Persisted", shardList, Sets.newHashSet(snapshot.getShardList()));
2216         }
2217
2218         @Override
2219         protected ActorRef newShardActor(final ShardInformation info) {
2220             if (shardActors.get(info.getShardName()) != null) {
2221                 return shardActors.get(info.getShardName());
2222             }
2223
2224             if (shardActor != null) {
2225                 return shardActor;
2226             }
2227
2228             return super.newShardActor(info);
2229         }
2230     }
2231
2232     private abstract static class AbstractGenericCreator<T extends AbstractGenericCreator<T, ?>, C extends ShardManager>
2233                                                      extends AbstractShardManagerCreator<T> {
2234         private final Class<C> shardManagerClass;
2235
2236         AbstractGenericCreator(final Class<C> shardManagerClass) {
2237             this.shardManagerClass = shardManagerClass;
2238             cluster(new MockClusterWrapper()).configuration(new MockConfiguration()).waitTillReadyCountDownLatch(ready)
2239                     .primaryShardInfoCache(new PrimaryShardInfoFutureCache());
2240         }
2241
2242         @Override
2243         public Props props() {
2244             verify();
2245             return Props.create(shardManagerClass, this);
2246         }
2247     }
2248
2249     private static class GenericCreator<C extends ShardManager> extends AbstractGenericCreator<GenericCreator<C>, C> {
2250         GenericCreator(final Class<C> shardManagerClass) {
2251             super(shardManagerClass);
2252         }
2253     }
2254
2255     private static class DelegatingShardManagerCreator implements Creator<ShardManager> {
2256         private static final long serialVersionUID = 1L;
2257         private final Creator<ShardManager> delegate;
2258
2259         DelegatingShardManagerCreator(final Creator<ShardManager> delegate) {
2260             this.delegate = delegate;
2261         }
2262
2263         @Override
2264         public ShardManager create() throws Exception {
2265             return delegate.create();
2266         }
2267     }
2268
2269     interface MessageInterceptor extends Function<Object, Object> {
2270         boolean canIntercept(Object message);
2271     }
2272
2273     private static MessageInterceptor newFindPrimaryInterceptor(final ActorRef primaryActor) {
2274         return new MessageInterceptor() {
2275             @Override
2276             public Object apply(final Object message) {
2277                 return new RemotePrimaryShardFound(Serialization.serializedActorPath(primaryActor), (short) 1);
2278             }
2279
2280             @Override
2281             public boolean canIntercept(final Object message) {
2282                 return message instanceof FindPrimary;
2283             }
2284         };
2285     }
2286
2287     private static class MockRespondActor extends MessageCollectorActor {
2288         static final String CLEAR_RESPONSE = "clear-response";
2289
2290         private Object responseMsg;
2291         private final Class<?> requestClass;
2292
2293         @SuppressWarnings("unused")
2294         MockRespondActor(final Class<?> requestClass, final Object responseMsg) {
2295             this.requestClass = requestClass;
2296             this.responseMsg = responseMsg;
2297         }
2298
2299         @Override
2300         public void onReceive(final Object message) throws Exception {
2301             if (message.equals(CLEAR_RESPONSE)) {
2302                 responseMsg = null;
2303             } else {
2304                 super.onReceive(message);
2305                 if (message.getClass().equals(requestClass) && responseMsg != null) {
2306                     getSender().tell(responseMsg, getSelf());
2307                 }
2308             }
2309         }
2310     }
2311 }