opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/shardmanager/ShardManagerTest.java

   1 /*
   2  * Copyright (c) 2014, 2015 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.controller.cluster.datastore.shardmanager;
  10
  11 import static org.junit.Assert.assertEquals;
  12 import static org.junit.Assert.assertNotNull;
  13 import static org.junit.Assert.assertNull;
  14 import static org.junit.Assert.assertSame;
  15 import static org.junit.Assert.assertTrue;
  16 import static org.junit.Assert.fail;
  17 import static org.mockito.Mockito.mock;
  18 import static org.mockito.Mockito.never;
  19 import static org.mockito.Mockito.times;
  20 import static org.mockito.Mockito.verify;
  21
  22 import akka.actor.ActorRef;
  23 import akka.actor.ActorSystem;
  24 import akka.actor.AddressFromURIString;
  25 import akka.actor.Props;
  26 import akka.actor.Status;
  27 import akka.actor.Status.Failure;
  28 import akka.actor.Status.Success;
  29 import akka.cluster.Cluster;
  30 import akka.cluster.ClusterEvent;
  31 import akka.cluster.Member;
  32 import akka.dispatch.Dispatchers;
  33 import akka.dispatch.OnComplete;
  34 import akka.japi.Creator;
  35 import akka.pattern.Patterns;
  36 import akka.persistence.RecoveryCompleted;
  37 import akka.serialization.Serialization;
  38 import akka.testkit.TestActorRef;
  39 import akka.testkit.javadsl.TestKit;
  40 import akka.util.Timeout;
  41 import com.google.common.base.Function;
  42 import com.google.common.base.Stopwatch;
  43 import com.google.common.collect.ImmutableMap;
  44 import com.google.common.collect.Lists;
  45 import com.google.common.collect.Sets;
  46 import com.google.common.util.concurrent.Uninterruptibles;
  47 import java.net.URI;
  48 import java.util.AbstractMap;
  49 import java.util.Arrays;
  50 import java.util.Collection;
  51 import java.util.Collections;
  52 import java.util.HashMap;
  53 import java.util.List;
  54 import java.util.Map;
  55 import java.util.Map.Entry;
  56 import java.util.Set;
  57 import java.util.concurrent.CountDownLatch;
  58 import java.util.concurrent.TimeUnit;
  59 import java.util.concurrent.TimeoutException;
  60 import java.util.stream.Collectors;
  61 import org.junit.AfterClass;
  62 import org.junit.BeforeClass;
  63 import org.junit.Test;
  64 import org.mockito.Mockito;
  65 import org.opendaylight.controller.cluster.access.concepts.MemberName;
  66 import org.opendaylight.controller.cluster.datastore.AbstractShardManagerTest;
  67 import org.opendaylight.controller.cluster.datastore.ClusterWrapperImpl;
  68 import org.opendaylight.controller.cluster.datastore.DataStoreVersions;
  69 import org.opendaylight.controller.cluster.datastore.DatastoreContext;
  70 import org.opendaylight.controller.cluster.datastore.DatastoreContextFactory;
  71 import org.opendaylight.controller.cluster.datastore.DistributedDataStore;
  72 import org.opendaylight.controller.cluster.datastore.Shard;
  73 import org.opendaylight.controller.cluster.datastore.config.ConfigurationImpl;
  74 import org.opendaylight.controller.cluster.datastore.config.EmptyModuleShardConfigProvider;
  75 import org.opendaylight.controller.cluster.datastore.config.ModuleShardConfiguration;
  76 import org.opendaylight.controller.cluster.datastore.exceptions.AlreadyExistsException;
  77 import org.opendaylight.controller.cluster.datastore.exceptions.NoShardLeaderException;
  78 import org.opendaylight.controller.cluster.datastore.exceptions.NotInitializedException;
  79 import org.opendaylight.controller.cluster.datastore.exceptions.PrimaryNotFoundException;
  80 import org.opendaylight.controller.cluster.datastore.identifiers.ShardIdentifier;
  81 import org.opendaylight.controller.cluster.datastore.identifiers.ShardManagerIdentifier;
  82 import org.opendaylight.controller.cluster.datastore.messages.ActorInitialized;
  83 import org.opendaylight.controller.cluster.datastore.messages.AddShardReplica;
  84 import org.opendaylight.controller.cluster.datastore.messages.ChangeShardMembersVotingStatus;
  85 import org.opendaylight.controller.cluster.datastore.messages.CreateShard;
  86 import org.opendaylight.controller.cluster.datastore.messages.FindLocalShard;
  87 import org.opendaylight.controller.cluster.datastore.messages.FindPrimary;
  88 import org.opendaylight.controller.cluster.datastore.messages.LocalPrimaryShardFound;
  89 import org.opendaylight.controller.cluster.datastore.messages.LocalShardFound;
  90 import org.opendaylight.controller.cluster.datastore.messages.LocalShardNotFound;
  91 import org.opendaylight.controller.cluster.datastore.messages.PeerDown;
  92 import org.opendaylight.controller.cluster.datastore.messages.PeerUp;
  93 import org.opendaylight.controller.cluster.datastore.messages.PrimaryShardInfo;
  94 import org.opendaylight.controller.cluster.datastore.messages.RemotePrimaryShardFound;
  95 import org.opendaylight.controller.cluster.datastore.messages.RemoveShardReplica;
  96 import org.opendaylight.controller.cluster.datastore.messages.ShardLeaderStateChanged;
  97 import org.opendaylight.controller.cluster.datastore.messages.UpdateSchemaContext;
  98 import org.opendaylight.controller.cluster.datastore.persisted.DatastoreSnapshot;
  99 import org.opendaylight.controller.cluster.datastore.persisted.DatastoreSnapshot.ShardSnapshot;
 100 import org.opendaylight.controller.cluster.datastore.persisted.ShardManagerSnapshot;
 101 import org.opendaylight.controller.cluster.datastore.utils.ForwardingActor;
 102 import org.opendaylight.controller.cluster.datastore.utils.MockClusterWrapper;
 103 import org.opendaylight.controller.cluster.datastore.utils.MockConfiguration;
 104 import org.opendaylight.controller.cluster.datastore.utils.PrimaryShardInfoFutureCache;
 105 import org.opendaylight.controller.cluster.notifications.LeaderStateChanged;
 106 import org.opendaylight.controller.cluster.notifications.RegisterRoleChangeListener;
 107 import org.opendaylight.controller.cluster.notifications.RoleChangeNotification;
 108 import org.opendaylight.controller.cluster.raft.RaftState;
 109 import org.opendaylight.controller.cluster.raft.base.messages.FollowerInitialSyncUpStatus;
 110 import org.opendaylight.controller.cluster.raft.base.messages.SwitchBehavior;
 111 import org.opendaylight.controller.cluster.raft.client.messages.GetSnapshot;
 112 import org.opendaylight.controller.cluster.raft.client.messages.Shutdown;
 113 import org.opendaylight.controller.cluster.raft.messages.AddServer;
 114 import org.opendaylight.controller.cluster.raft.messages.AddServerReply;
 115 import org.opendaylight.controller.cluster.raft.messages.ChangeServersVotingStatus;
 116 import org.opendaylight.controller.cluster.raft.messages.RemoveServer;
 117 import org.opendaylight.controller.cluster.raft.messages.RemoveServerReply;
 118 import org.opendaylight.controller.cluster.raft.messages.ServerChangeReply;
 119 import org.opendaylight.controller.cluster.raft.messages.ServerChangeStatus;
 120 import org.opendaylight.controller.cluster.raft.messages.ServerRemoved;
 121 import org.opendaylight.controller.cluster.raft.policy.DisableElectionsRaftPolicy;
 122 import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore;
 123 import org.opendaylight.controller.cluster.raft.utils.MessageCollectorActor;
 124 import org.opendaylight.controller.md.cluster.datastore.model.TestModel;
 125 import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTree;
 126 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
 127 import org.slf4j.Logger;
 128 import org.slf4j.LoggerFactory;
 129 import scala.concurrent.Await;
 130 import scala.concurrent.Future;
 131 import scala.concurrent.duration.FiniteDuration;
 132
 133 public class ShardManagerTest extends AbstractShardManagerTest {
 134     private static final Logger LOG = LoggerFactory.getLogger(ShardManagerTest.class);
 135     private static final MemberName MEMBER_2 = MemberName.forName("member-2");
 136     private static final MemberName MEMBER_3 = MemberName.forName("member-3");
 137
 138     private static SchemaContext TEST_SCHEMA_CONTEXT;
 139
 140     private final String shardMgrID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
 141
 142     @BeforeClass
 143     public static void beforeClass() {
 144         TEST_SCHEMA_CONTEXT = TestModel.createTestContext();
 145     }
 146
 147     @AfterClass
 148     public static void afterClass() {
 149         TEST_SCHEMA_CONTEXT = null;
 150     }
 151
 152     private ActorSystem newActorSystem(final String config) {
 153         return newActorSystem("cluster-test", config);
 154     }
 155
 156     private ActorRef newMockShardActor(final ActorSystem system, final String shardName, final String memberName) {
 157         String name = ShardIdentifier.create(shardName, MemberName.forName(memberName), "config").toString();
 158         if (system == getSystem()) {
 159             return actorFactory.createActor(MessageCollectorActor.props(), name);
 160         }
 161
 162         return system.actorOf(MessageCollectorActor.props(), name);
 163     }
 164
 165     private Props newShardMgrProps() {
 166         return newShardMgrProps(new MockConfiguration());
 167     }
 168
 169     private static DatastoreContextFactory newDatastoreContextFactory(final DatastoreContext datastoreContext) {
 170         DatastoreContextFactory mockFactory = mock(DatastoreContextFactory.class);
 171         Mockito.doReturn(datastoreContext).when(mockFactory).getBaseDatastoreContext();
 172         Mockito.doReturn(datastoreContext).when(mockFactory).getShardDatastoreContext(Mockito.anyString());
 173         return mockFactory;
 174     }
 175
 176     private TestShardManager.Builder newTestShardMgrBuilderWithMockShardActor() {
 177         return newTestShardMgrBuilderWithMockShardActor(mockShardActor);
 178     }
 179
 180     private TestShardManager.Builder newTestShardMgrBuilderWithMockShardActor(final ActorRef shardActor) {
 181         return TestShardManager.builder(datastoreContextBuilder).shardActor(shardActor)
 182                 .distributedDataStore(mock(DistributedDataStore.class));
 183     }
 184
 185
 186     private Props newPropsShardMgrWithMockShardActor() {
 187         return newTestShardMgrBuilderWithMockShardActor().props().withDispatcher(
 188                 Dispatchers.DefaultDispatcherId());
 189     }
 190
 191     private Props newPropsShardMgrWithMockShardActor(final ActorRef shardActor) {
 192         return newTestShardMgrBuilderWithMockShardActor(shardActor).props()
 193                 .withDispatcher(Dispatchers.DefaultDispatcherId());
 194     }
 195
 196
 197     private TestShardManager newTestShardManager() {
 198         return newTestShardManager(newShardMgrProps());
 199     }
 200
 201     private TestShardManager newTestShardManager(final Props props) {
 202         TestActorRef<TestShardManager> shardManagerActor = actorFactory.createTestActor(props);
 203         TestShardManager shardManager = shardManagerActor.underlyingActor();
 204         shardManager.waitForRecoveryComplete();
 205         return shardManager;
 206     }
 207
 208     private static void waitForShardInitialized(final ActorRef shardManager, final String shardName,
 209             final TestKit kit) {
 210         AssertionError last = null;
 211         Stopwatch sw = Stopwatch.createStarted();
 212         while (sw.elapsed(TimeUnit.SECONDS) <= 5) {
 213             try {
 214                 shardManager.tell(new FindLocalShard(shardName, true), kit.getRef());
 215                 kit.expectMsgClass(LocalShardFound.class);
 216                 return;
 217             } catch (AssertionError e) {
 218                 last = e;
 219             }
 220
 221             Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
 222         }
 223
 224         throw last;
 225     }
 226
 227     @SuppressWarnings("unchecked")
 228     private static <T> T expectMsgClassOrFailure(final Class<T> msgClass, final TestKit kit, final String msg) {
 229         Object reply = kit.expectMsgAnyClassOf(kit.duration("5 sec"), msgClass, Failure.class);
 230         if (reply instanceof Failure) {
 231             throw new AssertionError(msg + " failed", ((Failure)reply).cause());
 232         }
 233
 234         return (T)reply;
 235     }
 236
 237     @Test
 238     public void testPerShardDatastoreContext() throws Exception {
 239         LOG.info("testPerShardDatastoreContext starting");
 240         final DatastoreContextFactory mockFactory = newDatastoreContextFactory(
 241                 datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
 242
 243         Mockito.doReturn(
 244                 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(6).build())
 245                 .when(mockFactory).getShardDatastoreContext("default");
 246
 247         Mockito.doReturn(
 248                 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(7).build())
 249                 .when(mockFactory).getShardDatastoreContext("topology");
 250
 251         final MockConfiguration mockConfig = new MockConfiguration() {
 252             @Override
 253             public Collection<String> getMemberShardNames(final MemberName memberName) {
 254                 return Arrays.asList("default", "topology");
 255             }
 256
 257             @Override
 258             public Collection<MemberName> getMembersFromShardName(final String shardName) {
 259                 return members("member-1");
 260             }
 261         };
 262
 263         final ActorRef defaultShardActor = actorFactory.createActor(
 264                 MessageCollectorActor.props(), actorFactory.generateActorId("default"));
 265         final ActorRef topologyShardActor = actorFactory.createActor(
 266                 MessageCollectorActor.props(), actorFactory.generateActorId("topology"));
 267
 268         final Map<String, Entry<ActorRef, DatastoreContext>> shardInfoMap = Collections.synchronizedMap(
 269                 new HashMap<String, Entry<ActorRef, DatastoreContext>>());
 270         shardInfoMap.put("default", new AbstractMap.SimpleEntry<>(defaultShardActor, null));
 271         shardInfoMap.put("topology", new AbstractMap.SimpleEntry<>(topologyShardActor, null));
 272
 273         final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
 274         final CountDownLatch newShardActorLatch = new CountDownLatch(2);
 275         class LocalShardManager extends ShardManager {
 276             LocalShardManager(final AbstractShardManagerCreator<?> creator) {
 277                 super(creator);
 278             }
 279
 280             @Override
 281             protected ActorRef newShardActor(final ShardInformation info) {
 282                 Entry<ActorRef, DatastoreContext> entry = shardInfoMap.get(info.getShardName());
 283                 ActorRef ref = null;
 284                 if (entry != null) {
 285                     ref = entry.getKey();
 286                     entry.setValue(info.getDatastoreContext());
 287                 }
 288
 289                 newShardActorLatch.countDown();
 290                 return ref;
 291             }
 292         }
 293
 294         final Creator<ShardManager> creator = new Creator<ShardManager>() {
 295             private static final long serialVersionUID = 1L;
 296             @Override
 297             public ShardManager create() {
 298                 return new LocalShardManager(
 299                         new GenericCreator<>(LocalShardManager.class).datastoreContextFactory(mockFactory)
 300                                 .primaryShardInfoCache(primaryShardInfoCache).configuration(mockConfig));
 301             }
 302         };
 303
 304         TestKit kit = new TestKit(getSystem());
 305
 306         final ActorRef shardManager = actorFactory.createActor(Props.create(
 307                 new DelegatingShardManagerCreator(creator)).withDispatcher(Dispatchers.DefaultDispatcherId()));
 308
 309         shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), kit.getRef());
 310
 311         assertEquals("Shard actors created", true, newShardActorLatch.await(5, TimeUnit.SECONDS));
 312         assertEquals("getShardElectionTimeoutFactor", 6,
 313                 shardInfoMap.get("default").getValue().getShardElectionTimeoutFactor());
 314         assertEquals("getShardElectionTimeoutFactor", 7,
 315                 shardInfoMap.get("topology").getValue().getShardElectionTimeoutFactor());
 316
 317         DatastoreContextFactory newMockFactory = newDatastoreContextFactory(
 318                 datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
 319         Mockito.doReturn(
 320                 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(66).build())
 321                 .when(newMockFactory).getShardDatastoreContext("default");
 322
 323         Mockito.doReturn(
 324                 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(77).build())
 325                 .when(newMockFactory).getShardDatastoreContext("topology");
 326
 327         shardManager.tell(newMockFactory, kit.getRef());
 328
 329         DatastoreContext newContext = MessageCollectorActor.expectFirstMatching(defaultShardActor,
 330                 DatastoreContext.class);
 331         assertEquals("getShardElectionTimeoutFactor", 66, newContext.getShardElectionTimeoutFactor());
 332
 333         newContext = MessageCollectorActor.expectFirstMatching(topologyShardActor, DatastoreContext.class);
 334         assertEquals("getShardElectionTimeoutFactor", 77, newContext.getShardElectionTimeoutFactor());
 335
 336         LOG.info("testPerShardDatastoreContext ending");
 337     }
 338
 339     @Test
 340     public void testOnReceiveFindPrimaryForNonExistentShard() {
 341         new TestKit(getSystem()) {
 342             {
 343                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 344
 345                 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
 346
 347                 shardManager.tell(new FindPrimary("non-existent", false), getRef());
 348
 349                 expectMsgClass(duration("5 seconds"), PrimaryNotFoundException.class);
 350             }
 351         };
 352     }
 353
 354     @Test
 355     public void testOnReceiveFindPrimaryForLocalLeaderShard() {
 356         LOG.info("testOnReceiveFindPrimaryForLocalLeaderShard starting");
 357         new TestKit(getSystem()) {
 358             {
 359                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
 360
 361                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 362
 363                 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
 364                 shardManager.tell(new ActorInitialized(), mockShardActor);
 365
 366                 DataTree mockDataTree = mock(DataTree.class);
 367                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
 368                         DataStoreVersions.CURRENT_VERSION), getRef());
 369
 370                 MessageCollectorActor.expectFirstMatching(mockShardActor, RegisterRoleChangeListener.class);
 371                 shardManager.tell(
 372                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
 373                         mockShardActor);
 374
 375                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 376
 377                 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
 378                         LocalPrimaryShardFound.class);
 379                 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
 380                         primaryFound.getPrimaryPath().contains("member-1-shard-default"));
 381                 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
 382             }
 383         };
 384
 385         LOG.info("testOnReceiveFindPrimaryForLocalLeaderShard ending");
 386     }
 387
 388     @Test
 389     public void testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp() {
 390         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp starting");
 391         new TestKit(getSystem()) {
 392             {
 393                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 394
 395                 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
 396                 shardManager.tell(new ActorInitialized(), mockShardActor);
 397
 398                 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
 399                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
 400                 shardManager.tell(
 401                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
 402                         mockShardActor);
 403                 shardManager.tell(new LeaderStateChanged(memberId1, memberId2, DataStoreVersions.CURRENT_VERSION),
 404                         mockShardActor);
 405
 406                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 407
 408                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 409             }
 410         };
 411
 412         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp ending");
 413     }
 414
 415     @Test
 416     public void testOnReceiveFindPrimaryForNonLocalLeaderShard() {
 417         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShard starting");
 418         new TestKit(getSystem()) {
 419             {
 420                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 421
 422                 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
 423                 shardManager.tell(new ActorInitialized(), mockShardActor);
 424
 425                 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
 426                 MockClusterWrapper.sendMemberUp(shardManager, "member-2", getRef().path().toString());
 427
 428                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
 429                 shardManager.tell(
 430                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
 431                         mockShardActor);
 432                 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
 433                 shardManager.tell(new ShardLeaderStateChanged(memberId1, memberId2, leaderVersion), mockShardActor);
 434
 435                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 436
 437                 RemotePrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
 438                         RemotePrimaryShardFound.class);
 439                 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
 440                         primaryFound.getPrimaryPath().contains("member-2-shard-default"));
 441                 assertEquals("getPrimaryVersion", leaderVersion, primaryFound.getPrimaryVersion());
 442             }
 443         };
 444
 445         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShard ending");
 446     }
 447
 448     @Test
 449     public void testOnReceiveFindPrimaryForUninitializedShard() {
 450         new TestKit(getSystem()) {
 451             {
 452                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 453
 454                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 455
 456                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
 457             }
 458         };
 459     }
 460
 461     @Test
 462     public void testOnReceiveFindPrimaryForInitializedShardWithNoRole() {
 463         new TestKit(getSystem()) {
 464             {
 465                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 466
 467                 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
 468                 shardManager.tell(new ActorInitialized(), mockShardActor);
 469
 470                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 471
 472                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 473             }
 474         };
 475     }
 476
 477     @Test
 478     public void testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId() {
 479         LOG.info("testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId starting");
 480         new TestKit(getSystem()) {
 481             {
 482                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 483
 484                 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
 485                 shardManager.tell(new ActorInitialized(), mockShardActor);
 486
 487                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
 488                 shardManager.tell(
 489                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Follower.name()),
 490                         mockShardActor);
 491
 492                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 493
 494                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 495
 496                 DataTree mockDataTree = mock(DataTree.class);
 497                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
 498                         DataStoreVersions.CURRENT_VERSION), mockShardActor);
 499
 500                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 501
 502                 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
 503                         LocalPrimaryShardFound.class);
 504                 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
 505                         primaryFound.getPrimaryPath().contains("member-1-shard-default"));
 506                 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
 507             }
 508         };
 509
 510         LOG.info("testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId starting");
 511     }
 512
 513     @Test
 514     public void testOnReceiveFindPrimaryWaitForShardLeader() {
 515         LOG.info("testOnReceiveFindPrimaryWaitForShardLeader starting");
 516         datastoreContextBuilder.shardInitializationTimeout(10, TimeUnit.SECONDS);
 517         new TestKit(getSystem()) {
 518             {
 519                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 520
 521                 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
 522
 523                 // We're passing waitUntilInitialized = true to FindPrimary so
 524                 // the response should be
 525                 // delayed until we send ActorInitialized and
 526                 // RoleChangeNotification.
 527                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
 528
 529                 expectNoMessage(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
 530
 531                 shardManager.tell(new ActorInitialized(), mockShardActor);
 532
 533                 expectNoMessage(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
 534
 535                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
 536                 shardManager.tell(
 537                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
 538                         mockShardActor);
 539
 540                 expectNoMessage(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
 541
 542                 DataTree mockDataTree = mock(DataTree.class);
 543                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
 544                         DataStoreVersions.CURRENT_VERSION), mockShardActor);
 545
 546                 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
 547                         LocalPrimaryShardFound.class);
 548                 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
 549                         primaryFound.getPrimaryPath().contains("member-1-shard-default"));
 550                 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
 551
 552                 expectNoMessage(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
 553             }
 554         };
 555
 556         LOG.info("testOnReceiveFindPrimaryWaitForShardLeader ending");
 557     }
 558
 559     @Test
 560     public void testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard() {
 561         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard starting");
 562         new TestKit(getSystem()) {
 563             {
 564                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 565
 566                 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
 567
 568                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
 569
 570                 expectMsgClass(duration("2 seconds"), NotInitializedException.class);
 571
 572                 shardManager.tell(new ActorInitialized(), mockShardActor);
 573
 574                 expectNoMessage(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
 575             }
 576         };
 577
 578         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard ending");
 579     }
 580
 581     @Test
 582     public void testOnReceiveFindPrimaryWaitForReadyWithCandidateShard() {
 583         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithCandidateShard starting");
 584         new TestKit(getSystem()) {
 585             {
 586                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 587
 588                 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
 589                 shardManager.tell(new ActorInitialized(), mockShardActor);
 590                 shardManager.tell(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix, null,
 591                         RaftState.Candidate.name()), mockShardActor);
 592
 593                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
 594
 595                 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
 596             }
 597         };
 598
 599         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithCandidateShard ending");
 600     }
 601
 602     @Test
 603     public void testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard() {
 604         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard starting");
 605         new TestKit(getSystem()) {
 606             {
 607                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 608
 609                 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
 610                 shardManager.tell(new ActorInitialized(), mockShardActor);
 611                 shardManager.tell(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix, null,
 612                         RaftState.IsolatedLeader.name()), mockShardActor);
 613
 614                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
 615
 616                 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
 617             }
 618         };
 619
 620         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard ending");
 621     }
 622
 623     @Test
 624     public void testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard() {
 625         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard starting");
 626         new TestKit(getSystem()) {
 627             {
 628                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 629
 630                 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
 631                 shardManager.tell(new ActorInitialized(), mockShardActor);
 632
 633                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
 634
 635                 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
 636             }
 637         };
 638
 639         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard ending");
 640     }
 641
 642     @Test
 643     public void testOnReceiveFindPrimaryForRemoteShard() {
 644         LOG.info("testOnReceiveFindPrimaryForRemoteShard starting");
 645         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
 646
 647         // Create an ActorSystem ShardManager actor for member-1.
 648
 649         final ActorSystem system1 = newActorSystem("Member1");
 650         Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
 651
 652         final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
 653                 newTestShardMgrBuilderWithMockShardActor().cluster(
 654                         new ClusterWrapperImpl(system1)).props().withDispatcher(
 655                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
 656
 657         // Create an ActorSystem ShardManager actor for member-2.
 658
 659         final ActorSystem system2 = newActorSystem("Member2");
 660
 661         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
 662
 663         final ActorRef mockShardActor2 = newMockShardActor(system2, "astronauts", "member-2");
 664
 665         MockConfiguration mockConfig2 = new MockConfiguration(
 666                 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
 667                         .put("astronauts", Arrays.asList("member-2")).build());
 668
 669         final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
 670                 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
 671                         new ClusterWrapperImpl(system2)).props().withDispatcher(
 672                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
 673
 674         new TestKit(system1) {
 675             {
 676                 shardManager1.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
 677                 shardManager2.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
 678
 679                 shardManager2.tell(new ActorInitialized(), mockShardActor2);
 680
 681                 String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
 682                 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
 683                 shardManager2.tell(
 684                         new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
 685                         mockShardActor2);
 686                 shardManager2.tell(
 687                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
 688                         mockShardActor2);
 689
 690                 shardManager1.underlyingActor().waitForMemberUp();
 691                 shardManager1.tell(new FindPrimary("astronauts", false), getRef());
 692
 693                 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 694                 String path = found.getPrimaryPath();
 695                 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-astronauts-config"));
 696                 assertEquals("getPrimaryVersion", leaderVersion, found.getPrimaryVersion());
 697
 698                 shardManager2.underlyingActor().verifyFindPrimary();
 699
 700                 // This part times out quite a bit on jenkins for some reason
 701
 702 //                Cluster.get(system2).down(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
 703 //
 704 //                shardManager1.underlyingActor().waitForMemberRemoved();
 705 //
 706 //                shardManager1.tell(new FindPrimary("astronauts", false), getRef());
 707 //
 708 //                expectMsgClass(duration("5 seconds"), PrimaryNotFoundException.class);
 709             }
 710         };
 711
 712         LOG.info("testOnReceiveFindPrimaryForRemoteShard ending");
 713     }
 714
 715     @Test
 716     public void testShardAvailabilityOnChangeOfMemberReachability() {
 717         LOG.info("testShardAvailabilityOnChangeOfMemberReachability starting");
 718         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
 719
 720         // Create an ActorSystem ShardManager actor for member-1.
 721
 722         final ActorSystem system1 = newActorSystem("Member1");
 723         Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
 724
 725         final ActorRef mockShardActor1 = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
 726
 727         final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
 728                 newTestShardMgrBuilder().shardActor(mockShardActor1).cluster(
 729                         new ClusterWrapperImpl(system1)).props().withDispatcher(
 730                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
 731
 732         // Create an ActorSystem ShardManager actor for member-2.
 733
 734         final ActorSystem system2 = newActorSystem("Member2");
 735
 736         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
 737
 738         final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
 739
 740         MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
 741                 .put("default", Arrays.asList("member-1", "member-2")).build());
 742
 743         final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
 744                 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
 745                         new ClusterWrapperImpl(system2)).props().withDispatcher(
 746                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
 747
 748         new TestKit(system1) {
 749             {
 750                 shardManager1.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
 751                 shardManager2.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
 752                 shardManager1.tell(new ActorInitialized(), mockShardActor1);
 753                 shardManager2.tell(new ActorInitialized(), mockShardActor2);
 754
 755                 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
 756                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
 757                 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class),
 758                         DataStoreVersions.CURRENT_VERSION), mockShardActor1);
 759                 shardManager1.tell(
 760                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
 761                         mockShardActor1);
 762                 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class),
 763                         DataStoreVersions.CURRENT_VERSION), mockShardActor2);
 764                 shardManager2.tell(
 765                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
 766                         mockShardActor2);
 767                 shardManager1.underlyingActor().waitForMemberUp();
 768
 769                 shardManager1.tell(new FindPrimary("default", true), getRef());
 770
 771                 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 772                 String path = found.getPrimaryPath();
 773                 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
 774
 775                 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
 776                         "akka://cluster-test@127.0.0.1:2558"), getRef());
 777
 778                 shardManager1.underlyingActor().waitForUnreachableMember();
 779
 780                 PeerDown peerDown = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
 781                 assertEquals("getMemberName", MEMBER_2, peerDown.getMemberName());
 782                 MessageCollectorActor.clearMessages(mockShardActor1);
 783
 784                 shardManager1.tell(
 785                         MockClusterWrapper.createMemberRemoved("member-2", "akka://cluster-test@127.0.0.1:2558"),
 786                         getRef());
 787
 788                 MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
 789
 790                 shardManager1.tell(new FindPrimary("default", true), getRef());
 791
 792                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 793
 794                 shardManager1.tell(
 795                         MockClusterWrapper.createReachableMember("member-2", "akka://cluster-test@127.0.0.1:2558"),
 796                         getRef());
 797
 798                 shardManager1.underlyingActor().waitForReachableMember();
 799
 800                 PeerUp peerUp = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
 801                 assertEquals("getMemberName", MEMBER_2, peerUp.getMemberName());
 802                 MessageCollectorActor.clearMessages(mockShardActor1);
 803
 804                 shardManager1.tell(new FindPrimary("default", true), getRef());
 805
 806                 RemotePrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 807                 String path1 = found1.getPrimaryPath();
 808                 assertTrue("Unexpected primary path " + path1, path1.contains("member-2-shard-default-config"));
 809
 810                 shardManager1.tell(
 811                         MockClusterWrapper.createMemberUp("member-2", "akka://cluster-test@127.0.0.1:2558"),
 812                         getRef());
 813
 814                 MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
 815
 816                 // Test FindPrimary wait succeeds after reachable member event.
 817
 818                 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
 819                         "akka://cluster-test@127.0.0.1:2558"), getRef());
 820                 shardManager1.underlyingActor().waitForUnreachableMember();
 821
 822                 shardManager1.tell(new FindPrimary("default", true), getRef());
 823
 824                 shardManager1.tell(
 825                         MockClusterWrapper.createReachableMember("member-2", "akka://cluster-test@127.0.0.1:2558"),
 826                         getRef());
 827
 828                 RemotePrimaryShardFound found2 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 829                 String path2 = found2.getPrimaryPath();
 830                 assertTrue("Unexpected primary path " + path2, path2.contains("member-2-shard-default-config"));
 831             }
 832         };
 833
 834         LOG.info("testShardAvailabilityOnChangeOfMemberReachability ending");
 835     }
 836
 837     @Test
 838     public void testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange() {
 839         LOG.info("testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange starting");
 840         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
 841
 842         // Create an ActorSystem ShardManager actor for member-1.
 843
 844         final ActorSystem system1 = newActorSystem("Member1");
 845         Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
 846
 847         final ActorRef mockShardActor1 = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
 848
 849         final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
 850         final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
 851                 newTestShardMgrBuilder().shardActor(mockShardActor1).cluster(new ClusterWrapperImpl(system1))
 852                         .primaryShardInfoCache(primaryShardInfoCache).props()
 853                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
 854                 shardManagerID);
 855
 856         // Create an ActorSystem ShardManager actor for member-2.
 857
 858         final ActorSystem system2 = newActorSystem("Member2");
 859
 860         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
 861
 862         final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
 863
 864         MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
 865                 .put("default", Arrays.asList("member-1", "member-2")).build());
 866
 867         final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
 868                 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
 869                         new ClusterWrapperImpl(system2)).props().withDispatcher(
 870                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
 871
 872         new TestKit(system1) {
 873             {
 874                 shardManager1.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
 875                 shardManager2.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
 876                 shardManager1.tell(new ActorInitialized(), mockShardActor1);
 877                 shardManager2.tell(new ActorInitialized(), mockShardActor2);
 878
 879                 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
 880                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
 881                 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class),
 882                         DataStoreVersions.CURRENT_VERSION), mockShardActor1);
 883                 shardManager1.tell(
 884                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
 885                         mockShardActor1);
 886                 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class),
 887                         DataStoreVersions.CURRENT_VERSION), mockShardActor2);
 888                 shardManager2.tell(
 889                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
 890                         mockShardActor2);
 891                 shardManager1.underlyingActor().waitForMemberUp();
 892
 893                 shardManager1.tell(new FindPrimary("default", true), getRef());
 894
 895                 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 896                 String path = found.getPrimaryPath();
 897                 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
 898
 899                 primaryShardInfoCache.putSuccessful("default", new PrimaryShardInfo(
 900                         system1.actorSelection(mockShardActor1.path()), DataStoreVersions.CURRENT_VERSION));
 901
 902                 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
 903                         "akka://cluster-test@127.0.0.1:2558"), getRef());
 904
 905                 shardManager1.underlyingActor().waitForUnreachableMember();
 906
 907                 shardManager1.tell(new FindPrimary("default", true), getRef());
 908
 909                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 910
 911                 assertNull("Expected primaryShardInfoCache entry removed",
 912                         primaryShardInfoCache.getIfPresent("default"));
 913
 914                 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId1, mock(DataTree.class),
 915                         DataStoreVersions.CURRENT_VERSION), mockShardActor1);
 916                 shardManager1.tell(
 917                         new RoleChangeNotification(memberId1, RaftState.Follower.name(), RaftState.Leader.name()),
 918                         mockShardActor1);
 919
 920                 shardManager1.tell(new FindPrimary("default", true), getRef());
 921
 922                 LocalPrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
 923                 String path1 = found1.getPrimaryPath();
 924                 assertTrue("Unexpected primary path " + path1, path1.contains("member-1-shard-default-config"));
 925
 926             }
 927         };
 928
 929         LOG.info("testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange ending");
 930     }
 931
 932     @Test
 933     public void testShardAvailabilityChangeOnMemberWithNameContainedInLeaderIdUnreachable() {
 934         LOG.info("testShardAvailabilityChangeOnMemberWithNameContainedInLeaderIdUnreachable starting");
 935         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
 936
 937         MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
 938                 .put("default", Arrays.asList("member-256", "member-2")).build());
 939
 940         // Create an ActorSystem, ShardManager and actor for member-256.
 941
 942         final ActorSystem system256 = newActorSystem("Member256");
 943         // 2562 is the tcp port of Member256 in src/test/resources/application.conf.
 944         Cluster.get(system256).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2562"));
 945
 946         final ActorRef mockShardActor256 = newMockShardActor(system256, Shard.DEFAULT_NAME, "member-256");
 947
 948         final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
 949
 950         // ShardManager must be created with shard configuration to let its localShards has shards.
 951         final TestActorRef<TestShardManager> shardManager256 = TestActorRef.create(system256,
 952                 newTestShardMgrBuilder(mockConfig).shardActor(mockShardActor256)
 953                         .cluster(new ClusterWrapperImpl(system256))
 954                         .primaryShardInfoCache(primaryShardInfoCache).props()
 955                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
 956                 shardManagerID);
 957
 958         // Create an ActorSystem, ShardManager and actor for member-2 whose name is contained in member-256.
 959
 960         final ActorSystem system2 = newActorSystem("Member2");
 961
 962         // Join member-2 into the cluster of member-256.
 963         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2562"));
 964
 965         final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
 966
 967         final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
 968                 newTestShardMgrBuilder(mockConfig).shardActor(mockShardActor2).cluster(
 969                         new ClusterWrapperImpl(system2)).props().withDispatcher(
 970                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
 971
 972         new TestKit(system256) {
 973             {
 974                 shardManager256.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
 975                 shardManager2.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
 976                 shardManager256.tell(new ActorInitialized(), mockShardActor256);
 977                 shardManager2.tell(new ActorInitialized(), mockShardActor2);
 978
 979                 String memberId256 = "member-256-shard-default-" + shardMrgIDSuffix;
 980                 String memberId2   = "member-2-shard-default-"   + shardMrgIDSuffix;
 981                 shardManager256.tell(new ShardLeaderStateChanged(memberId256, memberId256, mock(DataTree.class),
 982                         DataStoreVersions.CURRENT_VERSION), mockShardActor256);
 983                 shardManager256.tell(
 984                         new RoleChangeNotification(memberId256, RaftState.Candidate.name(), RaftState.Leader.name()),
 985                         mockShardActor256);
 986                 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId256, mock(DataTree.class),
 987                         DataStoreVersions.CURRENT_VERSION), mockShardActor2);
 988                 shardManager2.tell(
 989                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Follower.name()),
 990                         mockShardActor2);
 991                 shardManager256.underlyingActor().waitForMemberUp();
 992
 993                 shardManager256.tell(new FindPrimary("default", true), getRef());
 994
 995                 LocalPrimaryShardFound found = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
 996                 String path = found.getPrimaryPath();
 997                 assertTrue("Unexpected primary path " + path + " which must on member-256",
 998                             path.contains("member-256-shard-default-config"));
 999
1000                 PrimaryShardInfo primaryShardInfo = new PrimaryShardInfo(
1001                         system256.actorSelection(mockShardActor256.path()), DataStoreVersions.CURRENT_VERSION);
1002                 primaryShardInfoCache.putSuccessful("default", primaryShardInfo);
1003
1004                 // Simulate member-2 become unreachable.
1005                 shardManager256.tell(MockClusterWrapper.createUnreachableMember("member-2",
1006                         "akka://cluster-test@127.0.0.1:2558"), getRef());
1007                 shardManager256.underlyingActor().waitForUnreachableMember();
1008
1009                 // Make sure leader shard on member-256 is still leader and still in the cache.
1010                 shardManager256.tell(new FindPrimary("default", true), getRef());
1011                 found = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
1012                 path = found.getPrimaryPath();
1013                 assertTrue("Unexpected primary path " + path + " which must still not on member-256",
1014                             path.contains("member-256-shard-default-config"));
1015                 Future<PrimaryShardInfo> futurePrimaryShard = primaryShardInfoCache.getIfPresent("default");
1016                 futurePrimaryShard.onComplete(new OnComplete<PrimaryShardInfo>() {
1017                     @Override
1018                     public void onComplete(final Throwable failure, final PrimaryShardInfo futurePrimaryShardInfo) {
1019                         if (failure != null) {
1020                             assertTrue("Primary shard info is unexpectedly removed from primaryShardInfoCache", false);
1021                         } else {
1022                             assertEquals("Expected primaryShardInfoCache entry",
1023                                         primaryShardInfo, futurePrimaryShardInfo);
1024                         }
1025                     }
1026                 }, system256.dispatchers().defaultGlobalDispatcher());
1027             }
1028         };
1029
1030         LOG.info("testShardAvailabilityChangeOnMemberWithNameContainedInLeaderIdUnreachable ending");
1031     }
1032
1033     @Test
1034     public void testOnReceiveFindLocalShardForNonExistentShard() {
1035         new TestKit(getSystem()) {
1036             {
1037                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1038
1039                 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1040
1041                 shardManager.tell(new FindLocalShard("non-existent", false), getRef());
1042
1043                 LocalShardNotFound notFound = expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
1044
1045                 assertEquals("getShardName", "non-existent", notFound.getShardName());
1046             }
1047         };
1048     }
1049
1050     @Test
1051     public void testOnReceiveFindLocalShardForExistentShard() {
1052         new TestKit(getSystem()) {
1053             {
1054                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1055
1056                 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1057                 shardManager.tell(new ActorInitialized(), mockShardActor);
1058
1059                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1060
1061                 LocalShardFound found = expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1062
1063                 assertTrue("Found path contains " + found.getPath().path().toString(),
1064                         found.getPath().path().toString().contains("member-1-shard-default-config"));
1065             }
1066         };
1067     }
1068
1069     @Test
1070     public void testOnReceiveFindLocalShardForNotInitializedShard() {
1071         new TestKit(getSystem()) {
1072             {
1073                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1074
1075                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1076
1077                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1078             }
1079         };
1080     }
1081
1082     @Test
1083     public void testOnReceiveFindLocalShardWaitForShardInitialized() throws Exception {
1084         LOG.info("testOnReceiveFindLocalShardWaitForShardInitialized starting");
1085         new TestKit(getSystem()) {
1086             {
1087                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1088
1089                 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1090
1091                 // We're passing waitUntilInitialized = true to FindLocalShard
1092                 // so the response should be
1093                 // delayed until we send ActorInitialized.
1094                 Future<Object> future = Patterns.ask(shardManager, new FindLocalShard(Shard.DEFAULT_NAME, true),
1095                         new Timeout(5, TimeUnit.SECONDS));
1096
1097                 shardManager.tell(new ActorInitialized(), mockShardActor);
1098
1099                 Object resp = Await.result(future, duration("5 seconds"));
1100                 assertTrue("Expected: LocalShardFound, Actual: " + resp, resp instanceof LocalShardFound);
1101             }
1102         };
1103
1104         LOG.info("testOnReceiveFindLocalShardWaitForShardInitialized starting");
1105     }
1106
1107     @Test
1108     public void testRoleChangeNotificationAndShardLeaderStateChangedReleaseReady() throws Exception {
1109         TestShardManager shardManager = newTestShardManager();
1110
1111         String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1112         shardManager.onReceiveCommand(new RoleChangeNotification(
1113                 memberId, RaftState.Candidate.name(), RaftState.Leader.name()));
1114
1115         verify(ready, never()).countDown();
1116
1117         shardManager.onReceiveCommand(new ShardLeaderStateChanged(memberId, memberId,
1118                 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1119
1120         verify(ready, times(1)).countDown();
1121     }
1122
1123     @Test
1124     public void testRoleChangeNotificationToFollowerWithShardLeaderStateChangedReleaseReady() throws Exception {
1125         new TestKit(getSystem()) {
1126             {
1127                 TestShardManager shardManager = newTestShardManager();
1128
1129                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1130                 shardManager.onReceiveCommand(new RoleChangeNotification(memberId, null, RaftState.Follower.name()));
1131
1132                 verify(ready, never()).countDown();
1133
1134                 shardManager
1135                         .onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
1136
1137                 shardManager.onReceiveCommand(
1138                         new ShardLeaderStateChanged(memberId, "member-2-shard-default-" + shardMrgIDSuffix,
1139                                 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1140
1141                 verify(ready, times(1)).countDown();
1142             }
1143         };
1144     }
1145
1146     @Test
1147     public void testReadyCountDownForMemberUpAfterLeaderStateChanged() throws Exception {
1148         new TestKit(getSystem()) {
1149             {
1150                 TestShardManager shardManager = newTestShardManager();
1151
1152                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1153                 shardManager.onReceiveCommand(new RoleChangeNotification(memberId, null, RaftState.Follower.name()));
1154
1155                 verify(ready, never()).countDown();
1156
1157                 shardManager.onReceiveCommand(
1158                         new ShardLeaderStateChanged(memberId, "member-2-shard-default-" + shardMrgIDSuffix,
1159                                 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1160
1161                 shardManager
1162                         .onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
1163
1164                 verify(ready, times(1)).countDown();
1165             }
1166         };
1167     }
1168
1169     @Test
1170     public void testRoleChangeNotificationDoNothingForUnknownShard() throws Exception {
1171         TestShardManager shardManager = newTestShardManager();
1172
1173         shardManager.onReceiveCommand(new RoleChangeNotification(
1174                 "unknown", RaftState.Candidate.name(), RaftState.Leader.name()));
1175
1176         verify(ready, never()).countDown();
1177     }
1178
1179     @Test
1180     public void testByDefaultSyncStatusIsFalse() {
1181         TestShardManager shardManager = newTestShardManager();
1182
1183         assertEquals(false, shardManager.getMBean().getSyncStatus());
1184     }
1185
1186     @Test
1187     public void testWhenShardIsLeaderSyncStatusIsTrue() throws Exception {
1188         TestShardManager shardManager = newTestShardManager();
1189
1190         shardManager.onReceiveCommand(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix,
1191                 RaftState.Follower.name(), RaftState.Leader.name()));
1192
1193         assertEquals(true, shardManager.getMBean().getSyncStatus());
1194     }
1195
1196     @Test
1197     public void testWhenShardIsCandidateSyncStatusIsFalse() throws Exception {
1198         TestShardManager shardManager = newTestShardManager();
1199
1200         String shardId = "member-1-shard-default-" + shardMrgIDSuffix;
1201         shardManager.onReceiveCommand(new RoleChangeNotification(shardId,
1202                 RaftState.Follower.name(), RaftState.Candidate.name()));
1203
1204         assertEquals(false, shardManager.getMBean().getSyncStatus());
1205
1206         // Send a FollowerInitialSyncStatus with status = true for the replica whose current state is candidate
1207         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(
1208                 true, shardId));
1209
1210         assertEquals(false, shardManager.getMBean().getSyncStatus());
1211     }
1212
1213     @Test
1214     public void testWhenShardIsFollowerSyncStatusDependsOnFollowerInitialSyncStatus() throws Exception {
1215         TestShardManager shardManager = newTestShardManager();
1216
1217         String shardId = "member-1-shard-default-" + shardMrgIDSuffix;
1218         shardManager.onReceiveCommand(new RoleChangeNotification(shardId,
1219                 RaftState.Candidate.name(), RaftState.Follower.name()));
1220
1221         // Initially will be false
1222         assertEquals(false, shardManager.getMBean().getSyncStatus());
1223
1224         // Send status true will make sync status true
1225         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(true, shardId));
1226
1227         assertEquals(true, shardManager.getMBean().getSyncStatus());
1228
1229         // Send status false will make sync status false
1230         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(false, shardId));
1231
1232         assertEquals(false, shardManager.getMBean().getSyncStatus());
1233
1234     }
1235
1236     @Test
1237     public void testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards() throws Exception {
1238         LOG.info("testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards starting");
1239         TestShardManager shardManager = newTestShardManager(newShardMgrProps(new MockConfiguration() {
1240             @Override
1241             public List<String> getMemberShardNames(final MemberName memberName) {
1242                 return Arrays.asList("default", "astronauts");
1243             }
1244         }));
1245
1246         // Initially will be false
1247         assertEquals(false, shardManager.getMBean().getSyncStatus());
1248
1249         // Make default shard leader
1250         String defaultShardId = "member-1-shard-default-" + shardMrgIDSuffix;
1251         shardManager.onReceiveCommand(new RoleChangeNotification(defaultShardId,
1252                 RaftState.Follower.name(), RaftState.Leader.name()));
1253
1254         // default = Leader, astronauts is unknown so sync status remains false
1255         assertEquals(false, shardManager.getMBean().getSyncStatus());
1256
1257         // Make astronauts shard leader as well
1258         String astronautsShardId = "member-1-shard-astronauts-" + shardMrgIDSuffix;
1259         shardManager.onReceiveCommand(new RoleChangeNotification(astronautsShardId,
1260                 RaftState.Follower.name(), RaftState.Leader.name()));
1261
1262         // Now sync status should be true
1263         assertEquals(true, shardManager.getMBean().getSyncStatus());
1264
1265         // Make astronauts a Follower
1266         shardManager.onReceiveCommand(new RoleChangeNotification(astronautsShardId,
1267                 RaftState.Leader.name(), RaftState.Follower.name()));
1268
1269         // Sync status is not true
1270         assertEquals(false, shardManager.getMBean().getSyncStatus());
1271
1272         // Make the astronauts follower sync status true
1273         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(true, astronautsShardId));
1274
1275         // Sync status is now true
1276         assertEquals(true, shardManager.getMBean().getSyncStatus());
1277
1278         LOG.info("testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards ending");
1279     }
1280
1281     @Test
1282     public void testOnReceiveSwitchShardBehavior() {
1283         new TestKit(getSystem()) {
1284             {
1285                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1286
1287                 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1288                 shardManager.tell(new ActorInitialized(), mockShardActor);
1289
1290                 shardManager.tell(new SwitchShardBehavior(mockShardName, RaftState.Leader, 1000), getRef());
1291
1292                 SwitchBehavior switchBehavior = MessageCollectorActor.expectFirstMatching(mockShardActor,
1293                         SwitchBehavior.class);
1294
1295                 assertEquals(RaftState.Leader, switchBehavior.getNewState());
1296                 assertEquals(1000, switchBehavior.getNewTerm());
1297             }
1298         };
1299     }
1300
1301     private static List<MemberName> members(final String... names) {
1302         return Arrays.asList(names).stream().map(MemberName::forName).collect(Collectors.toList());
1303     }
1304
1305     @Test
1306     public void testOnCreateShard() {
1307         LOG.info("testOnCreateShard starting");
1308         new TestKit(getSystem()) {
1309             {
1310                 datastoreContextBuilder.shardInitializationTimeout(1, TimeUnit.MINUTES).persistent(true);
1311
1312                 ActorRef shardManager = actorFactory
1313                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1314                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1315
1316                 SchemaContext schemaContext = TEST_SCHEMA_CONTEXT;
1317                 shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
1318
1319                 DatastoreContext datastoreContext = DatastoreContext.newBuilder().shardElectionTimeoutFactor(100)
1320                         .persistent(false).build();
1321                 Shard.Builder shardBuilder = Shard.builder();
1322
1323                 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1324                         "foo", null, members("member-1", "member-5", "member-6"));
1325                 shardManager.tell(new CreateShard(config, shardBuilder, datastoreContext), getRef());
1326
1327                 expectMsgClass(duration("5 seconds"), Success.class);
1328
1329                 shardManager.tell(new FindLocalShard("foo", true), getRef());
1330
1331                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1332
1333                 assertEquals("isRecoveryApplicable", false, shardBuilder.getDatastoreContext().isPersistent());
1334                 assertTrue("Epxected ShardPeerAddressResolver", shardBuilder.getDatastoreContext().getShardRaftConfig()
1335                         .getPeerAddressResolver() instanceof ShardPeerAddressResolver);
1336                 assertEquals("peerMembers", Sets.newHashSet(
1337                         ShardIdentifier.create("foo", MemberName.forName("member-5"), shardMrgIDSuffix).toString(),
1338                         ShardIdentifier.create("foo", MemberName.forName("member-6"), shardMrgIDSuffix).toString()),
1339                         shardBuilder.getPeerAddresses().keySet());
1340                 assertEquals("ShardIdentifier", ShardIdentifier.create("foo", MEMBER_1, shardMrgIDSuffix),
1341                         shardBuilder.getId());
1342                 assertSame("schemaContext", schemaContext, shardBuilder.getSchemaContext());
1343
1344                 // Send CreateShard with same name - should return Success with
1345                 // a message.
1346
1347                 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1348
1349                 Success success = expectMsgClass(duration("5 seconds"), Success.class);
1350                 assertNotNull("Success status is null", success.status());
1351             }
1352         };
1353
1354         LOG.info("testOnCreateShard ending");
1355     }
1356
1357     @Test
1358     public void testOnCreateShardWithLocalMemberNotInShardConfig() {
1359         LOG.info("testOnCreateShardWithLocalMemberNotInShardConfig starting");
1360         new TestKit(getSystem()) {
1361             {
1362                 datastoreContextBuilder.shardInitializationTimeout(1, TimeUnit.MINUTES).persistent(true);
1363
1364                 ActorRef shardManager = actorFactory
1365                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1366                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1367
1368                 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), ActorRef.noSender());
1369
1370                 Shard.Builder shardBuilder = Shard.builder();
1371                 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1372                         "foo", null, members("member-5", "member-6"));
1373
1374                 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1375                 expectMsgClass(duration("5 seconds"), Success.class);
1376
1377                 shardManager.tell(new FindLocalShard("foo", true), getRef());
1378                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1379
1380                 assertEquals("peerMembers size", 0, shardBuilder.getPeerAddresses().size());
1381                 assertEquals("schemaContext", DisableElectionsRaftPolicy.class.getName(), shardBuilder
1382                         .getDatastoreContext().getShardRaftConfig().getCustomRaftPolicyImplementationClass());
1383             }
1384         };
1385
1386         LOG.info("testOnCreateShardWithLocalMemberNotInShardConfig ending");
1387     }
1388
1389     @Test
1390     public void testOnCreateShardWithNoInitialSchemaContext() {
1391         LOG.info("testOnCreateShardWithNoInitialSchemaContext starting");
1392         new TestKit(getSystem()) {
1393             {
1394                 ActorRef shardManager = actorFactory
1395                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1396                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1397
1398                 Shard.Builder shardBuilder = Shard.builder();
1399
1400                 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1401                         "foo", null, members("member-1"));
1402                 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1403
1404                 expectMsgClass(duration("5 seconds"), Success.class);
1405
1406                 SchemaContext schemaContext = TEST_SCHEMA_CONTEXT;
1407                 shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
1408
1409                 shardManager.tell(new FindLocalShard("foo", true), getRef());
1410
1411                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1412
1413                 assertSame("schemaContext", schemaContext, shardBuilder.getSchemaContext());
1414                 assertNotNull("schemaContext is null", shardBuilder.getDatastoreContext());
1415             }
1416         };
1417
1418         LOG.info("testOnCreateShardWithNoInitialSchemaContext ending");
1419     }
1420
1421     @Test
1422     public void testGetSnapshot() {
1423         LOG.info("testGetSnapshot starting");
1424         TestKit kit = new TestKit(getSystem());
1425
1426         MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1427                 .put("shard1", Arrays.asList("member-1")).put("shard2", Arrays.asList("member-1"))
1428                 .put("astronauts", Collections.<String>emptyList()).build());
1429
1430         TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(newShardMgrProps(mockConfig)
1431                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1432
1433         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1434         Failure failure = kit.expectMsgClass(Failure.class);
1435         assertEquals("Failure cause type", IllegalStateException.class, failure.cause().getClass());
1436
1437         shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), ActorRef.noSender());
1438
1439         waitForShardInitialized(shardManager, "shard1", kit);
1440         waitForShardInitialized(shardManager, "shard2", kit);
1441
1442         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1443
1444         DatastoreSnapshot datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1445
1446         assertEquals("getType", shardMrgIDSuffix, datastoreSnapshot.getType());
1447         assertNull("Expected null ShardManagerSnapshot", datastoreSnapshot.getShardManagerSnapshot());
1448
1449         Function<ShardSnapshot, String> shardNameTransformer = ShardSnapshot::getName;
1450
1451         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2"), Sets.newHashSet(
1452             datastoreSnapshot.getShardSnapshots().stream().map(shardNameTransformer).collect(Collectors.toSet())));
1453
1454         // Add a new replica
1455
1456         TestKit mockShardLeaderKit = new TestKit(getSystem());
1457
1458         TestShardManager shardManagerInstance = shardManager.underlyingActor();
1459         shardManagerInstance.setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1460
1461         shardManager.tell(new AddShardReplica("astronauts"), kit.getRef());
1462         mockShardLeaderKit.expectMsgClass(AddServer.class);
1463         mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.OK, ""));
1464         kit.expectMsgClass(Status.Success.class);
1465         waitForShardInitialized(shardManager, "astronauts", kit);
1466
1467         // Send another GetSnapshot and verify
1468
1469         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1470         datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1471
1472         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"), Sets.newHashSet(
1473                 Lists.transform(datastoreSnapshot.getShardSnapshots(), shardNameTransformer)));
1474
1475         ShardManagerSnapshot snapshot = datastoreSnapshot.getShardManagerSnapshot();
1476         assertNotNull("Expected ShardManagerSnapshot", snapshot);
1477         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"),
1478                 Sets.newHashSet(snapshot.getShardList()));
1479
1480         LOG.info("testGetSnapshot ending");
1481     }
1482
1483     @Test
1484     public void testRestoreFromSnapshot() {
1485         LOG.info("testRestoreFromSnapshot starting");
1486
1487         datastoreContextBuilder.shardInitializationTimeout(3, TimeUnit.SECONDS);
1488
1489         TestKit kit = new TestKit(getSystem());
1490
1491         MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1492                 .put("shard1", Collections.<String>emptyList()).put("shard2", Collections.<String>emptyList())
1493                 .put("astronauts", Collections.<String>emptyList()).build());
1494
1495         ShardManagerSnapshot snapshot =
1496                 new ShardManagerSnapshot(Arrays.asList("shard1", "shard2", "astronauts"), Collections.emptyMap());
1497         DatastoreSnapshot restoreFromSnapshot = new DatastoreSnapshot(shardMrgIDSuffix, snapshot,
1498                 Collections.<ShardSnapshot>emptyList());
1499         TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(newTestShardMgrBuilder(mockConfig)
1500                 .restoreFromSnapshot(restoreFromSnapshot).props().withDispatcher(Dispatchers.DefaultDispatcherId()));
1501
1502         shardManager.underlyingActor().waitForRecoveryComplete();
1503
1504         shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), ActorRef.noSender());
1505
1506         waitForShardInitialized(shardManager, "shard1", kit);
1507         waitForShardInitialized(shardManager, "shard2", kit);
1508         waitForShardInitialized(shardManager, "astronauts", kit);
1509
1510         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1511
1512         DatastoreSnapshot datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1513
1514         assertEquals("getType", shardMrgIDSuffix, datastoreSnapshot.getType());
1515
1516         assertNotNull("Expected ShardManagerSnapshot", datastoreSnapshot.getShardManagerSnapshot());
1517         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"),
1518                 Sets.newHashSet(datastoreSnapshot.getShardManagerSnapshot().getShardList()));
1519
1520         LOG.info("testRestoreFromSnapshot ending");
1521     }
1522
1523     @Test
1524     public void testAddShardReplicaForNonExistentShardConfig() {
1525         new TestKit(getSystem()) {
1526             {
1527                 ActorRef shardManager = actorFactory
1528                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1529                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1530
1531                 shardManager.tell(new AddShardReplica("model-inventory"), getRef());
1532                 Status.Failure resp = expectMsgClass(duration("2 seconds"), Status.Failure.class);
1533
1534                 assertEquals("Failure obtained", true, resp.cause() instanceof IllegalArgumentException);
1535             }
1536         };
1537     }
1538
1539     @Test
1540     public void testAddShardReplica() {
1541         LOG.info("testAddShardReplica starting");
1542         MockConfiguration mockConfig = new MockConfiguration(
1543                 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
1544                         .put("astronauts", Arrays.asList("member-2")).build());
1545
1546         final String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
1547         datastoreContextBuilder.shardManagerPersistenceId(shardManagerID);
1548
1549         // Create an ActorSystem ShardManager actor for member-1.
1550         final ActorSystem system1 = newActorSystem("Member1");
1551         Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1552         ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
1553         final TestActorRef<TestShardManager> newReplicaShardManager = TestActorRef.create(system1,
1554                 newTestShardMgrBuilder(mockConfig).shardActor(mockDefaultShardActor)
1555                         .cluster(new ClusterWrapperImpl(system1)).props()
1556                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
1557                 shardManagerID);
1558
1559         // Create an ActorSystem ShardManager actor for member-2.
1560         final ActorSystem system2 = newActorSystem("Member2");
1561         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1562
1563         String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
1564         String name = ShardIdentifier.create("astronauts", MEMBER_2, "config").toString();
1565         final TestActorRef<MockRespondActor> mockShardLeaderActor = TestActorRef.create(system2,
1566                 Props.create(MockRespondActor.class, AddServer.class,
1567                         new AddServerReply(ServerChangeStatus.OK, memberId2))
1568                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
1569                 name);
1570         final TestActorRef<TestShardManager> leaderShardManager = TestActorRef.create(system2,
1571                 newTestShardMgrBuilder(mockConfig).shardActor(mockShardLeaderActor)
1572                         .cluster(new ClusterWrapperImpl(system2)).props()
1573                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
1574                 shardManagerID);
1575
1576         new TestKit(system1) {
1577             {
1578                 newReplicaShardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1579                 leaderShardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1580
1581                 leaderShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1582
1583                 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
1584                 leaderShardManager.tell(
1585                         new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
1586                         mockShardLeaderActor);
1587                 leaderShardManager.tell(
1588                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
1589                         mockShardLeaderActor);
1590
1591                 newReplicaShardManager.underlyingActor().waitForMemberUp();
1592                 leaderShardManager.underlyingActor().waitForMemberUp();
1593
1594                 // Have a dummy snapshot to be overwritten by the new data
1595                 // persisted.
1596                 String[] restoredShards = { "default", "people" };
1597                 ShardManagerSnapshot snapshot =
1598                         new ShardManagerSnapshot(Arrays.asList(restoredShards), Collections.emptyMap());
1599                 InMemorySnapshotStore.addSnapshot(shardManagerID, snapshot);
1600                 Uninterruptibles.sleepUninterruptibly(2, TimeUnit.MILLISECONDS);
1601
1602                 InMemorySnapshotStore.addSnapshotSavedLatch(shardManagerID);
1603                 InMemorySnapshotStore.addSnapshotDeletedLatch(shardManagerID);
1604
1605                 // construct a mock response message
1606                 newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
1607                 AddServer addServerMsg = MessageCollectorActor.expectFirstMatching(mockShardLeaderActor,
1608                         AddServer.class);
1609                 String addServerId = "member-1-shard-astronauts-" + shardMrgIDSuffix;
1610                 assertEquals("AddServer serverId", addServerId, addServerMsg.getNewServerId());
1611                 expectMsgClass(duration("5 seconds"), Status.Success.class);
1612
1613                 InMemorySnapshotStore.waitForSavedSnapshot(shardManagerID, ShardManagerSnapshot.class);
1614                 InMemorySnapshotStore.waitForDeletedSnapshot(shardManagerID);
1615                 List<ShardManagerSnapshot> persistedSnapshots = InMemorySnapshotStore.getSnapshots(shardManagerID,
1616                         ShardManagerSnapshot.class);
1617                 assertEquals("Number of snapshots persisted", 1, persistedSnapshots.size());
1618                 ShardManagerSnapshot shardManagerSnapshot = persistedSnapshots.get(0);
1619                 assertEquals("Persisted local shards", Sets.newHashSet("default", "astronauts"),
1620                         Sets.newHashSet(shardManagerSnapshot.getShardList()));
1621             }
1622         };
1623         LOG.info("testAddShardReplica ending");
1624     }
1625
1626     @Test
1627     public void testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader() {
1628         LOG.info("testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader starting");
1629         new TestKit(getSystem()) {
1630             {
1631                 TestActorRef<TestShardManager> shardManager = actorFactory
1632                         .createTestActor(newPropsShardMgrWithMockShardActor(), shardMgrID);
1633
1634                 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1635                 shardManager.tell(new ActorInitialized(), mockShardActor);
1636
1637                 String leaderId = "leader-member-shard-default-" + shardMrgIDSuffix;
1638                 AddServerReply addServerReply = new AddServerReply(ServerChangeStatus.ALREADY_EXISTS, null);
1639                 ActorRef leaderShardActor = shardManager.underlyingActor().getContext()
1640                         .actorOf(Props.create(MockRespondActor.class, AddServer.class, addServerReply), leaderId);
1641
1642                 MockClusterWrapper.sendMemberUp(shardManager, "leader-member", leaderShardActor.path().toString());
1643
1644                 String newReplicaId = "member-1-shard-default-" + shardMrgIDSuffix;
1645                 shardManager.tell(
1646                         new RoleChangeNotification(newReplicaId, RaftState.Candidate.name(), RaftState.Follower.name()),
1647                         mockShardActor);
1648                 shardManager.tell(
1649                         new ShardLeaderStateChanged(newReplicaId, leaderId, DataStoreVersions.CURRENT_VERSION),
1650                         mockShardActor);
1651
1652                 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1653
1654                 MessageCollectorActor.expectFirstMatching(leaderShardActor, AddServer.class);
1655
1656                 Failure resp = expectMsgClass(duration("5 seconds"), Failure.class);
1657                 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1658
1659                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1660                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1661
1662                 // Send message again to verify previous in progress state is
1663                 // cleared
1664
1665                 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1666                 resp = expectMsgClass(duration("5 seconds"), Failure.class);
1667                 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1668
1669                 // Send message again with an AddServer timeout to verify the
1670                 // pre-existing shard actor isn't terminated.
1671
1672                 shardManager.tell(
1673                         newDatastoreContextFactory(
1674                                 datastoreContextBuilder.shardLeaderElectionTimeout(100, TimeUnit.MILLISECONDS).build()),
1675                         getRef());
1676                 leaderShardActor.tell(MockRespondActor.CLEAR_RESPONSE, ActorRef.noSender());
1677                 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1678                 expectMsgClass(duration("5 seconds"), Failure.class);
1679
1680                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1681                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1682             }
1683         };
1684
1685         LOG.info("testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader ending");
1686     }
1687
1688     @Test
1689     public void testAddShardReplicaWithPreExistingLocalReplicaLeader() {
1690         LOG.info("testAddShardReplicaWithPreExistingLocalReplicaLeader starting");
1691         new TestKit(getSystem()) {
1692             {
1693                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1694                 ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1695
1696                 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1697                 shardManager.tell(new ActorInitialized(), mockShardActor);
1698                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
1699                         DataStoreVersions.CURRENT_VERSION), getRef());
1700                 shardManager.tell(
1701                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
1702                         mockShardActor);
1703
1704                 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1705                 Failure resp = expectMsgClass(duration("5 seconds"), Failure.class);
1706                 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1707
1708                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1709                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1710             }
1711         };
1712
1713         LOG.info("testAddShardReplicaWithPreExistingLocalReplicaLeader ending");
1714     }
1715
1716     @Test
1717     public void testAddShardReplicaWithAddServerReplyFailure() {
1718         LOG.info("testAddShardReplicaWithAddServerReplyFailure starting");
1719         new TestKit(getSystem()) {
1720             {
1721                 TestKit mockShardLeaderKit = new TestKit(getSystem());
1722
1723                 MockConfiguration mockConfig = new MockConfiguration(
1724                     ImmutableMap.of("astronauts", Arrays.asList("member-2")));
1725
1726                 ActorRef mockNewReplicaShardActor = newMockShardActor(getSystem(), "astronauts", "member-1");
1727                 final TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(
1728                         newTestShardMgrBuilder(mockConfig).shardActor(mockNewReplicaShardActor).props()
1729                             .withDispatcher(Dispatchers.DefaultDispatcherId()), shardMgrID);
1730                 shardManager.underlyingActor()
1731                         .setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1732
1733                 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1734
1735                 TestKit terminateWatcher = new TestKit(getSystem());
1736                 terminateWatcher.watch(mockNewReplicaShardActor);
1737
1738                 shardManager.tell(new AddShardReplica("astronauts"), getRef());
1739
1740                 AddServer addServerMsg = mockShardLeaderKit.expectMsgClass(AddServer.class);
1741                 assertEquals("AddServer serverId", "member-1-shard-astronauts-" + shardMrgIDSuffix,
1742                         addServerMsg.getNewServerId());
1743                 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.TIMEOUT, null));
1744
1745                 Failure failure = expectMsgClass(duration("5 seconds"), Failure.class);
1746                 assertEquals("Failure cause", TimeoutException.class, failure.cause().getClass());
1747
1748                 shardManager.tell(new FindLocalShard("astronauts", false), getRef());
1749                 expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
1750
1751                 terminateWatcher.expectTerminated(mockNewReplicaShardActor);
1752
1753                 shardManager.tell(new AddShardReplica("astronauts"), getRef());
1754                 mockShardLeaderKit.expectMsgClass(AddServer.class);
1755                 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.NO_LEADER, null));
1756                 failure = expectMsgClass(duration("5 seconds"), Failure.class);
1757                 assertEquals("Failure cause", NoShardLeaderException.class, failure.cause().getClass());
1758             }
1759         };
1760
1761         LOG.info("testAddShardReplicaWithAddServerReplyFailure ending");
1762     }
1763
1764     @Test
1765     public void testAddShardReplicaWithAlreadyInProgress() {
1766         testServerChangeWhenAlreadyInProgress("astronauts", new AddShardReplica("astronauts"),
1767                 AddServer.class, new AddShardReplica("astronauts"));
1768     }
1769
1770     @Test
1771     public void testAddShardReplicaWithFindPrimaryTimeout() {
1772         LOG.info("testAddShardReplicaWithFindPrimaryTimeout starting");
1773         datastoreContextBuilder.shardInitializationTimeout(100, TimeUnit.MILLISECONDS);
1774         new TestKit(getSystem()) {
1775             {
1776                 MockConfiguration mockConfig = new MockConfiguration(
1777                     ImmutableMap.of("astronauts", Arrays.asList("member-2")));
1778
1779                 final ActorRef newReplicaShardManager = actorFactory
1780                         .createActor(newTestShardMgrBuilder(mockConfig).shardActor(mockShardActor).props()
1781                                 .withDispatcher(Dispatchers.DefaultDispatcherId()), shardMgrID);
1782
1783                 newReplicaShardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1784                 MockClusterWrapper.sendMemberUp(newReplicaShardManager, "member-2",
1785                         AddressFromURIString.parse("akka://non-existent@127.0.0.1:5").toString());
1786
1787                 newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
1788                 Status.Failure resp = expectMsgClass(duration("5 seconds"), Status.Failure.class);
1789                 assertEquals("Failure obtained", true, resp.cause() instanceof RuntimeException);
1790             }
1791         };
1792
1793         LOG.info("testAddShardReplicaWithFindPrimaryTimeout ending");
1794     }
1795
1796     @Test
1797     public void testRemoveShardReplicaForNonExistentShard() {
1798         new TestKit(getSystem()) {
1799             {
1800                 ActorRef shardManager = actorFactory
1801                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider()))
1802                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
1803
1804                 shardManager.tell(new RemoveShardReplica("model-inventory", MEMBER_1), getRef());
1805                 Status.Failure resp = expectMsgClass(duration("10 seconds"), Status.Failure.class);
1806                 assertEquals("Failure obtained", true, resp.cause() instanceof PrimaryNotFoundException);
1807             }
1808         };
1809     }
1810
1811     @Test
1812     /**
1813      * Primary is Local.
1814      */
1815     public void testRemoveShardReplicaLocal() {
1816         new TestKit(getSystem()) {
1817             {
1818                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1819
1820                 final ActorRef respondActor = actorFactory.createActor(Props.create(MockRespondActor.class,
1821                         RemoveServer.class, new RemoveServerReply(ServerChangeStatus.OK, null)), memberId);
1822
1823                 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
1824
1825                 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1826                 shardManager.tell(new ActorInitialized(), respondActor);
1827                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
1828                         DataStoreVersions.CURRENT_VERSION), getRef());
1829                 shardManager.tell(
1830                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
1831                         respondActor);
1832
1833                 shardManager.tell(new RemoveShardReplica(Shard.DEFAULT_NAME, MEMBER_1), getRef());
1834                 final RemoveServer removeServer = MessageCollectorActor.expectFirstMatching(respondActor,
1835                         RemoveServer.class);
1836                 assertEquals(ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString(),
1837                         removeServer.getServerId());
1838                 expectMsgClass(duration("5 seconds"), Success.class);
1839             }
1840         };
1841     }
1842
1843     @Test
1844     public void testRemoveShardReplicaRemote() {
1845         MockConfiguration mockConfig = new MockConfiguration(
1846                 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
1847                         .put("astronauts", Arrays.asList("member-1")).build());
1848
1849         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
1850
1851         // Create an ActorSystem ShardManager actor for member-1.
1852         final ActorSystem system1 = newActorSystem("Member1");
1853         Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1854         ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
1855
1856         final TestActorRef<TestShardManager> newReplicaShardManager = TestActorRef.create(system1,
1857                 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockDefaultShardActor).cluster(
1858                         new ClusterWrapperImpl(system1)).props().withDispatcher(Dispatchers.DefaultDispatcherId()),
1859                 shardManagerID);
1860
1861         // Create an ActorSystem ShardManager actor for member-2.
1862         final ActorSystem system2 = newActorSystem("Member2");
1863         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1864
1865         String name = ShardIdentifier.create("default", MEMBER_2, shardMrgIDSuffix).toString();
1866         String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
1867         final TestActorRef<MockRespondActor> mockShardLeaderActor =
1868                 TestActorRef.create(system2, Props.create(MockRespondActor.class, RemoveServer.class,
1869                         new RemoveServerReply(ServerChangeStatus.OK, memberId2)), name);
1870
1871         LOG.error("Mock Shard Leader Actor : {}", mockShardLeaderActor);
1872
1873         final TestActorRef<TestShardManager> leaderShardManager = TestActorRef.create(system2,
1874                 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockShardLeaderActor).cluster(
1875                         new ClusterWrapperImpl(system2)).props().withDispatcher(Dispatchers.DefaultDispatcherId()),
1876                 shardManagerID);
1877
1878         // Because mockShardLeaderActor is created at the top level of the actor system it has an address like so,
1879         //    akka://cluster-test@127.0.0.1:2559/user/member-2-shard-default-config1
1880         // However when a shard manager has a local shard which is a follower and a leader that is remote it will
1881         // try to compute an address for the remote shard leader using the ShardPeerAddressResolver. This address will
1882         // look like so,
1883         //    akka://cluster-test@127.0.0.1:2559/user/shardmanager-config1/member-2-shard-default-config1
1884         // In this specific case if we did a FindPrimary for shard default from member-1 we would come up
1885         // with the address of an actor which does not exist, therefore any message sent to that actor would go to
1886         // dead letters.
1887         // To work around this problem we create a ForwardingActor with the right address and pass to it the
1888         // mockShardLeaderActor. The ForwardingActor simply forwards all messages to the mockShardLeaderActor and every
1889         // thing works as expected
1890         final ActorRef actorRef = leaderShardManager.underlyingActor().context()
1891                 .actorOf(Props.create(ForwardingActor.class, mockShardLeaderActor),
1892                         "member-2-shard-default-" + shardMrgIDSuffix);
1893
1894         LOG.error("Forwarding actor : {}", actorRef);
1895
1896         new TestKit(system1) {
1897             {
1898                 newReplicaShardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1899                 leaderShardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1900
1901                 leaderShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1902                 newReplicaShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1903
1904                 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
1905                 leaderShardManager.tell(
1906                         new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
1907                         mockShardLeaderActor);
1908                 leaderShardManager.tell(
1909                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
1910                         mockShardLeaderActor);
1911
1912                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
1913                 newReplicaShardManager.tell(
1914                         new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class), leaderVersion),
1915                         mockShardActor);
1916                 newReplicaShardManager.tell(
1917                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
1918                         mockShardActor);
1919
1920                 newReplicaShardManager.underlyingActor().waitForMemberUp();
1921                 leaderShardManager.underlyingActor().waitForMemberUp();
1922
1923                 // construct a mock response message
1924                 newReplicaShardManager.tell(new RemoveShardReplica("default", MEMBER_1), getRef());
1925                 RemoveServer removeServer = MessageCollectorActor.expectFirstMatching(mockShardLeaderActor,
1926                         RemoveServer.class);
1927                 String removeServerId = ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString();
1928                 assertEquals("RemoveServer serverId", removeServerId, removeServer.getServerId());
1929                 expectMsgClass(duration("5 seconds"), Status.Success.class);
1930             }
1931         };
1932     }
1933
1934     @Test
1935     public void testRemoveShardReplicaWhenAnotherRemoveShardReplicaAlreadyInProgress() {
1936         testServerChangeWhenAlreadyInProgress("astronauts", new RemoveShardReplica("astronauts", MEMBER_2),
1937                 RemoveServer.class, new RemoveShardReplica("astronauts", MEMBER_3));
1938     }
1939
1940     @Test
1941     public void testRemoveShardReplicaWhenAddShardReplicaAlreadyInProgress() {
1942         testServerChangeWhenAlreadyInProgress("astronauts", new AddShardReplica("astronauts"),
1943                 AddServer.class, new RemoveShardReplica("astronauts", MEMBER_2));
1944     }
1945
1946
1947     public void testServerChangeWhenAlreadyInProgress(final String shardName, final Object firstServerChange,
1948                                                       final Class<?> firstForwardedServerChangeClass,
1949                                                       final Object secondServerChange) {
1950         new TestKit(getSystem()) {
1951             {
1952                 TestKit mockShardLeaderKit = new TestKit(getSystem());
1953                 final TestKit secondRequestKit = new TestKit(getSystem());
1954
1955                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1956                         .put(shardName, Arrays.asList("member-2")).build());
1957
1958                 final TestActorRef<TestShardManager> shardManager = TestActorRef.create(getSystem(),
1959                         newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockShardActor)
1960                                 .cluster(new MockClusterWrapper()).props()
1961                                 .withDispatcher(Dispatchers.DefaultDispatcherId()),
1962                         shardMgrID);
1963
1964                 shardManager.underlyingActor()
1965                         .setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1966
1967                 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
1968
1969                 shardManager.tell(firstServerChange, getRef());
1970
1971                 mockShardLeaderKit.expectMsgClass(firstForwardedServerChangeClass);
1972
1973                 shardManager.tell(secondServerChange, secondRequestKit.getRef());
1974
1975                 secondRequestKit.expectMsgClass(duration("5 seconds"), Failure.class);
1976             }
1977         };
1978     }
1979
1980     @Test
1981     public void testServerRemovedShardActorNotRunning() {
1982         LOG.info("testServerRemovedShardActorNotRunning starting");
1983         new TestKit(getSystem()) {
1984             {
1985                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1986                         .put("default", Arrays.asList("member-1", "member-2"))
1987                         .put("astronauts", Arrays.asList("member-2"))
1988                         .put("people", Arrays.asList("member-1", "member-2")).build());
1989
1990                 TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(
1991                         newShardMgrProps(mockConfig).withDispatcher(Dispatchers.DefaultDispatcherId()));
1992
1993                 shardManager.underlyingActor().waitForRecoveryComplete();
1994                 shardManager.tell(new FindLocalShard("people", false), getRef());
1995                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1996
1997                 shardManager.tell(new FindLocalShard("default", false), getRef());
1998                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1999
2000                 // Removed the default shard replica from member-1
2001                 ShardIdentifier.Builder builder = new ShardIdentifier.Builder();
2002                 ShardIdentifier shardId = builder.shardName("default").memberName(MEMBER_1).type(shardMrgIDSuffix)
2003                         .build();
2004                 shardManager.tell(new ServerRemoved(shardId.toString()), getRef());
2005
2006                 shardManager.underlyingActor().verifySnapshotPersisted(Sets.newHashSet("people"));
2007             }
2008         };
2009
2010         LOG.info("testServerRemovedShardActorNotRunning ending");
2011     }
2012
2013     @Test
2014     public void testServerRemovedShardActorRunning() {
2015         LOG.info("testServerRemovedShardActorRunning starting");
2016         new TestKit(getSystem()) {
2017             {
2018                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
2019                         .put("default", Arrays.asList("member-1", "member-2"))
2020                         .put("astronauts", Arrays.asList("member-2"))
2021                         .put("people", Arrays.asList("member-1", "member-2")).build());
2022
2023                 String shardId = ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString();
2024                 ActorRef shard = actorFactory.createActor(MessageCollectorActor.props(), shardId);
2025
2026                 TestActorRef<TestShardManager> shardManager = actorFactory
2027                         .createTestActor(newTestShardMgrBuilder(mockConfig).addShardActor("default", shard).props()
2028                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
2029
2030                 shardManager.underlyingActor().waitForRecoveryComplete();
2031
2032                 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
2033                 shardManager.tell(new ActorInitialized(), shard);
2034
2035                 waitForShardInitialized(shardManager, "people", this);
2036                 waitForShardInitialized(shardManager, "default", this);
2037
2038                 // Removed the default shard replica from member-1
2039                 shardManager.tell(new ServerRemoved(shardId), getRef());
2040
2041                 shardManager.underlyingActor().verifySnapshotPersisted(Sets.newHashSet("people"));
2042
2043                 MessageCollectorActor.expectFirstMatching(shard, Shutdown.class);
2044             }
2045         };
2046
2047         LOG.info("testServerRemovedShardActorRunning ending");
2048     }
2049
2050     @Test
2051     public void testShardPersistenceWithRestoredData() {
2052         LOG.info("testShardPersistenceWithRestoredData starting");
2053         new TestKit(getSystem()) {
2054             {
2055                 MockConfiguration mockConfig =
2056                     new MockConfiguration(ImmutableMap.<String, List<String>>builder()
2057                             .put("default", Arrays.asList("member-1", "member-2"))
2058                             .put("astronauts", Arrays.asList("member-2"))
2059                             .put("people", Arrays.asList("member-1", "member-2")).build());
2060                 String[] restoredShards = {"default", "astronauts"};
2061                 ShardManagerSnapshot snapshot =
2062                         new ShardManagerSnapshot(Arrays.asList(restoredShards), Collections.emptyMap());
2063                 InMemorySnapshotStore.addSnapshot("shard-manager-" + shardMrgIDSuffix, snapshot);
2064
2065                 // create shardManager to come up with restored data
2066                 TestActorRef<TestShardManager> newRestoredShardManager = actorFactory.createTestActor(
2067                         newShardMgrProps(mockConfig).withDispatcher(Dispatchers.DefaultDispatcherId()));
2068
2069                 newRestoredShardManager.underlyingActor().waitForRecoveryComplete();
2070
2071                 newRestoredShardManager.tell(new FindLocalShard("people", false), getRef());
2072                 LocalShardNotFound notFound = expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
2073                 assertEquals("for uninitialized shard", "people", notFound.getShardName());
2074
2075                 // Verify a local shard is created for the restored shards,
2076                 // although we expect a NotInitializedException for the shards
2077                 // as the actor initialization
2078                 // message is not sent for them
2079                 newRestoredShardManager.tell(new FindLocalShard("default", false), getRef());
2080                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
2081
2082                 newRestoredShardManager.tell(new FindLocalShard("astronauts", false), getRef());
2083                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
2084             }
2085         };
2086
2087         LOG.info("testShardPersistenceWithRestoredData ending");
2088     }
2089
2090     @Test
2091     public void testShutDown() throws Exception {
2092         LOG.info("testShutDown starting");
2093         new TestKit(getSystem()) {
2094             {
2095                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
2096                         .put("shard1", Arrays.asList("member-1")).put("shard2", Arrays.asList("member-1")).build());
2097
2098                 String shardId1 = ShardIdentifier.create("shard1", MEMBER_1, shardMrgIDSuffix).toString();
2099                 ActorRef shard1 = actorFactory.createActor(MessageCollectorActor.props(), shardId1);
2100
2101                 String shardId2 = ShardIdentifier.create("shard2", MEMBER_1, shardMrgIDSuffix).toString();
2102                 ActorRef shard2 = actorFactory.createActor(MessageCollectorActor.props(), shardId2);
2103
2104                 ActorRef shardManager = actorFactory.createActor(newTestShardMgrBuilder(mockConfig)
2105                         .addShardActor("shard1", shard1).addShardActor("shard2", shard2).props());
2106
2107                 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
2108                 shardManager.tell(new ActorInitialized(), shard1);
2109                 shardManager.tell(new ActorInitialized(), shard2);
2110
2111                 FiniteDuration duration = FiniteDuration.create(5, TimeUnit.SECONDS);
2112                 Future<Boolean> stopFuture = Patterns.gracefulStop(shardManager, duration, Shutdown.INSTANCE);
2113
2114                 MessageCollectorActor.expectFirstMatching(shard1, Shutdown.class);
2115                 MessageCollectorActor.expectFirstMatching(shard2, Shutdown.class);
2116
2117                 try {
2118                     Await.ready(stopFuture, FiniteDuration.create(500, TimeUnit.MILLISECONDS));
2119                     fail("ShardManager actor stopped without waiting for the Shards to be stopped");
2120                 } catch (TimeoutException e) {
2121                     // expected
2122                 }
2123
2124                 actorFactory.killActor(shard1, this);
2125                 actorFactory.killActor(shard2, this);
2126
2127                 Boolean stopped = Await.result(stopFuture, duration);
2128                 assertEquals("Stopped", Boolean.TRUE, stopped);
2129             }
2130         };
2131
2132         LOG.info("testShutDown ending");
2133     }
2134
2135     @Test
2136     public void testChangeServersVotingStatus() {
2137         new TestKit(getSystem()) {
2138             {
2139                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
2140
2141                 ActorRef respondActor = actorFactory
2142                         .createActor(Props.create(MockRespondActor.class, ChangeServersVotingStatus.class,
2143                                 new ServerChangeReply(ServerChangeStatus.OK, null)), memberId);
2144
2145                 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
2146
2147                 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
2148                 shardManager.tell(new ActorInitialized(), respondActor);
2149                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
2150                         DataStoreVersions.CURRENT_VERSION), getRef());
2151                 shardManager.tell(
2152                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
2153                         respondActor);
2154
2155                 shardManager.tell(
2156                         new ChangeShardMembersVotingStatus("default", ImmutableMap.of("member-2", Boolean.TRUE)),
2157                         getRef());
2158
2159                 ChangeServersVotingStatus actualChangeStatusMsg = MessageCollectorActor
2160                         .expectFirstMatching(respondActor, ChangeServersVotingStatus.class);
2161                 assertEquals("ChangeServersVotingStatus map", actualChangeStatusMsg.getServerVotingStatusMap(),
2162                         ImmutableMap.of(ShardIdentifier
2163                                 .create("default", MemberName.forName("member-2"), shardMrgIDSuffix).toString(),
2164                                 Boolean.TRUE));
2165
2166                 expectMsgClass(duration("5 seconds"), Success.class);
2167             }
2168         };
2169     }
2170
2171     @Test
2172     public void testChangeServersVotingStatusWithNoLeader() {
2173         new TestKit(getSystem()) {
2174             {
2175                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
2176
2177                 ActorRef respondActor = actorFactory
2178                         .createActor(Props.create(MockRespondActor.class, ChangeServersVotingStatus.class,
2179                                 new ServerChangeReply(ServerChangeStatus.NO_LEADER, null)), memberId);
2180
2181                 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
2182
2183                 shardManager.tell(new UpdateSchemaContext(TEST_SCHEMA_CONTEXT), getRef());
2184                 shardManager.tell(new ActorInitialized(), respondActor);
2185                 shardManager.tell(new RoleChangeNotification(memberId, null, RaftState.Follower.name()), respondActor);
2186
2187                 shardManager.tell(
2188                         new ChangeShardMembersVotingStatus("default", ImmutableMap.of("member-2", Boolean.TRUE)),
2189                         getRef());
2190
2191                 MessageCollectorActor.expectFirstMatching(respondActor, ChangeServersVotingStatus.class);
2192
2193                 Status.Failure resp = expectMsgClass(duration("5 seconds"), Status.Failure.class);
2194                 assertEquals("Failure resposnse", true, resp.cause() instanceof NoShardLeaderException);
2195             }
2196         };
2197     }
2198
2199     public static class TestShardManager extends ShardManager {
2200         private final CountDownLatch recoveryComplete = new CountDownLatch(1);
2201         private final CountDownLatch snapshotPersist = new CountDownLatch(1);
2202         private ShardManagerSnapshot snapshot;
2203         private final Map<String, ActorRef> shardActors;
2204         private final ActorRef shardActor;
2205         private CountDownLatch findPrimaryMessageReceived = new CountDownLatch(1);
2206         private CountDownLatch memberUpReceived = new CountDownLatch(1);
2207         private CountDownLatch memberRemovedReceived = new CountDownLatch(1);
2208         private CountDownLatch memberUnreachableReceived = new CountDownLatch(1);
2209         private CountDownLatch memberReachableReceived = new CountDownLatch(1);
2210         private volatile MessageInterceptor messageInterceptor;
2211
2212         TestShardManager(final Builder builder) {
2213             super(builder);
2214             shardActor = builder.shardActor;
2215             shardActors = builder.shardActors;
2216         }
2217
2218         @Override
2219         protected void handleRecover(final Object message) throws Exception {
2220             try {
2221                 super.handleRecover(message);
2222             } finally {
2223                 if (message instanceof RecoveryCompleted) {
2224                     recoveryComplete.countDown();
2225                 }
2226             }
2227         }
2228
2229         private void countDownIfOther(final Member member, final CountDownLatch latch) {
2230             if (!getCluster().getCurrentMemberName().equals(memberToName(member))) {
2231                 latch.countDown();
2232             }
2233         }
2234
2235         @Override
2236         public void handleCommand(final Object message) throws Exception {
2237             try {
2238                 if (messageInterceptor != null && messageInterceptor.canIntercept(message)) {
2239                     getSender().tell(messageInterceptor.apply(message), getSelf());
2240                 } else {
2241                     super.handleCommand(message);
2242                 }
2243             } finally {
2244                 if (message instanceof FindPrimary) {
2245                     findPrimaryMessageReceived.countDown();
2246                 } else if (message instanceof ClusterEvent.MemberUp) {
2247                     countDownIfOther(((ClusterEvent.MemberUp) message).member(), memberUpReceived);
2248                 } else if (message instanceof ClusterEvent.MemberRemoved) {
2249                     countDownIfOther(((ClusterEvent.MemberRemoved) message).member(), memberRemovedReceived);
2250                 } else if (message instanceof ClusterEvent.UnreachableMember) {
2251                     countDownIfOther(((ClusterEvent.UnreachableMember) message).member(), memberUnreachableReceived);
2252                 } else if (message instanceof ClusterEvent.ReachableMember) {
2253                     countDownIfOther(((ClusterEvent.ReachableMember) message).member(), memberReachableReceived);
2254                 }
2255             }
2256         }
2257
2258         void setMessageInterceptor(final MessageInterceptor messageInterceptor) {
2259             this.messageInterceptor = messageInterceptor;
2260         }
2261
2262         void waitForRecoveryComplete() {
2263             assertEquals("Recovery complete", true,
2264                     Uninterruptibles.awaitUninterruptibly(recoveryComplete, 5, TimeUnit.SECONDS));
2265         }
2266
2267         public void waitForMemberUp() {
2268             assertEquals("MemberUp received", true,
2269                     Uninterruptibles.awaitUninterruptibly(memberUpReceived, 5, TimeUnit.SECONDS));
2270             memberUpReceived = new CountDownLatch(1);
2271         }
2272
2273         void waitForMemberRemoved() {
2274             assertEquals("MemberRemoved received", true,
2275                     Uninterruptibles.awaitUninterruptibly(memberRemovedReceived, 5, TimeUnit.SECONDS));
2276             memberRemovedReceived = new CountDownLatch(1);
2277         }
2278
2279         void waitForUnreachableMember() {
2280             assertEquals("UnreachableMember received", true,
2281                 Uninterruptibles.awaitUninterruptibly(memberUnreachableReceived, 5, TimeUnit.SECONDS
2282                 ));
2283             memberUnreachableReceived = new CountDownLatch(1);
2284         }
2285
2286         void waitForReachableMember() {
2287             assertEquals("ReachableMember received", true,
2288                 Uninterruptibles.awaitUninterruptibly(memberReachableReceived, 5, TimeUnit.SECONDS));
2289             memberReachableReceived = new CountDownLatch(1);
2290         }
2291
2292         void verifyFindPrimary() {
2293             assertEquals("FindPrimary received", true,
2294                     Uninterruptibles.awaitUninterruptibly(findPrimaryMessageReceived, 5, TimeUnit.SECONDS));
2295             findPrimaryMessageReceived = new CountDownLatch(1);
2296         }
2297
2298         public static Builder builder(final DatastoreContext.Builder datastoreContextBuilder) {
2299             return new Builder(datastoreContextBuilder);
2300         }
2301
2302         public static class Builder extends AbstractGenericCreator<Builder, TestShardManager> {
2303             private ActorRef shardActor;
2304             private final Map<String, ActorRef> shardActors = new HashMap<>();
2305
2306             Builder(final DatastoreContext.Builder datastoreContextBuilder) {
2307                 super(TestShardManager.class);
2308                 datastoreContextFactory(newDatastoreContextFactory(datastoreContextBuilder.build()));
2309             }
2310
2311             Builder shardActor(final ActorRef newShardActor) {
2312                 this.shardActor = newShardActor;
2313                 return this;
2314             }
2315
2316             Builder addShardActor(final String shardName, final ActorRef actorRef) {
2317                 shardActors.put(shardName, actorRef);
2318                 return this;
2319             }
2320         }
2321
2322         @Override
2323         public void saveSnapshot(final Object obj) {
2324             snapshot = (ShardManagerSnapshot) obj;
2325             snapshotPersist.countDown();
2326             super.saveSnapshot(obj);
2327         }
2328
2329         void verifySnapshotPersisted(final Set<String> shardList) {
2330             assertEquals("saveSnapshot invoked", true,
2331                     Uninterruptibles.awaitUninterruptibly(snapshotPersist, 5, TimeUnit.SECONDS));
2332             assertEquals("Shard Persisted", shardList, Sets.newHashSet(snapshot.getShardList()));
2333         }
2334
2335         @Override
2336         protected ActorRef newShardActor(final ShardInformation info) {
2337             if (shardActors.get(info.getShardName()) != null) {
2338                 return shardActors.get(info.getShardName());
2339             }
2340
2341             if (shardActor != null) {
2342                 return shardActor;
2343             }
2344
2345             return super.newShardActor(info);
2346         }
2347     }
2348
2349     private abstract static class AbstractGenericCreator<T extends AbstractGenericCreator<T, ?>, C extends ShardManager>
2350                                                      extends AbstractShardManagerCreator<T> {
2351         private final Class<C> shardManagerClass;
2352
2353         AbstractGenericCreator(final Class<C> shardManagerClass) {
2354             this.shardManagerClass = shardManagerClass;
2355             cluster(new MockClusterWrapper()).configuration(new MockConfiguration()).waitTillReadyCountDownLatch(ready)
2356                     .primaryShardInfoCache(new PrimaryShardInfoFutureCache());
2357         }
2358
2359         @Override
2360         public Props props() {
2361             verify();
2362             return Props.create(shardManagerClass, this);
2363         }
2364     }
2365
2366     private static class GenericCreator<C extends ShardManager> extends AbstractGenericCreator<GenericCreator<C>, C> {
2367         GenericCreator(final Class<C> shardManagerClass) {
2368             super(shardManagerClass);
2369         }
2370     }
2371
2372     private static class DelegatingShardManagerCreator implements Creator<ShardManager> {
2373         private static final long serialVersionUID = 1L;
2374         private final Creator<ShardManager> delegate;
2375
2376         DelegatingShardManagerCreator(final Creator<ShardManager> delegate) {
2377             this.delegate = delegate;
2378         }
2379
2380         @Override
2381         public ShardManager create() throws Exception {
2382             return delegate.create();
2383         }
2384     }
2385
2386     interface MessageInterceptor extends Function<Object, Object> {
2387         boolean canIntercept(Object message);
2388     }
2389
2390     private static MessageInterceptor newFindPrimaryInterceptor(final ActorRef primaryActor) {
2391         return new MessageInterceptor() {
2392             @Override
2393             public Object apply(final Object message) {
2394                 return new RemotePrimaryShardFound(Serialization.serializedActorPath(primaryActor), (short) 1);
2395             }
2396
2397             @Override
2398             public boolean canIntercept(final Object message) {
2399                 return message instanceof FindPrimary;
2400             }
2401         };
2402     }
2403
2404     private static class MockRespondActor extends MessageCollectorActor {
2405         static final String CLEAR_RESPONSE = "clear-response";
2406
2407         private Object responseMsg;
2408         private final Class<?> requestClass;
2409
2410         @SuppressWarnings("unused")
2411         MockRespondActor(final Class<?> requestClass, final Object responseMsg) {
2412             this.requestClass = requestClass;
2413             this.responseMsg = responseMsg;
2414         }
2415
2416         @Override
2417         public void onReceive(final Object message) throws Exception {
2418             if (message.equals(CLEAR_RESPONSE)) {
2419                 responseMsg = null;
2420             } else {
2421                 super.onReceive(message);
2422                 if (message.getClass().equals(requestClass) && responseMsg != null) {
2423                     getSender().tell(responseMsg, getSelf());
2424                 }
2425             }
2426         }
2427     }
2428 }