opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/shardmanager/ShardManagerTest.java

   1 /*
   2  * Copyright (c) 2014, 2015 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.controller.cluster.datastore.shardmanager;
  10
  11 import static org.junit.Assert.assertEquals;
  12 import static org.junit.Assert.assertNotNull;
  13 import static org.junit.Assert.assertNull;
  14 import static org.junit.Assert.assertSame;
  15 import static org.junit.Assert.assertTrue;
  16 import static org.junit.Assert.fail;
  17 import static org.mockito.Mockito.mock;
  18 import static org.mockito.Mockito.never;
  19 import static org.mockito.Mockito.times;
  20 import static org.mockito.Mockito.verify;
  21
  22 import akka.actor.ActorRef;
  23 import akka.actor.ActorSystem;
  24 import akka.actor.AddressFromURIString;
  25 import akka.actor.Props;
  26 import akka.actor.Status;
  27 import akka.actor.Status.Failure;
  28 import akka.actor.Status.Success;
  29 import akka.cluster.Cluster;
  30 import akka.cluster.ClusterEvent;
  31 import akka.cluster.Member;
  32 import akka.dispatch.Dispatchers;
  33 import akka.japi.Creator;
  34 import akka.pattern.Patterns;
  35 import akka.persistence.RecoveryCompleted;
  36 import akka.serialization.Serialization;
  37 import akka.testkit.JavaTestKit;
  38 import akka.testkit.TestActorRef;
  39 import akka.util.Timeout;
  40 import com.google.common.base.Function;
  41 import com.google.common.base.Stopwatch;
  42 import com.google.common.collect.ImmutableMap;
  43 import com.google.common.collect.ImmutableSet;
  44 import com.google.common.collect.Lists;
  45 import com.google.common.collect.Sets;
  46 import com.google.common.util.concurrent.Uninterruptibles;
  47 import com.typesafe.config.ConfigFactory;
  48 import java.net.URI;
  49 import java.util.AbstractMap;
  50 import java.util.Arrays;
  51 import java.util.Collection;
  52 import java.util.Collections;
  53 import java.util.HashMap;
  54 import java.util.List;
  55 import java.util.Map;
  56 import java.util.Map.Entry;
  57 import java.util.Set;
  58 import java.util.concurrent.CountDownLatch;
  59 import java.util.concurrent.TimeUnit;
  60 import java.util.concurrent.TimeoutException;
  61 import java.util.stream.Collectors;
  62 import org.apache.commons.lang3.SerializationUtils;
  63 import org.junit.Test;
  64 import org.mockito.Mockito;
  65 import org.opendaylight.controller.cluster.access.concepts.MemberName;
  66 import org.opendaylight.controller.cluster.datastore.AbstractShardManagerTest;
  67 import org.opendaylight.controller.cluster.datastore.ClusterWrapperImpl;
  68 import org.opendaylight.controller.cluster.datastore.DataStoreVersions;
  69 import org.opendaylight.controller.cluster.datastore.DatastoreContext;
  70 import org.opendaylight.controller.cluster.datastore.DatastoreContextFactory;
  71 import org.opendaylight.controller.cluster.datastore.Shard;
  72 import org.opendaylight.controller.cluster.datastore.ShardManager.SchemaContextModules;
  73 import org.opendaylight.controller.cluster.datastore.config.ConfigurationImpl;
  74 import org.opendaylight.controller.cluster.datastore.config.EmptyModuleShardConfigProvider;
  75 import org.opendaylight.controller.cluster.datastore.config.ModuleShardConfiguration;
  76 import org.opendaylight.controller.cluster.datastore.exceptions.AlreadyExistsException;
  77 import org.opendaylight.controller.cluster.datastore.exceptions.NoShardLeaderException;
  78 import org.opendaylight.controller.cluster.datastore.exceptions.NotInitializedException;
  79 import org.opendaylight.controller.cluster.datastore.exceptions.PrimaryNotFoundException;
  80 import org.opendaylight.controller.cluster.datastore.identifiers.ShardIdentifier;
  81 import org.opendaylight.controller.cluster.datastore.identifiers.ShardManagerIdentifier;
  82 import org.opendaylight.controller.cluster.datastore.messages.ActorInitialized;
  83 import org.opendaylight.controller.cluster.datastore.messages.AddShardReplica;
  84 import org.opendaylight.controller.cluster.datastore.messages.ChangeShardMembersVotingStatus;
  85 import org.opendaylight.controller.cluster.datastore.messages.CreateShard;
  86 import org.opendaylight.controller.cluster.datastore.messages.DatastoreSnapshot;
  87 import org.opendaylight.controller.cluster.datastore.messages.DatastoreSnapshot.ShardSnapshot;
  88 import org.opendaylight.controller.cluster.datastore.messages.FindLocalShard;
  89 import org.opendaylight.controller.cluster.datastore.messages.FindPrimary;
  90 import org.opendaylight.controller.cluster.datastore.messages.LocalPrimaryShardFound;
  91 import org.opendaylight.controller.cluster.datastore.messages.LocalShardFound;
  92 import org.opendaylight.controller.cluster.datastore.messages.LocalShardNotFound;
  93 import org.opendaylight.controller.cluster.datastore.messages.PeerDown;
  94 import org.opendaylight.controller.cluster.datastore.messages.PeerUp;
  95 import org.opendaylight.controller.cluster.datastore.messages.PrimaryShardInfo;
  96 import org.opendaylight.controller.cluster.datastore.messages.RemotePrimaryShardFound;
  97 import org.opendaylight.controller.cluster.datastore.messages.RemoveShardReplica;
  98 import org.opendaylight.controller.cluster.datastore.messages.ShardLeaderStateChanged;
  99 import org.opendaylight.controller.cluster.datastore.messages.UpdateSchemaContext;
 100 import org.opendaylight.controller.cluster.datastore.utils.ForwardingActor;
 101 import org.opendaylight.controller.cluster.datastore.utils.MockClusterWrapper;
 102 import org.opendaylight.controller.cluster.datastore.utils.MockConfiguration;
 103 import org.opendaylight.controller.cluster.datastore.utils.PrimaryShardInfoFutureCache;
 104 import org.opendaylight.controller.cluster.notifications.LeaderStateChanged;
 105 import org.opendaylight.controller.cluster.notifications.RegisterRoleChangeListener;
 106 import org.opendaylight.controller.cluster.notifications.RoleChangeNotification;
 107 import org.opendaylight.controller.cluster.raft.RaftState;
 108 import org.opendaylight.controller.cluster.raft.base.messages.FollowerInitialSyncUpStatus;
 109 import org.opendaylight.controller.cluster.raft.base.messages.SwitchBehavior;
 110 import org.opendaylight.controller.cluster.raft.client.messages.GetSnapshot;
 111 import org.opendaylight.controller.cluster.raft.client.messages.Shutdown;
 112 import org.opendaylight.controller.cluster.raft.messages.AddServer;
 113 import org.opendaylight.controller.cluster.raft.messages.AddServerReply;
 114 import org.opendaylight.controller.cluster.raft.messages.ChangeServersVotingStatus;
 115 import org.opendaylight.controller.cluster.raft.messages.RemoveServer;
 116 import org.opendaylight.controller.cluster.raft.messages.RemoveServerReply;
 117 import org.opendaylight.controller.cluster.raft.messages.ServerChangeReply;
 118 import org.opendaylight.controller.cluster.raft.messages.ServerChangeStatus;
 119 import org.opendaylight.controller.cluster.raft.messages.ServerRemoved;
 120 import org.opendaylight.controller.cluster.raft.policy.DisableElectionsRaftPolicy;
 121 import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal;
 122 import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore;
 123 import org.opendaylight.controller.cluster.raft.utils.MessageCollectorActor;
 124 import org.opendaylight.controller.md.cluster.datastore.model.TestModel;
 125 import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTree;
 126 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
 127 import org.slf4j.Logger;
 128 import org.slf4j.LoggerFactory;
 129 import scala.concurrent.Await;
 130 import scala.concurrent.Future;
 131 import scala.concurrent.duration.FiniteDuration;
 132
 133 public class ShardManagerTest extends AbstractShardManagerTest {
 134     private static final Logger LOG = LoggerFactory.getLogger(ShardManagerTest.class);
 135     private static final MemberName MEMBER_2 = MemberName.forName("member-2");
 136     private static final MemberName MEMBER_3 = MemberName.forName("member-3");
 137
 138     private final String shardMgrID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
 139
 140     private ActorSystem newActorSystem(String config) {
 141         ActorSystem system = ActorSystem.create("cluster-test", ConfigFactory.load().getConfig(config));
 142         actorSystems.add(system);
 143         return system;
 144     }
 145
 146     private ActorRef newMockShardActor(ActorSystem system, String shardName, String memberName) {
 147         String name = ShardIdentifier.create(shardName, MemberName.forName(memberName), "config").toString();
 148         if (system == getSystem()) {
 149             return actorFactory.createTestActor(Props.create(MessageCollectorActor.class), name);
 150         }
 151
 152         return TestActorRef.create(system, Props.create(MessageCollectorActor.class), name);
 153     }
 154
 155     private Props newShardMgrProps() {
 156         return newShardMgrProps(new MockConfiguration());
 157     }
 158
 159     private static DatastoreContextFactory newDatastoreContextFactory(DatastoreContext datastoreContext) {
 160         DatastoreContextFactory mockFactory = mock(DatastoreContextFactory.class);
 161         Mockito.doReturn(datastoreContext).when(mockFactory).getBaseDatastoreContext();
 162         Mockito.doReturn(datastoreContext).when(mockFactory).getShardDatastoreContext(Mockito.anyString());
 163         return mockFactory;
 164     }
 165
 166     private TestShardManager.Builder newTestShardMgrBuilderWithMockShardActor() {
 167         return newTestShardMgrBuilderWithMockShardActor(mockShardActor);
 168     }
 169
 170     private TestShardManager.Builder newTestShardMgrBuilderWithMockShardActor(ActorRef shardActor) {
 171         return TestShardManager.builder(datastoreContextBuilder).shardActor(shardActor);
 172     }
 173
 174
 175     private Props newPropsShardMgrWithMockShardActor() {
 176         return newTestShardMgrBuilderWithMockShardActor().props().withDispatcher(
 177                 Dispatchers.DefaultDispatcherId());
 178     }
 179
 180     private Props newPropsShardMgrWithMockShardActor(ActorRef shardActor) {
 181         return newTestShardMgrBuilderWithMockShardActor(shardActor).props();
 182     }
 183
 184
 185     private TestShardManager newTestShardManager() {
 186         return newTestShardManager(newShardMgrProps());
 187     }
 188
 189     private TestShardManager newTestShardManager(Props props) {
 190         TestActorRef<TestShardManager> shardManagerActor = actorFactory.createTestActor(props);
 191         TestShardManager shardManager = shardManagerActor.underlyingActor();
 192         shardManager.waitForRecoveryComplete();
 193         return shardManager;
 194     }
 195
 196     private static void waitForShardInitialized(ActorRef shardManager, String shardName, JavaTestKit kit) {
 197         AssertionError last = null;
 198         Stopwatch sw = Stopwatch.createStarted();
 199         while (sw.elapsed(TimeUnit.SECONDS) <= 5) {
 200             try {
 201                 shardManager.tell(new FindLocalShard(shardName, true), kit.getRef());
 202                 kit.expectMsgClass(LocalShardFound.class);
 203                 return;
 204             } catch (AssertionError e) {
 205                 last = e;
 206             }
 207
 208             Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
 209         }
 210
 211         throw last;
 212     }
 213
 214     @SuppressWarnings("unchecked")
 215     private static <T> T expectMsgClassOrFailure(Class<T> msgClass, JavaTestKit kit, String msg) {
 216         Object reply = kit.expectMsgAnyClassOf(JavaTestKit.duration("5 sec"), msgClass, Failure.class);
 217         if (reply instanceof Failure) {
 218             throw new AssertionError(msg + " failed", ((Failure)reply).cause());
 219         }
 220
 221         return (T)reply;
 222     }
 223
 224     @Test
 225     public void testPerShardDatastoreContext() throws Exception {
 226         LOG.info("testPerShardDatastoreContext starting");
 227         final DatastoreContextFactory mockFactory = newDatastoreContextFactory(
 228                 datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
 229
 230         Mockito.doReturn(
 231                 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(6).build())
 232                 .when(mockFactory).getShardDatastoreContext("default");
 233
 234         Mockito.doReturn(
 235                 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(7).build())
 236                 .when(mockFactory).getShardDatastoreContext("topology");
 237
 238         final MockConfiguration mockConfig = new MockConfiguration() {
 239             @Override
 240             public Collection<String> getMemberShardNames(MemberName memberName) {
 241                 return Arrays.asList("default", "topology");
 242             }
 243
 244             @Override
 245             public Collection<MemberName> getMembersFromShardName(String shardName) {
 246                 return members("member-1");
 247             }
 248         };
 249
 250         final TestActorRef<MessageCollectorActor> defaultShardActor = actorFactory.createTestActor(
 251                 Props.create(MessageCollectorActor.class), actorFactory.generateActorId("default"));
 252         final TestActorRef<MessageCollectorActor> topologyShardActor = actorFactory.createTestActor(
 253                 Props.create(MessageCollectorActor.class), actorFactory.generateActorId("topology"));
 254
 255         final Map<String, Entry<ActorRef, DatastoreContext>> shardInfoMap = Collections.synchronizedMap(
 256                 new HashMap<String, Entry<ActorRef, DatastoreContext>>());
 257         shardInfoMap.put("default", new AbstractMap.SimpleEntry<>(defaultShardActor, null));
 258         shardInfoMap.put("topology", new AbstractMap.SimpleEntry<>(topologyShardActor, null));
 259
 260         final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
 261         final CountDownLatch newShardActorLatch = new CountDownLatch(2);
 262         class LocalShardManager extends ShardManager {
 263             LocalShardManager(AbstractShardManagerCreator<?> creator) {
 264                 super(creator);
 265             }
 266
 267             @Override
 268             protected ActorRef newShardActor(SchemaContext schemaContext, ShardInformation info) {
 269                 Entry<ActorRef, DatastoreContext> entry = shardInfoMap.get(info.getShardName());
 270                 ActorRef ref = null;
 271                 if (entry != null) {
 272                     ref = entry.getKey();
 273                     entry.setValue(info.getDatastoreContext());
 274                 }
 275
 276                 newShardActorLatch.countDown();
 277                 return ref;
 278             }
 279         }
 280
 281         final Creator<ShardManager> creator = new Creator<ShardManager>() {
 282             private static final long serialVersionUID = 1L;
 283             @Override
 284             public ShardManager create() throws Exception {
 285                 return new LocalShardManager(
 286                         new GenericCreator<>(LocalShardManager.class).datastoreContextFactory(mockFactory)
 287                                 .primaryShardInfoCache(primaryShardInfoCache).configuration(mockConfig));
 288             }
 289         };
 290
 291         JavaTestKit kit = new JavaTestKit(getSystem());
 292
 293         final ActorRef shardManager = actorFactory.createActor(Props.create(
 294                 new DelegatingShardManagerCreator(creator)).withDispatcher(Dispatchers.DefaultDispatcherId()));
 295
 296         shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), kit.getRef());
 297
 298         assertEquals("Shard actors created", true, newShardActorLatch.await(5, TimeUnit.SECONDS));
 299         assertEquals("getShardElectionTimeoutFactor", 6,
 300                 shardInfoMap.get("default").getValue().getShardElectionTimeoutFactor());
 301         assertEquals("getShardElectionTimeoutFactor", 7,
 302                 shardInfoMap.get("topology").getValue().getShardElectionTimeoutFactor());
 303
 304         DatastoreContextFactory newMockFactory = newDatastoreContextFactory(
 305                 datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
 306         Mockito.doReturn(
 307                 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(66).build())
 308                 .when(newMockFactory).getShardDatastoreContext("default");
 309
 310         Mockito.doReturn(
 311                 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(77).build())
 312                 .when(newMockFactory).getShardDatastoreContext("topology");
 313
 314         shardManager.tell(newMockFactory, kit.getRef());
 315
 316         DatastoreContext newContext = MessageCollectorActor.expectFirstMatching(defaultShardActor,
 317                 DatastoreContext.class);
 318         assertEquals("getShardElectionTimeoutFactor", 66, newContext.getShardElectionTimeoutFactor());
 319
 320         newContext = MessageCollectorActor.expectFirstMatching(topologyShardActor, DatastoreContext.class);
 321         assertEquals("getShardElectionTimeoutFactor", 77, newContext.getShardElectionTimeoutFactor());
 322
 323         LOG.info("testPerShardDatastoreContext ending");
 324     }
 325
 326     @Test
 327     public void testOnReceiveFindPrimaryForNonExistentShard() throws Exception {
 328         new JavaTestKit(getSystem()) {
 329             {
 330                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 331
 332                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 333
 334                 shardManager.tell(new FindPrimary("non-existent", false), getRef());
 335
 336                 expectMsgClass(duration("5 seconds"), PrimaryNotFoundException.class);
 337             }
 338         };
 339     }
 340
 341     @Test
 342     public void testOnReceiveFindPrimaryForLocalLeaderShard() throws Exception {
 343         LOG.info("testOnReceiveFindPrimaryForLocalLeaderShard starting");
 344         new JavaTestKit(getSystem()) {
 345             {
 346                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
 347
 348                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 349
 350                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 351                 shardManager.tell(new ActorInitialized(), mockShardActor);
 352
 353                 DataTree mockDataTree = mock(DataTree.class);
 354                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
 355                         DataStoreVersions.CURRENT_VERSION), getRef());
 356
 357                 MessageCollectorActor.expectFirstMatching(mockShardActor, RegisterRoleChangeListener.class);
 358                 shardManager.tell(
 359                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
 360                         mockShardActor);
 361
 362                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 363
 364                 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
 365                         LocalPrimaryShardFound.class);
 366                 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
 367                         primaryFound.getPrimaryPath().contains("member-1-shard-default"));
 368                 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
 369             }
 370         };
 371
 372         LOG.info("testOnReceiveFindPrimaryForLocalLeaderShard ending");
 373     }
 374
 375     @Test
 376     public void testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp() throws Exception {
 377         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp starting");
 378         new JavaTestKit(getSystem()) {
 379             {
 380                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 381
 382                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 383                 shardManager.tell(new ActorInitialized(), mockShardActor);
 384
 385                 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
 386                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
 387                 shardManager.tell(
 388                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
 389                         mockShardActor);
 390                 shardManager.tell(new LeaderStateChanged(memberId1, memberId2, DataStoreVersions.CURRENT_VERSION),
 391                         mockShardActor);
 392
 393                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 394
 395                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 396             }
 397         };
 398
 399         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp ending");
 400     }
 401
 402     @Test
 403     public void testOnReceiveFindPrimaryForNonLocalLeaderShard() throws Exception {
 404         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShard starting");
 405         new JavaTestKit(getSystem()) {
 406             {
 407                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 408
 409                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 410                 shardManager.tell(new ActorInitialized(), mockShardActor);
 411
 412                 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
 413                 MockClusterWrapper.sendMemberUp(shardManager, "member-2", getRef().path().toString());
 414
 415                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
 416                 shardManager.tell(
 417                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
 418                         mockShardActor);
 419                 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
 420                 shardManager.tell(new ShardLeaderStateChanged(memberId1, memberId2, leaderVersion), mockShardActor);
 421
 422                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 423
 424                 RemotePrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
 425                         RemotePrimaryShardFound.class);
 426                 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
 427                         primaryFound.getPrimaryPath().contains("member-2-shard-default"));
 428                 assertEquals("getPrimaryVersion", leaderVersion, primaryFound.getPrimaryVersion());
 429             }
 430         };
 431
 432         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShard ending");
 433     }
 434
 435     @Test
 436     public void testOnReceiveFindPrimaryForUninitializedShard() throws Exception {
 437         new JavaTestKit(getSystem()) {
 438             {
 439                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 440
 441                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 442
 443                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
 444             }
 445         };
 446     }
 447
 448     @Test
 449     public void testOnReceiveFindPrimaryForInitializedShardWithNoRole() throws Exception {
 450         new JavaTestKit(getSystem()) {
 451             {
 452                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 453
 454                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 455                 shardManager.tell(new ActorInitialized(), mockShardActor);
 456
 457                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 458
 459                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 460             }
 461         };
 462     }
 463
 464     @Test
 465     public void testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId() throws Exception {
 466         LOG.info("testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId starting");
 467         new JavaTestKit(getSystem()) {
 468             {
 469                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 470
 471                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 472                 shardManager.tell(new ActorInitialized(), mockShardActor);
 473
 474                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
 475                 shardManager.tell(
 476                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Follower.name()),
 477                         mockShardActor);
 478
 479                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 480
 481                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 482
 483                 DataTree mockDataTree = mock(DataTree.class);
 484                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
 485                         DataStoreVersions.CURRENT_VERSION), mockShardActor);
 486
 487                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 488
 489                 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
 490                         LocalPrimaryShardFound.class);
 491                 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
 492                         primaryFound.getPrimaryPath().contains("member-1-shard-default"));
 493                 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
 494             }
 495         };
 496
 497         LOG.info("testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId starting");
 498     }
 499
 500     @Test
 501     public void testOnReceiveFindPrimaryWaitForShardLeader() throws Exception {
 502         LOG.info("testOnReceiveFindPrimaryWaitForShardLeader starting");
 503         datastoreContextBuilder.shardInitializationTimeout(10, TimeUnit.SECONDS);
 504         new JavaTestKit(getSystem()) {
 505             {
 506                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 507
 508                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 509
 510                 // We're passing waitUntilInitialized = true to FindPrimary so
 511                 // the response should be
 512                 // delayed until we send ActorInitialized and
 513                 // RoleChangeNotification.
 514                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
 515
 516                 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
 517
 518                 shardManager.tell(new ActorInitialized(), mockShardActor);
 519
 520                 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
 521
 522                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
 523                 shardManager.tell(
 524                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
 525                         mockShardActor);
 526
 527                 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
 528
 529                 DataTree mockDataTree = mock(DataTree.class);
 530                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
 531                         DataStoreVersions.CURRENT_VERSION), mockShardActor);
 532
 533                 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
 534                         LocalPrimaryShardFound.class);
 535                 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
 536                         primaryFound.getPrimaryPath().contains("member-1-shard-default"));
 537                 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
 538
 539                 expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
 540             }
 541         };
 542
 543         LOG.info("testOnReceiveFindPrimaryWaitForShardLeader ending");
 544     }
 545
 546     @Test
 547     public void testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard() throws Exception {
 548         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard starting");
 549         new JavaTestKit(getSystem()) {
 550             {
 551                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 552
 553                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 554
 555                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
 556
 557                 expectMsgClass(duration("2 seconds"), NotInitializedException.class);
 558
 559                 shardManager.tell(new ActorInitialized(), mockShardActor);
 560
 561                 expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
 562             }
 563         };
 564
 565         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard ending");
 566     }
 567
 568     @Test
 569     public void testOnReceiveFindPrimaryWaitForReadyWithCandidateShard() throws Exception {
 570         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithCandidateShard starting");
 571         new JavaTestKit(getSystem()) {
 572             {
 573                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 574
 575                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 576                 shardManager.tell(new ActorInitialized(), mockShardActor);
 577                 shardManager.tell(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix, null,
 578                         RaftState.Candidate.name()), mockShardActor);
 579
 580                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
 581
 582                 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
 583             }
 584         };
 585
 586         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithCandidateShard ending");
 587     }
 588
 589     @Test
 590     public void testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard() throws Exception {
 591         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard starting");
 592         new JavaTestKit(getSystem()) {
 593             {
 594                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 595
 596                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 597                 shardManager.tell(new ActorInitialized(), mockShardActor);
 598                 shardManager.tell(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix, null,
 599                         RaftState.IsolatedLeader.name()), mockShardActor);
 600
 601                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
 602
 603                 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
 604             }
 605         };
 606
 607         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard ending");
 608     }
 609
 610     @Test
 611     public void testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard() throws Exception {
 612         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard starting");
 613         new JavaTestKit(getSystem()) {
 614             {
 615                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 616
 617                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 618                 shardManager.tell(new ActorInitialized(), mockShardActor);
 619
 620                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
 621
 622                 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
 623             }
 624         };
 625
 626         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard ending");
 627     }
 628
 629     @Test
 630     public void testOnReceiveFindPrimaryForRemoteShard() throws Exception {
 631         LOG.info("testOnReceiveFindPrimaryForRemoteShard starting");
 632         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
 633
 634         // Create an ActorSystem ShardManager actor for member-1.
 635
 636         final ActorSystem system1 = newActorSystem("Member1");
 637         Cluster.get(system1).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
 638
 639         final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
 640                 newTestShardMgrBuilderWithMockShardActor().cluster(
 641                         new ClusterWrapperImpl(system1)).props().withDispatcher(
 642                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
 643
 644         // Create an ActorSystem ShardManager actor for member-2.
 645
 646         final ActorSystem system2 = newActorSystem("Member2");
 647
 648         Cluster.get(system2).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
 649
 650         final ActorRef mockShardActor2 = newMockShardActor(system2, "astronauts", "member-2");
 651
 652         MockConfiguration mockConfig2 = new MockConfiguration(
 653                 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
 654                         .put("astronauts", Arrays.asList("member-2")).build());
 655
 656         final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
 657                 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
 658                         new ClusterWrapperImpl(system2)).props().withDispatcher(
 659                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
 660
 661         new JavaTestKit(system1) {
 662             {
 663                 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 664                 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 665
 666                 shardManager2.tell(new ActorInitialized(), mockShardActor2);
 667
 668                 String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
 669                 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
 670                 shardManager2.tell(
 671                         new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
 672                         mockShardActor2);
 673                 shardManager2.tell(
 674                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
 675                         mockShardActor2);
 676
 677                 shardManager1.underlyingActor().waitForMemberUp();
 678                 shardManager1.tell(new FindPrimary("astronauts", false), getRef());
 679
 680                 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 681                 String path = found.getPrimaryPath();
 682                 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-astronauts-config"));
 683                 assertEquals("getPrimaryVersion", leaderVersion, found.getPrimaryVersion());
 684
 685                 shardManager2.underlyingActor().verifyFindPrimary();
 686
 687                 Cluster.get(system2).down(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
 688
 689                 shardManager1.underlyingActor().waitForMemberRemoved();
 690
 691                 shardManager1.tell(new FindPrimary("astronauts", false), getRef());
 692
 693                 expectMsgClass(duration("5 seconds"), PrimaryNotFoundException.class);
 694             }
 695         };
 696
 697         LOG.info("testOnReceiveFindPrimaryForRemoteShard ending");
 698     }
 699
 700     @Test
 701     public void testShardAvailabilityOnChangeOfMemberReachability() throws Exception {
 702         LOG.info("testShardAvailabilityOnChangeOfMemberReachability starting");
 703         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
 704
 705         // Create an ActorSystem ShardManager actor for member-1.
 706
 707         final ActorSystem system1 = newActorSystem("Member1");
 708         Cluster.get(system1).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
 709
 710         final ActorRef mockShardActor1 = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
 711
 712         final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
 713                 newTestShardMgrBuilder().shardActor(mockShardActor1).cluster(
 714                         new ClusterWrapperImpl(system1)).props().withDispatcher(
 715                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
 716
 717         // Create an ActorSystem ShardManager actor for member-2.
 718
 719         final ActorSystem system2 = newActorSystem("Member2");
 720
 721         Cluster.get(system2).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
 722
 723         final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
 724
 725         MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
 726                 .put("default", Arrays.asList("member-1", "member-2")).build());
 727
 728         final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
 729                 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
 730                         new ClusterWrapperImpl(system2)).props().withDispatcher(
 731                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
 732
 733         new JavaTestKit(system1) {
 734             {
 735                 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 736                 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 737                 shardManager1.tell(new ActorInitialized(), mockShardActor1);
 738                 shardManager2.tell(new ActorInitialized(), mockShardActor2);
 739
 740                 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
 741                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
 742                 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class),
 743                         DataStoreVersions.CURRENT_VERSION), mockShardActor1);
 744                 shardManager1.tell(
 745                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
 746                         mockShardActor1);
 747                 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class),
 748                         DataStoreVersions.CURRENT_VERSION), mockShardActor2);
 749                 shardManager2.tell(
 750                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
 751                         mockShardActor2);
 752                 shardManager1.underlyingActor().waitForMemberUp();
 753
 754                 shardManager1.tell(new FindPrimary("default", true), getRef());
 755
 756                 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 757                 String path = found.getPrimaryPath();
 758                 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
 759
 760                 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
 761                         "akka.tcp://cluster-test@127.0.0.1:2558"), getRef());
 762
 763                 shardManager1.underlyingActor().waitForUnreachableMember();
 764
 765                 PeerDown peerDown = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
 766                 assertEquals("getMemberName", MEMBER_2, peerDown.getMemberName());
 767                 MessageCollectorActor.clearMessages(mockShardActor1);
 768
 769                 shardManager1.tell(
 770                         MockClusterWrapper.createMemberRemoved("member-2", "akka.tcp://cluster-test@127.0.0.1:2558"),
 771                         getRef());
 772
 773                 MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
 774
 775                 shardManager1.tell(new FindPrimary("default", true), getRef());
 776
 777                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 778
 779                 shardManager1.tell(
 780                         MockClusterWrapper.createReachableMember("member-2", "akka.tcp://cluster-test@127.0.0.1:2558"),
 781                         getRef());
 782
 783                 shardManager1.underlyingActor().waitForReachableMember();
 784
 785                 PeerUp peerUp = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
 786                 assertEquals("getMemberName", MEMBER_2, peerUp.getMemberName());
 787                 MessageCollectorActor.clearMessages(mockShardActor1);
 788
 789                 shardManager1.tell(new FindPrimary("default", true), getRef());
 790
 791                 RemotePrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 792                 String path1 = found1.getPrimaryPath();
 793                 assertTrue("Unexpected primary path " + path1, path1.contains("member-2-shard-default-config"));
 794
 795                 shardManager1.tell(
 796                         MockClusterWrapper.createMemberUp("member-2", "akka.tcp://cluster-test@127.0.0.1:2558"),
 797                         getRef());
 798
 799                 MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
 800
 801                 // Test FindPrimary wait succeeds after reachable member event.
 802
 803                 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
 804                         "akka.tcp://cluster-test@127.0.0.1:2558"), getRef());
 805                 shardManager1.underlyingActor().waitForUnreachableMember();
 806
 807                 shardManager1.tell(new FindPrimary("default", true), getRef());
 808
 809                 shardManager1.tell(
 810                         MockClusterWrapper.createReachableMember("member-2", "akka.tcp://cluster-test@127.0.0.1:2558"),
 811                         getRef());
 812
 813                 RemotePrimaryShardFound found2 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 814                 String path2 = found2.getPrimaryPath();
 815                 assertTrue("Unexpected primary path " + path2, path2.contains("member-2-shard-default-config"));
 816             }
 817         };
 818
 819         LOG.info("testShardAvailabilityOnChangeOfMemberReachability ending");
 820     }
 821
 822     @Test
 823     public void testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange() throws Exception {
 824         LOG.info("testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange starting");
 825         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
 826
 827         // Create an ActorSystem ShardManager actor for member-1.
 828
 829         final ActorSystem system1 = newActorSystem("Member1");
 830         Cluster.get(system1).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
 831
 832         final ActorRef mockShardActor1 = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
 833
 834         final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
 835         final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
 836                 newTestShardMgrBuilder().shardActor(mockShardActor1).cluster(new ClusterWrapperImpl(system1))
 837                         .primaryShardInfoCache(primaryShardInfoCache).props()
 838                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
 839                 shardManagerID);
 840
 841         // Create an ActorSystem ShardManager actor for member-2.
 842
 843         final ActorSystem system2 = newActorSystem("Member2");
 844
 845         Cluster.get(system2).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
 846
 847         final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
 848
 849         MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
 850                 .put("default", Arrays.asList("member-1", "member-2")).build());
 851
 852         final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
 853                 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
 854                         new ClusterWrapperImpl(system2)).props().withDispatcher(
 855                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
 856
 857         new JavaTestKit(system1) {
 858             {
 859                 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 860                 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 861                 shardManager1.tell(new ActorInitialized(), mockShardActor1);
 862                 shardManager2.tell(new ActorInitialized(), mockShardActor2);
 863
 864                 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
 865                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
 866                 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class),
 867                         DataStoreVersions.CURRENT_VERSION), mockShardActor1);
 868                 shardManager1.tell(
 869                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
 870                         mockShardActor1);
 871                 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class),
 872                         DataStoreVersions.CURRENT_VERSION), mockShardActor2);
 873                 shardManager2.tell(
 874                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
 875                         mockShardActor2);
 876                 shardManager1.underlyingActor().waitForMemberUp();
 877
 878                 shardManager1.tell(new FindPrimary("default", true), getRef());
 879
 880                 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 881                 String path = found.getPrimaryPath();
 882                 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
 883
 884                 primaryShardInfoCache.putSuccessful("default", new PrimaryShardInfo(
 885                         system1.actorSelection(mockShardActor1.path()), DataStoreVersions.CURRENT_VERSION));
 886
 887                 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
 888                         "akka.tcp://cluster-test@127.0.0.1:2558"), getRef());
 889
 890                 shardManager1.underlyingActor().waitForUnreachableMember();
 891
 892                 shardManager1.tell(new FindPrimary("default", true), getRef());
 893
 894                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 895
 896                 assertNull("Expected primaryShardInfoCache entry removed",
 897                         primaryShardInfoCache.getIfPresent("default"));
 898
 899                 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId1, mock(DataTree.class),
 900                         DataStoreVersions.CURRENT_VERSION), mockShardActor1);
 901                 shardManager1.tell(
 902                         new RoleChangeNotification(memberId1, RaftState.Follower.name(), RaftState.Leader.name()),
 903                         mockShardActor1);
 904
 905                 shardManager1.tell(new FindPrimary("default", true), getRef());
 906
 907                 LocalPrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
 908                 String path1 = found1.getPrimaryPath();
 909                 assertTrue("Unexpected primary path " + path1, path1.contains("member-1-shard-default-config"));
 910
 911             }
 912         };
 913
 914         LOG.info("testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange ending");
 915     }
 916
 917
 918     @Test
 919     public void testOnReceiveFindLocalShardForNonExistentShard() throws Exception {
 920         new JavaTestKit(getSystem()) {
 921             {
 922                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 923
 924                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 925
 926                 shardManager.tell(new FindLocalShard("non-existent", false), getRef());
 927
 928                 LocalShardNotFound notFound = expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
 929
 930                 assertEquals("getShardName", "non-existent", notFound.getShardName());
 931             }
 932         };
 933     }
 934
 935     @Test
 936     public void testOnReceiveFindLocalShardForExistentShard() throws Exception {
 937         new JavaTestKit(getSystem()) {
 938             {
 939                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 940
 941                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 942                 shardManager.tell(new ActorInitialized(), mockShardActor);
 943
 944                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
 945
 946                 LocalShardFound found = expectMsgClass(duration("5 seconds"), LocalShardFound.class);
 947
 948                 assertTrue("Found path contains " + found.getPath().path().toString(),
 949                         found.getPath().path().toString().contains("member-1-shard-default-config"));
 950             }
 951         };
 952     }
 953
 954     @Test
 955     public void testOnReceiveFindLocalShardForNotInitializedShard() throws Exception {
 956         new JavaTestKit(getSystem()) {
 957             {
 958                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 959
 960                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
 961
 962                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
 963             }
 964         };
 965     }
 966
 967     @Test
 968     public void testOnReceiveFindLocalShardWaitForShardInitialized() throws Exception {
 969         LOG.info("testOnReceiveFindLocalShardWaitForShardInitialized starting");
 970         new JavaTestKit(getSystem()) {
 971             {
 972                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 973
 974                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 975
 976                 // We're passing waitUntilInitialized = true to FindLocalShard
 977                 // so the response should be
 978                 // delayed until we send ActorInitialized.
 979                 Future<Object> future = Patterns.ask(shardManager, new FindLocalShard(Shard.DEFAULT_NAME, true),
 980                         new Timeout(5, TimeUnit.SECONDS));
 981
 982                 shardManager.tell(new ActorInitialized(), mockShardActor);
 983
 984                 Object resp = Await.result(future, duration("5 seconds"));
 985                 assertTrue("Expected: LocalShardFound, Actual: " + resp, resp instanceof LocalShardFound);
 986             }
 987         };
 988
 989         LOG.info("testOnReceiveFindLocalShardWaitForShardInitialized starting");
 990     }
 991
 992     @Test
 993     public void testOnRecoveryJournalIsCleaned() {
 994         String persistenceID = "shard-manager-" + shardMrgIDSuffix;
 995         InMemoryJournal.addEntry(persistenceID, 1L, new SchemaContextModules(ImmutableSet.of("foo")));
 996         InMemoryJournal.addEntry(persistenceID, 2L, new SchemaContextModules(ImmutableSet.of("bar")));
 997         InMemoryJournal.addDeleteMessagesCompleteLatch(persistenceID);
 998
 999         newTestShardManager();
1000
1001         InMemoryJournal.waitForDeleteMessagesComplete(persistenceID);
1002
1003         // Journal entries up to the last one should've been deleted
1004         Map<Long, Object> journal = InMemoryJournal.get(persistenceID);
1005         synchronized (journal) {
1006             assertEquals("Journal size", 0, journal.size());
1007         }
1008     }
1009
1010     @Test
1011     public void testRoleChangeNotificationAndShardLeaderStateChangedReleaseReady() throws Exception {
1012         TestShardManager shardManager = newTestShardManager();
1013
1014         String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1015         shardManager.onReceiveCommand(new RoleChangeNotification(
1016                 memberId, RaftState.Candidate.name(), RaftState.Leader.name()));
1017
1018         verify(ready, never()).countDown();
1019
1020         shardManager.onReceiveCommand(new ShardLeaderStateChanged(memberId, memberId,
1021                 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1022
1023         verify(ready, times(1)).countDown();
1024     }
1025
1026     @Test
1027     public void testRoleChangeNotificationToFollowerWithShardLeaderStateChangedReleaseReady() throws Exception {
1028         new JavaTestKit(getSystem()) {
1029             {
1030                 TestShardManager shardManager = newTestShardManager();
1031
1032                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1033                 shardManager.onReceiveCommand(new RoleChangeNotification(memberId, null, RaftState.Follower.name()));
1034
1035                 verify(ready, never()).countDown();
1036
1037                 shardManager
1038                         .onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
1039
1040                 shardManager.onReceiveCommand(
1041                         new ShardLeaderStateChanged(memberId, "member-2-shard-default-" + shardMrgIDSuffix,
1042                                 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1043
1044                 verify(ready, times(1)).countDown();
1045             }
1046         };
1047     }
1048
1049     @Test
1050     public void testReadyCountDownForMemberUpAfterLeaderStateChanged() throws Exception {
1051         new JavaTestKit(getSystem()) {
1052             {
1053                 TestShardManager shardManager = newTestShardManager();
1054
1055                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1056                 shardManager.onReceiveCommand(new RoleChangeNotification(memberId, null, RaftState.Follower.name()));
1057
1058                 verify(ready, never()).countDown();
1059
1060                 shardManager.onReceiveCommand(
1061                         new ShardLeaderStateChanged(memberId, "member-2-shard-default-" + shardMrgIDSuffix,
1062                                 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1063
1064                 shardManager
1065                         .onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
1066
1067                 verify(ready, times(1)).countDown();
1068             }
1069         };
1070     }
1071
1072     @Test
1073     public void testRoleChangeNotificationDoNothingForUnknownShard() throws Exception {
1074         TestShardManager shardManager = newTestShardManager();
1075
1076         shardManager.onReceiveCommand(new RoleChangeNotification(
1077                 "unknown", RaftState.Candidate.name(), RaftState.Leader.name()));
1078
1079         verify(ready, never()).countDown();
1080     }
1081
1082     @Test
1083     public void testByDefaultSyncStatusIsFalse() throws Exception {
1084         TestShardManager shardManager = newTestShardManager();
1085
1086         assertEquals(false, shardManager.getMBean().getSyncStatus());
1087     }
1088
1089     @Test
1090     public void testWhenShardIsLeaderSyncStatusIsTrue() throws Exception {
1091         TestShardManager shardManager = newTestShardManager();
1092
1093         shardManager.onReceiveCommand(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix,
1094                 RaftState.Follower.name(), RaftState.Leader.name()));
1095
1096         assertEquals(true, shardManager.getMBean().getSyncStatus());
1097     }
1098
1099     @Test
1100     public void testWhenShardIsCandidateSyncStatusIsFalse() throws Exception {
1101         TestShardManager shardManager = newTestShardManager();
1102
1103         String shardId = "member-1-shard-default-" + shardMrgIDSuffix;
1104         shardManager.onReceiveCommand(new RoleChangeNotification(shardId,
1105                 RaftState.Follower.name(), RaftState.Candidate.name()));
1106
1107         assertEquals(false, shardManager.getMBean().getSyncStatus());
1108
1109         // Send a FollowerInitialSyncStatus with status = true for the replica whose current state is candidate
1110         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(
1111                 true, shardId));
1112
1113         assertEquals(false, shardManager.getMBean().getSyncStatus());
1114     }
1115
1116     @Test
1117     public void testWhenShardIsFollowerSyncStatusDependsOnFollowerInitialSyncStatus() throws Exception {
1118         TestShardManager shardManager = newTestShardManager();
1119
1120         String shardId = "member-1-shard-default-" + shardMrgIDSuffix;
1121         shardManager.onReceiveCommand(new RoleChangeNotification(shardId,
1122                 RaftState.Candidate.name(), RaftState.Follower.name()));
1123
1124         // Initially will be false
1125         assertEquals(false, shardManager.getMBean().getSyncStatus());
1126
1127         // Send status true will make sync status true
1128         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(true, shardId));
1129
1130         assertEquals(true, shardManager.getMBean().getSyncStatus());
1131
1132         // Send status false will make sync status false
1133         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(false, shardId));
1134
1135         assertEquals(false, shardManager.getMBean().getSyncStatus());
1136
1137     }
1138
1139     @Test
1140     public void testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards() throws Exception {
1141         LOG.info("testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards starting");
1142         TestShardManager shardManager = newTestShardManager(newShardMgrProps(new MockConfiguration() {
1143             @Override
1144             public List<String> getMemberShardNames(MemberName memberName) {
1145                 return Arrays.asList("default", "astronauts");
1146             }
1147         }));
1148
1149         // Initially will be false
1150         assertEquals(false, shardManager.getMBean().getSyncStatus());
1151
1152         // Make default shard leader
1153         String defaultShardId = "member-1-shard-default-" + shardMrgIDSuffix;
1154         shardManager.onReceiveCommand(new RoleChangeNotification(defaultShardId,
1155                 RaftState.Follower.name(), RaftState.Leader.name()));
1156
1157         // default = Leader, astronauts is unknown so sync status remains false
1158         assertEquals(false, shardManager.getMBean().getSyncStatus());
1159
1160         // Make astronauts shard leader as well
1161         String astronautsShardId = "member-1-shard-astronauts-" + shardMrgIDSuffix;
1162         shardManager.onReceiveCommand(new RoleChangeNotification(astronautsShardId,
1163                 RaftState.Follower.name(), RaftState.Leader.name()));
1164
1165         // Now sync status should be true
1166         assertEquals(true, shardManager.getMBean().getSyncStatus());
1167
1168         // Make astronauts a Follower
1169         shardManager.onReceiveCommand(new RoleChangeNotification(astronautsShardId,
1170                 RaftState.Leader.name(), RaftState.Follower.name()));
1171
1172         // Sync status is not true
1173         assertEquals(false, shardManager.getMBean().getSyncStatus());
1174
1175         // Make the astronauts follower sync status true
1176         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(true, astronautsShardId));
1177
1178         // Sync status is now true
1179         assertEquals(true, shardManager.getMBean().getSyncStatus());
1180
1181         LOG.info("testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards ending");
1182     }
1183
1184     @Test
1185     public void testOnReceiveSwitchShardBehavior() throws Exception {
1186         new JavaTestKit(getSystem()) {
1187             {
1188                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1189
1190                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1191                 shardManager.tell(new ActorInitialized(), mockShardActor);
1192
1193                 shardManager.tell(new SwitchShardBehavior(mockShardName, RaftState.Leader, 1000), getRef());
1194
1195                 SwitchBehavior switchBehavior = MessageCollectorActor.expectFirstMatching(mockShardActor,
1196                         SwitchBehavior.class);
1197
1198                 assertEquals(RaftState.Leader, switchBehavior.getNewState());
1199                 assertEquals(1000, switchBehavior.getNewTerm());
1200             }
1201         };
1202     }
1203
1204     private static List<MemberName> members(String... names) {
1205         return Arrays.asList(names).stream().map(MemberName::forName).collect(Collectors.toList());
1206     }
1207
1208     @Test
1209     public void testOnCreateShard() {
1210         LOG.info("testOnCreateShard starting");
1211         new JavaTestKit(getSystem()) {
1212             {
1213                 datastoreContextBuilder.shardInitializationTimeout(1, TimeUnit.MINUTES).persistent(true);
1214
1215                 ActorRef shardManager = actorFactory
1216                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1217
1218                 SchemaContext schemaContext = TestModel.createTestContext();
1219                 shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
1220
1221                 DatastoreContext datastoreContext = DatastoreContext.newBuilder().shardElectionTimeoutFactor(100)
1222                         .persistent(false).build();
1223                 Shard.Builder shardBuilder = Shard.builder();
1224
1225                 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1226                         "foo", null, members("member-1", "member-5", "member-6"));
1227                 shardManager.tell(new CreateShard(config, shardBuilder, datastoreContext), getRef());
1228
1229                 expectMsgClass(duration("5 seconds"), Success.class);
1230
1231                 shardManager.tell(new FindLocalShard("foo", true), getRef());
1232
1233                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1234
1235                 assertEquals("isRecoveryApplicable", false, shardBuilder.getDatastoreContext().isPersistent());
1236                 assertTrue("Epxected ShardPeerAddressResolver", shardBuilder.getDatastoreContext().getShardRaftConfig()
1237                         .getPeerAddressResolver() instanceof ShardPeerAddressResolver);
1238                 assertEquals("peerMembers", Sets.newHashSet(
1239                         ShardIdentifier.create("foo", MemberName.forName("member-5"), shardMrgIDSuffix).toString(),
1240                         ShardIdentifier.create("foo", MemberName.forName("member-6"), shardMrgIDSuffix).toString()),
1241                         shardBuilder.getPeerAddresses().keySet());
1242                 assertEquals("ShardIdentifier", ShardIdentifier.create("foo", MEMBER_1, shardMrgIDSuffix),
1243                         shardBuilder.getId());
1244                 assertSame("schemaContext", schemaContext, shardBuilder.getSchemaContext());
1245
1246                 // Send CreateShard with same name - should return Success with
1247                 // a message.
1248
1249                 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1250
1251                 Success success = expectMsgClass(duration("5 seconds"), Success.class);
1252                 assertNotNull("Success status is null", success.status());
1253             }
1254         };
1255
1256         LOG.info("testOnCreateShard ending");
1257     }
1258
1259     @Test
1260     public void testOnCreateShardWithLocalMemberNotInShardConfig() {
1261         LOG.info("testOnCreateShardWithLocalMemberNotInShardConfig starting");
1262         new JavaTestKit(getSystem()) {
1263             {
1264                 datastoreContextBuilder.shardInitializationTimeout(1, TimeUnit.MINUTES).persistent(true);
1265
1266                 ActorRef shardManager = actorFactory
1267                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1268
1269                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1270
1271                 Shard.Builder shardBuilder = Shard.builder();
1272                 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1273                         "foo", null, members("member-5", "member-6"));
1274
1275                 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1276                 expectMsgClass(duration("5 seconds"), Success.class);
1277
1278                 shardManager.tell(new FindLocalShard("foo", true), getRef());
1279                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1280
1281                 assertEquals("peerMembers size", 0, shardBuilder.getPeerAddresses().size());
1282                 assertEquals("schemaContext", DisableElectionsRaftPolicy.class.getName(), shardBuilder
1283                         .getDatastoreContext().getShardRaftConfig().getCustomRaftPolicyImplementationClass());
1284             }
1285         };
1286
1287         LOG.info("testOnCreateShardWithLocalMemberNotInShardConfig ending");
1288     }
1289
1290     @Test
1291     public void testOnCreateShardWithNoInitialSchemaContext() {
1292         LOG.info("testOnCreateShardWithNoInitialSchemaContext starting");
1293         new JavaTestKit(getSystem()) {
1294             {
1295                 ActorRef shardManager = actorFactory
1296                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1297
1298                 Shard.Builder shardBuilder = Shard.builder();
1299
1300                 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1301                         "foo", null, members("member-1"));
1302                 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1303
1304                 expectMsgClass(duration("5 seconds"), Success.class);
1305
1306                 SchemaContext schemaContext = TestModel.createTestContext();
1307                 shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
1308
1309                 shardManager.tell(new FindLocalShard("foo", true), getRef());
1310
1311                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1312
1313                 assertSame("schemaContext", schemaContext, shardBuilder.getSchemaContext());
1314                 assertNotNull("schemaContext is null", shardBuilder.getDatastoreContext());
1315             }
1316         };
1317
1318         LOG.info("testOnCreateShardWithNoInitialSchemaContext ending");
1319     }
1320
1321     @Test
1322     public void testGetSnapshot() throws Exception {
1323         LOG.info("testGetSnapshot starting");
1324         JavaTestKit kit = new JavaTestKit(getSystem());
1325
1326         MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1327                 .put("shard1", Arrays.asList("member-1")).put("shard2", Arrays.asList("member-1"))
1328                 .put("astronauts", Collections.<String>emptyList()).build());
1329
1330         TestActorRef<TestShardManager> shardManager = actorFactory
1331                 .createTestActor(newShardMgrProps(mockConfig).withDispatcher(Dispatchers.DefaultDispatcherId()));
1332
1333         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1334         Failure failure = kit.expectMsgClass(Failure.class);
1335         assertEquals("Failure cause type", IllegalStateException.class, failure.cause().getClass());
1336
1337         shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1338
1339         waitForShardInitialized(shardManager, "shard1", kit);
1340         waitForShardInitialized(shardManager, "shard2", kit);
1341
1342         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1343
1344         DatastoreSnapshot datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1345
1346         assertEquals("getType", shardMrgIDSuffix, datastoreSnapshot.getType());
1347         assertNull("Expected null ShardManagerSnapshot", datastoreSnapshot.getShardManagerSnapshot());
1348
1349         Function<ShardSnapshot, String> shardNameTransformer = s -> s.getName();
1350
1351         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2"), Sets.newHashSet(
1352                 Lists.transform(datastoreSnapshot.getShardSnapshots(), shardNameTransformer)));
1353
1354         // Add a new replica
1355
1356         JavaTestKit mockShardLeaderKit = new JavaTestKit(getSystem());
1357
1358         TestShardManager shardManagerInstance = shardManager.underlyingActor();
1359         shardManagerInstance.setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1360
1361         shardManager.tell(new AddShardReplica("astronauts"), kit.getRef());
1362         mockShardLeaderKit.expectMsgClass(AddServer.class);
1363         mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.OK, ""));
1364         kit.expectMsgClass(Status.Success.class);
1365         waitForShardInitialized(shardManager, "astronauts", kit);
1366
1367         // Send another GetSnapshot and verify
1368
1369         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1370         datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1371
1372         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"), Sets.newHashSet(
1373                 Lists.transform(datastoreSnapshot.getShardSnapshots(), shardNameTransformer)));
1374
1375         byte[] snapshotBytes = datastoreSnapshot.getShardManagerSnapshot();
1376         assertNotNull("Expected ShardManagerSnapshot", snapshotBytes);
1377         ShardManagerSnapshot snapshot = SerializationUtils.deserialize(snapshotBytes);
1378         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"),
1379                 Sets.newHashSet(snapshot.getShardList()));
1380
1381         LOG.info("testGetSnapshot ending");
1382     }
1383
1384     @Test
1385     public void testRestoreFromSnapshot() throws Exception {
1386         LOG.info("testRestoreFromSnapshot starting");
1387
1388         datastoreContextBuilder.shardInitializationTimeout(3, TimeUnit.SECONDS);
1389
1390         JavaTestKit kit = new JavaTestKit(getSystem());
1391
1392         MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1393                 .put("shard1", Collections.<String>emptyList()).put("shard2", Collections.<String>emptyList())
1394                 .put("astronauts", Collections.<String>emptyList()).build());
1395
1396         ShardManagerSnapshot snapshot = new ShardManagerSnapshot(Arrays.asList("shard1", "shard2", "astronauts"));
1397         DatastoreSnapshot restoreFromSnapshot = new DatastoreSnapshot(shardMrgIDSuffix,
1398                 SerializationUtils.serialize(snapshot), Collections.<ShardSnapshot>emptyList());
1399         TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(newTestShardMgrBuilder(mockConfig)
1400                 .restoreFromSnapshot(restoreFromSnapshot).props().withDispatcher(Dispatchers.DefaultDispatcherId()));
1401
1402         shardManager.underlyingActor().waitForRecoveryComplete();
1403
1404         shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1405
1406         waitForShardInitialized(shardManager, "shard1", kit);
1407         waitForShardInitialized(shardManager, "shard2", kit);
1408         waitForShardInitialized(shardManager, "astronauts", kit);
1409
1410         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1411
1412         DatastoreSnapshot datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1413
1414         assertEquals("getType", shardMrgIDSuffix, datastoreSnapshot.getType());
1415
1416         byte[] snapshotBytes = datastoreSnapshot.getShardManagerSnapshot();
1417         assertNotNull("Expected ShardManagerSnapshot", snapshotBytes);
1418         snapshot = SerializationUtils.deserialize(snapshotBytes);
1419         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"),
1420                 Sets.newHashSet(snapshot.getShardList()));
1421
1422         LOG.info("testRestoreFromSnapshot ending");
1423     }
1424
1425     @Test
1426     public void testAddShardReplicaForNonExistentShardConfig() throws Exception {
1427         new JavaTestKit(getSystem()) {
1428             {
1429                 ActorRef shardManager = actorFactory
1430                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1431
1432                 shardManager.tell(new AddShardReplica("model-inventory"), getRef());
1433                 Status.Failure resp = expectMsgClass(duration("2 seconds"), Status.Failure.class);
1434
1435                 assertEquals("Failure obtained", true, resp.cause() instanceof IllegalArgumentException);
1436             }
1437         };
1438     }
1439
1440     @Test
1441     public void testAddShardReplica() throws Exception {
1442         LOG.info("testAddShardReplica starting");
1443         MockConfiguration mockConfig = new MockConfiguration(
1444                 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
1445                         .put("astronauts", Arrays.asList("member-2")).build());
1446
1447         final String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
1448         datastoreContextBuilder.shardManagerPersistenceId(shardManagerID);
1449
1450         // Create an ActorSystem ShardManager actor for member-1.
1451         final ActorSystem system1 = newActorSystem("Member1");
1452         Cluster.get(system1).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
1453         ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
1454         final TestActorRef<TestShardManager> newReplicaShardManager = TestActorRef.create(system1,
1455                 newTestShardMgrBuilder(mockConfig).shardActor(mockDefaultShardActor)
1456                         .cluster(new ClusterWrapperImpl(system1)).props()
1457                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
1458                 shardManagerID);
1459
1460         // Create an ActorSystem ShardManager actor for member-2.
1461         final ActorSystem system2 = newActorSystem("Member2");
1462         Cluster.get(system2).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
1463
1464         String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
1465         String name = ShardIdentifier.create("astronauts", MEMBER_2, "config").toString();
1466         final TestActorRef<MockRespondActor> mockShardLeaderActor = TestActorRef.create(system2,
1467                 Props.create(MockRespondActor.class, AddServer.class,
1468                         new AddServerReply(ServerChangeStatus.OK, memberId2))
1469                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
1470                 name);
1471         final TestActorRef<TestShardManager> leaderShardManager = TestActorRef.create(system2,
1472                 newTestShardMgrBuilder(mockConfig).shardActor(mockShardLeaderActor)
1473                         .cluster(new ClusterWrapperImpl(system2)).props()
1474                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
1475                 shardManagerID);
1476
1477         new JavaTestKit(system1) {
1478             {
1479                 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1480                 leaderShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1481
1482                 leaderShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1483
1484                 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
1485                 leaderShardManager.tell(
1486                         new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
1487                         mockShardLeaderActor);
1488                 leaderShardManager.tell(
1489                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
1490                         mockShardLeaderActor);
1491
1492                 newReplicaShardManager.underlyingActor().waitForMemberUp();
1493                 leaderShardManager.underlyingActor().waitForMemberUp();
1494
1495                 // Have a dummy snapshot to be overwritten by the new data
1496                 // persisted.
1497                 String[] restoredShards = { "default", "people" };
1498                 ShardManagerSnapshot snapshot = new ShardManagerSnapshot(Arrays.asList(restoredShards));
1499                 InMemorySnapshotStore.addSnapshot(shardManagerID, snapshot);
1500                 Uninterruptibles.sleepUninterruptibly(2, TimeUnit.MILLISECONDS);
1501
1502                 InMemorySnapshotStore.addSnapshotSavedLatch(shardManagerID);
1503                 InMemorySnapshotStore.addSnapshotDeletedLatch(shardManagerID);
1504
1505                 // construct a mock response message
1506                 newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
1507                 AddServer addServerMsg = MessageCollectorActor.expectFirstMatching(mockShardLeaderActor,
1508                         AddServer.class);
1509                 String addServerId = "member-1-shard-astronauts-" + shardMrgIDSuffix;
1510                 assertEquals("AddServer serverId", addServerId, addServerMsg.getNewServerId());
1511                 expectMsgClass(duration("5 seconds"), Status.Success.class);
1512
1513                 InMemorySnapshotStore.waitForSavedSnapshot(shardManagerID, ShardManagerSnapshot.class);
1514                 InMemorySnapshotStore.waitForDeletedSnapshot(shardManagerID);
1515                 List<ShardManagerSnapshot> persistedSnapshots = InMemorySnapshotStore.getSnapshots(shardManagerID,
1516                         ShardManagerSnapshot.class);
1517                 assertEquals("Number of snapshots persisted", 1, persistedSnapshots.size());
1518                 ShardManagerSnapshot shardManagerSnapshot = persistedSnapshots.get(0);
1519                 assertEquals("Persisted local shards", Sets.newHashSet("default", "astronauts"),
1520                         Sets.newHashSet(shardManagerSnapshot.getShardList()));
1521             }
1522         };
1523         LOG.info("testAddShardReplica ending");
1524     }
1525
1526     @Test
1527     public void testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader() throws Exception {
1528         LOG.info("testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader starting");
1529         new JavaTestKit(getSystem()) {
1530             {
1531                 TestActorRef<TestShardManager> shardManager = actorFactory
1532                         .createTestActor(newPropsShardMgrWithMockShardActor(), shardMgrID);
1533
1534                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1535                 shardManager.tell(new ActorInitialized(), mockShardActor);
1536
1537                 String leaderId = "leader-member-shard-default-" + shardMrgIDSuffix;
1538                 AddServerReply addServerReply = new AddServerReply(ServerChangeStatus.ALREADY_EXISTS, null);
1539                 ActorRef leaderShardActor = shardManager.underlyingActor().getContext()
1540                         .actorOf(Props.create(MockRespondActor.class, AddServer.class, addServerReply), leaderId);
1541
1542                 MockClusterWrapper.sendMemberUp(shardManager, "leader-member", leaderShardActor.path().toString());
1543
1544                 String newReplicaId = "member-1-shard-default-" + shardMrgIDSuffix;
1545                 shardManager.tell(
1546                         new RoleChangeNotification(newReplicaId, RaftState.Candidate.name(), RaftState.Follower.name()),
1547                         mockShardActor);
1548                 shardManager.tell(
1549                         new ShardLeaderStateChanged(newReplicaId, leaderId, DataStoreVersions.CURRENT_VERSION),
1550                         mockShardActor);
1551
1552                 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1553
1554                 MessageCollectorActor.expectFirstMatching(leaderShardActor, AddServer.class);
1555
1556                 Failure resp = expectMsgClass(duration("5 seconds"), Failure.class);
1557                 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1558
1559                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1560                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1561
1562                 // Send message again to verify previous in progress state is
1563                 // cleared
1564
1565                 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1566                 resp = expectMsgClass(duration("5 seconds"), Failure.class);
1567                 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1568
1569                 // Send message again with an AddServer timeout to verify the
1570                 // pre-existing shard actor isn't terminated.
1571
1572                 shardManager.tell(
1573                         newDatastoreContextFactory(
1574                                 datastoreContextBuilder.shardLeaderElectionTimeout(100, TimeUnit.MILLISECONDS).build()),
1575                         getRef());
1576                 leaderShardActor.tell(MockRespondActor.CLEAR_RESPONSE, ActorRef.noSender());
1577                 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1578                 expectMsgClass(duration("5 seconds"), Failure.class);
1579
1580                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1581                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1582             }
1583         };
1584
1585         LOG.info("testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader ending");
1586     }
1587
1588     @Test
1589     public void testAddShardReplicaWithPreExistingLocalReplicaLeader() throws Exception {
1590         LOG.info("testAddShardReplicaWithPreExistingLocalReplicaLeader starting");
1591         new JavaTestKit(getSystem()) {
1592             {
1593                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1594                 ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1595
1596                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1597                 shardManager.tell(new ActorInitialized(), mockShardActor);
1598                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
1599                         DataStoreVersions.CURRENT_VERSION), getRef());
1600                 shardManager.tell(
1601                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
1602                         mockShardActor);
1603
1604                 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1605                 Failure resp = expectMsgClass(duration("5 seconds"), Failure.class);
1606                 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1607
1608                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1609                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1610             }
1611         };
1612
1613         LOG.info("testAddShardReplicaWithPreExistingLocalReplicaLeader ending");
1614     }
1615
1616     @Test
1617     public void testAddShardReplicaWithAddServerReplyFailure() throws Exception {
1618         LOG.info("testAddShardReplicaWithAddServerReplyFailure starting");
1619         new JavaTestKit(getSystem()) {
1620             {
1621                 JavaTestKit mockShardLeaderKit = new JavaTestKit(getSystem());
1622
1623                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1624                         .put("astronauts", Arrays.asList("member-2")).build());
1625
1626                 ActorRef mockNewReplicaShardActor = newMockShardActor(getSystem(), "astronauts", "member-1");
1627                 final TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(
1628                         newTestShardMgrBuilder(mockConfig).shardActor(mockNewReplicaShardActor).props(), shardMgrID);
1629                 shardManager.underlyingActor()
1630                         .setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1631
1632                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1633
1634                 JavaTestKit terminateWatcher = new JavaTestKit(getSystem());
1635                 terminateWatcher.watch(mockNewReplicaShardActor);
1636
1637                 shardManager.tell(new AddShardReplica("astronauts"), getRef());
1638
1639                 AddServer addServerMsg = mockShardLeaderKit.expectMsgClass(AddServer.class);
1640                 assertEquals("AddServer serverId", "member-1-shard-astronauts-" + shardMrgIDSuffix,
1641                         addServerMsg.getNewServerId());
1642                 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.TIMEOUT, null));
1643
1644                 Failure failure = expectMsgClass(duration("5 seconds"), Failure.class);
1645                 assertEquals("Failure cause", TimeoutException.class, failure.cause().getClass());
1646
1647                 shardManager.tell(new FindLocalShard("astronauts", false), getRef());
1648                 expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
1649
1650                 terminateWatcher.expectTerminated(mockNewReplicaShardActor);
1651
1652                 shardManager.tell(new AddShardReplica("astronauts"), getRef());
1653                 mockShardLeaderKit.expectMsgClass(AddServer.class);
1654                 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.NO_LEADER, null));
1655                 failure = expectMsgClass(duration("5 seconds"), Failure.class);
1656                 assertEquals("Failure cause", NoShardLeaderException.class, failure.cause().getClass());
1657             }
1658         };
1659
1660         LOG.info("testAddShardReplicaWithAddServerReplyFailure ending");
1661     }
1662
1663     @Test
1664     public void testAddShardReplicaWithAlreadyInProgress() throws Exception {
1665         testServerChangeWhenAlreadyInProgress("astronauts", new AddShardReplica("astronauts"),
1666                 AddServer.class, new AddShardReplica("astronauts"));
1667     }
1668
1669     @Test
1670     public void testAddShardReplicaWithFindPrimaryTimeout() throws Exception {
1671         LOG.info("testAddShardReplicaWithFindPrimaryTimeout starting");
1672         datastoreContextBuilder.shardInitializationTimeout(100, TimeUnit.MILLISECONDS);
1673         new JavaTestKit(getSystem()) {
1674             {
1675                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1676                         .put("astronauts", Arrays.asList("member-2")).build());
1677
1678                 final ActorRef newReplicaShardManager = actorFactory
1679                         .createActor(newTestShardMgrBuilder(mockConfig).shardActor(mockShardActor).props(), shardMgrID);
1680
1681                 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1682                 MockClusterWrapper.sendMemberUp(newReplicaShardManager, "member-2",
1683                         AddressFromURIString.parse("akka.tcp://non-existent@127.0.0.1:5").toString());
1684
1685                 newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
1686                 Status.Failure resp = expectMsgClass(duration("5 seconds"), Status.Failure.class);
1687                 assertEquals("Failure obtained", true, resp.cause() instanceof RuntimeException);
1688             }
1689         };
1690
1691         LOG.info("testAddShardReplicaWithFindPrimaryTimeout ending");
1692     }
1693
1694     @Test
1695     public void testRemoveShardReplicaForNonExistentShard() throws Exception {
1696         new JavaTestKit(getSystem()) {
1697             {
1698                 ActorRef shardManager = actorFactory
1699                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1700
1701                 shardManager.tell(new RemoveShardReplica("model-inventory", MEMBER_1), getRef());
1702                 Status.Failure resp = expectMsgClass(duration("10 seconds"), Status.Failure.class);
1703                 assertEquals("Failure obtained", true, resp.cause() instanceof PrimaryNotFoundException);
1704             }
1705         };
1706     }
1707
1708     @Test
1709     /**
1710      * Primary is Local
1711      */
1712     public void testRemoveShardReplicaLocal() throws Exception {
1713         new JavaTestKit(getSystem()) {
1714             {
1715                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1716
1717                 final TestActorRef<MockRespondActor> respondActor = actorFactory
1718                         .createTestActor(Props.create(MockRespondActor.class, RemoveServer.class,
1719                                 new RemoveServerReply(ServerChangeStatus.OK, null)), memberId);
1720
1721                 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
1722
1723                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1724                 shardManager.tell(new ActorInitialized(), respondActor);
1725                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
1726                         DataStoreVersions.CURRENT_VERSION), getRef());
1727                 shardManager.tell(
1728                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
1729                         respondActor);
1730
1731                 shardManager.tell(new RemoveShardReplica(Shard.DEFAULT_NAME, MEMBER_1), getRef());
1732                 final RemoveServer removeServer = MessageCollectorActor.expectFirstMatching(respondActor,
1733                         RemoveServer.class);
1734                 assertEquals(ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString(),
1735                         removeServer.getServerId());
1736                 expectMsgClass(duration("5 seconds"), Success.class);
1737             }
1738         };
1739     }
1740
1741     @Test
1742     public void testRemoveShardReplicaRemote() throws Exception {
1743         MockConfiguration mockConfig = new MockConfiguration(
1744                 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
1745                         .put("astronauts", Arrays.asList("member-1")).build());
1746
1747         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
1748
1749         // Create an ActorSystem ShardManager actor for member-1.
1750         final ActorSystem system1 = newActorSystem("Member1");
1751         Cluster.get(system1).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
1752         ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
1753
1754         final TestActorRef<TestShardManager> newReplicaShardManager = TestActorRef.create(system1,
1755                 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockDefaultShardActor).cluster(
1756                         new ClusterWrapperImpl(system1)).props().withDispatcher(Dispatchers.DefaultDispatcherId()),
1757                 shardManagerID);
1758
1759         // Create an ActorSystem ShardManager actor for member-2.
1760         final ActorSystem system2 = newActorSystem("Member2");
1761         Cluster.get(system2).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
1762
1763         String name = ShardIdentifier.create("default", MEMBER_2, shardMrgIDSuffix).toString();
1764         String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
1765         final TestActorRef<MockRespondActor> mockShardLeaderActor =
1766                 TestActorRef.create(system2, Props.create(MockRespondActor.class, RemoveServer.class,
1767                         new RemoveServerReply(ServerChangeStatus.OK, memberId2)), name);
1768
1769         LOG.error("Mock Shard Leader Actor : {}", mockShardLeaderActor);
1770
1771         final TestActorRef<TestShardManager> leaderShardManager = TestActorRef.create(system2,
1772                 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockShardLeaderActor).cluster(
1773                         new ClusterWrapperImpl(system2)).props().withDispatcher(Dispatchers.DefaultDispatcherId()),
1774                 shardManagerID);
1775
1776         // Because mockShardLeaderActor is created at the top level of the actor system it has an address like so,
1777         //    akka.tcp://cluster-test@127.0.0.1:2559/user/member-2-shard-default-config1
1778         // However when a shard manager has a local shard which is a follower and a leader that is remote it will
1779         // try to compute an address for the remote shard leader using the ShardPeerAddressResolver. This address will
1780         // look like so,
1781         //    akka.tcp://cluster-test@127.0.0.1:2559/user/shardmanager-config1/member-2-shard-default-config1
1782         // In this specific case if we did a FindPrimary for shard default from member-1 we would come up
1783         // with the address of an actor which does not exist, therefore any message sent to that actor would go to
1784         // dead letters.
1785         // To work around this problem we create a ForwardingActor with the right address and pass to it the
1786         // mockShardLeaderActor. The ForwardingActor simply forwards all messages to the mockShardLeaderActor and every
1787         // thing works as expected
1788         final ActorRef actorRef = leaderShardManager.underlyingActor().context()
1789                 .actorOf(Props.create(ForwardingActor.class, mockShardLeaderActor),
1790                         "member-2-shard-default-" + shardMrgIDSuffix);
1791
1792         LOG.error("Forwarding actor : {}", actorRef);
1793
1794         new JavaTestKit(system1) {
1795             {
1796                 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1797                 leaderShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1798
1799                 leaderShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1800                 newReplicaShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1801
1802                 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
1803                 leaderShardManager.tell(
1804                         new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
1805                         mockShardLeaderActor);
1806                 leaderShardManager.tell(
1807                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
1808                         mockShardLeaderActor);
1809
1810                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
1811                 newReplicaShardManager.tell(
1812                         new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class), leaderVersion),
1813                         mockShardActor);
1814                 newReplicaShardManager.tell(
1815                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
1816                         mockShardActor);
1817
1818                 newReplicaShardManager.underlyingActor().waitForMemberUp();
1819                 leaderShardManager.underlyingActor().waitForMemberUp();
1820
1821                 // construct a mock response message
1822                 newReplicaShardManager.tell(new RemoveShardReplica("default", MEMBER_1), getRef());
1823                 RemoveServer removeServer = MessageCollectorActor.expectFirstMatching(mockShardLeaderActor,
1824                         RemoveServer.class);
1825                 String removeServerId = ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString();
1826                 assertEquals("RemoveServer serverId", removeServerId, removeServer.getServerId());
1827                 expectMsgClass(duration("5 seconds"), Status.Success.class);
1828             }
1829         };
1830     }
1831
1832     @Test
1833     public void testRemoveShardReplicaWhenAnotherRemoveShardReplicaAlreadyInProgress() throws Exception {
1834         testServerChangeWhenAlreadyInProgress("astronauts", new RemoveShardReplica("astronauts", MEMBER_2),
1835                 RemoveServer.class, new RemoveShardReplica("astronauts", MEMBER_3));
1836     }
1837
1838     @Test
1839     public void testRemoveShardReplicaWhenAddShardReplicaAlreadyInProgress() throws Exception {
1840         testServerChangeWhenAlreadyInProgress("astronauts", new AddShardReplica("astronauts"),
1841                 AddServer.class, new RemoveShardReplica("astronauts", MEMBER_2));
1842     }
1843
1844
1845     public void testServerChangeWhenAlreadyInProgress(final String shardName, final Object firstServerChange,
1846                                                       final Class<?> firstForwardedServerChangeClass,
1847                                                       final Object secondServerChange) throws Exception {
1848         new JavaTestKit(getSystem()) {
1849             {
1850                 JavaTestKit mockShardLeaderKit = new JavaTestKit(getSystem());
1851                 final JavaTestKit secondRequestKit = new JavaTestKit(getSystem());
1852
1853                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1854                         .put(shardName, Arrays.asList("member-2")).build());
1855
1856                 final TestActorRef<TestShardManager> shardManager = TestActorRef.create(getSystem(),
1857                         newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockShardActor)
1858                                 .cluster(new MockClusterWrapper()).props()
1859                                 .withDispatcher(Dispatchers.DefaultDispatcherId()),
1860                         shardMgrID);
1861
1862                 shardManager.underlyingActor()
1863                         .setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1864
1865                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1866
1867                 shardManager.tell(firstServerChange, getRef());
1868
1869                 mockShardLeaderKit.expectMsgClass(firstForwardedServerChangeClass);
1870
1871                 shardManager.tell(secondServerChange, secondRequestKit.getRef());
1872
1873                 secondRequestKit.expectMsgClass(duration("5 seconds"), Failure.class);
1874             }
1875         };
1876     }
1877
1878     @Test
1879     public void testServerRemovedShardActorNotRunning() throws Exception {
1880         LOG.info("testServerRemovedShardActorNotRunning starting");
1881         new JavaTestKit(getSystem()) {
1882             {
1883                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1884                         .put("default", Arrays.asList("member-1", "member-2"))
1885                         .put("astronauts", Arrays.asList("member-2"))
1886                         .put("people", Arrays.asList("member-1", "member-2")).build());
1887
1888                 TestActorRef<TestShardManager> shardManager = actorFactory
1889                         .createTestActor(newShardMgrProps(mockConfig));
1890
1891                 shardManager.underlyingActor().waitForRecoveryComplete();
1892                 shardManager.tell(new FindLocalShard("people", false), getRef());
1893                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1894
1895                 shardManager.tell(new FindLocalShard("default", false), getRef());
1896                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1897
1898                 // Removed the default shard replica from member-1
1899                 ShardIdentifier.Builder builder = new ShardIdentifier.Builder();
1900                 ShardIdentifier shardId = builder.shardName("default").memberName(MEMBER_1).type(shardMrgIDSuffix)
1901                         .build();
1902                 shardManager.tell(new ServerRemoved(shardId.toString()), getRef());
1903
1904                 shardManager.underlyingActor().verifySnapshotPersisted(Sets.newHashSet("people"));
1905             }
1906         };
1907
1908         LOG.info("testServerRemovedShardActorNotRunning ending");
1909     }
1910
1911     @Test
1912     public void testServerRemovedShardActorRunning() throws Exception {
1913         LOG.info("testServerRemovedShardActorRunning starting");
1914         new JavaTestKit(getSystem()) {
1915             {
1916                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1917                         .put("default", Arrays.asList("member-1", "member-2"))
1918                         .put("astronauts", Arrays.asList("member-2"))
1919                         .put("people", Arrays.asList("member-1", "member-2")).build());
1920
1921                 String shardId = ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString();
1922                 TestActorRef<MessageCollectorActor> shard = actorFactory.createTestActor(MessageCollectorActor.props(),
1923                         shardId);
1924
1925                 TestActorRef<TestShardManager> shardManager = actorFactory
1926                         .createTestActor(newTestShardMgrBuilder(mockConfig).addShardActor("default", shard).props());
1927
1928                 shardManager.underlyingActor().waitForRecoveryComplete();
1929
1930                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1931                 shardManager.tell(new ActorInitialized(), shard);
1932
1933                 waitForShardInitialized(shardManager, "people", this);
1934                 waitForShardInitialized(shardManager, "default", this);
1935
1936                 // Removed the default shard replica from member-1
1937                 shardManager.tell(new ServerRemoved(shardId), getRef());
1938
1939                 shardManager.underlyingActor().verifySnapshotPersisted(Sets.newHashSet("people"));
1940
1941                 MessageCollectorActor.expectFirstMatching(shard, Shutdown.class);
1942             }
1943         };
1944
1945         LOG.info("testServerRemovedShardActorRunning ending");
1946     }
1947
1948     @Test
1949     public void testShardPersistenceWithRestoredData() throws Exception {
1950         LOG.info("testShardPersistenceWithRestoredData starting");
1951         new JavaTestKit(getSystem()) {
1952             {
1953                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1954                         .put("default", Arrays.asList("member-1", "member-2"))
1955                         .put("astronauts", Arrays.asList("member-2"))
1956                         .put("people", Arrays.asList("member-1", "member-2")).build());
1957                 String[] restoredShards = { "default", "astronauts" };
1958                 ShardManagerSnapshot snapshot = new ShardManagerSnapshot(Arrays.asList(restoredShards));
1959                 InMemorySnapshotStore.addSnapshot("shard-manager-" + shardMrgIDSuffix, snapshot);
1960
1961                 // create shardManager to come up with restored data
1962                 TestActorRef<TestShardManager> newRestoredShardManager = actorFactory
1963                         .createTestActor(newShardMgrProps(mockConfig));
1964
1965                 newRestoredShardManager.underlyingActor().waitForRecoveryComplete();
1966
1967                 newRestoredShardManager.tell(new FindLocalShard("people", false), getRef());
1968                 LocalShardNotFound notFound = expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
1969                 assertEquals("for uninitialized shard", "people", notFound.getShardName());
1970
1971                 // Verify a local shard is created for the restored shards,
1972                 // although we expect a NotInitializedException for the shards
1973                 // as the actor initialization
1974                 // message is not sent for them
1975                 newRestoredShardManager.tell(new FindLocalShard("default", false), getRef());
1976                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1977
1978                 newRestoredShardManager.tell(new FindLocalShard("astronauts", false), getRef());
1979                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1980             }
1981         };
1982
1983         LOG.info("testShardPersistenceWithRestoredData ending");
1984     }
1985
1986     @Test
1987     public void testShutDown() throws Exception {
1988         LOG.info("testShutDown starting");
1989         new JavaTestKit(getSystem()) {
1990             {
1991                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1992                         .put("shard1", Arrays.asList("member-1")).put("shard2", Arrays.asList("member-1")).build());
1993
1994                 String shardId1 = ShardIdentifier.create("shard1", MEMBER_1, shardMrgIDSuffix).toString();
1995                 TestActorRef<MessageCollectorActor> shard1 = actorFactory.createTestActor(
1996                         MessageCollectorActor.props().withDispatcher(Dispatchers.DefaultDispatcherId()), shardId1);
1997
1998                 String shardId2 = ShardIdentifier.create("shard2", MEMBER_1, shardMrgIDSuffix).toString();
1999                 TestActorRef<MessageCollectorActor> shard2 = actorFactory.createTestActor(
2000                         MessageCollectorActor.props().withDispatcher(Dispatchers.DefaultDispatcherId()), shardId2);
2001
2002                 TestActorRef<TestShardManager> shardManager = actorFactory
2003                         .createTestActor(newTestShardMgrBuilder(mockConfig).addShardActor("shard1", shard1)
2004                                 .addShardActor("shard2", shard2).props()
2005                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
2006
2007                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
2008                 shardManager.tell(new ActorInitialized(), shard1);
2009                 shardManager.tell(new ActorInitialized(), shard2);
2010
2011                 FiniteDuration duration = FiniteDuration.create(5, TimeUnit.SECONDS);
2012                 Future<Boolean> stopFuture = Patterns.gracefulStop(shardManager, duration, Shutdown.INSTANCE);
2013
2014                 MessageCollectorActor.expectFirstMatching(shard1, Shutdown.class);
2015                 MessageCollectorActor.expectFirstMatching(shard2, Shutdown.class);
2016
2017                 try {
2018                     Await.ready(stopFuture, FiniteDuration.create(500, TimeUnit.MILLISECONDS));
2019                     fail("ShardManager actor stopped without waiting for the Shards to be stopped");
2020                 } catch (TimeoutException e) {
2021                     // expected
2022                 }
2023
2024                 actorFactory.killActor(shard1, this);
2025                 actorFactory.killActor(shard2, this);
2026
2027                 Boolean stopped = Await.result(stopFuture, duration);
2028                 assertEquals("Stopped", Boolean.TRUE, stopped);
2029             }
2030         };
2031
2032         LOG.info("testShutDown ending");
2033     }
2034
2035     @Test
2036     public void testChangeServersVotingStatus() throws Exception {
2037         new JavaTestKit(getSystem()) {
2038             {
2039                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
2040
2041                 TestActorRef<MockRespondActor> respondActor = actorFactory
2042                         .createTestActor(Props.create(MockRespondActor.class, ChangeServersVotingStatus.class,
2043                                 new ServerChangeReply(ServerChangeStatus.OK, null)), memberId);
2044
2045                 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
2046
2047                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
2048                 shardManager.tell(new ActorInitialized(), respondActor);
2049                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
2050                         DataStoreVersions.CURRENT_VERSION), getRef());
2051                 shardManager.tell(
2052                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
2053                         respondActor);
2054
2055                 shardManager.tell(
2056                         new ChangeShardMembersVotingStatus("default", ImmutableMap.of("member-2", Boolean.TRUE)),
2057                         getRef());
2058
2059                 ChangeServersVotingStatus actualChangeStatusMsg = MessageCollectorActor
2060                         .expectFirstMatching(respondActor, ChangeServersVotingStatus.class);
2061                 assertEquals("ChangeServersVotingStatus map", actualChangeStatusMsg.getServerVotingStatusMap(),
2062                         ImmutableMap.of(ShardIdentifier
2063                                 .create("default", MemberName.forName("member-2"), shardMrgIDSuffix).toString(),
2064                                 Boolean.TRUE));
2065
2066                 expectMsgClass(duration("5 seconds"), Success.class);
2067             }
2068         };
2069     }
2070
2071     @Test
2072     public void testChangeServersVotingStatusWithNoLeader() throws Exception {
2073         new JavaTestKit(getSystem()) {
2074             {
2075                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
2076
2077                 TestActorRef<MockRespondActor> respondActor = actorFactory
2078                         .createTestActor(Props.create(MockRespondActor.class, ChangeServersVotingStatus.class,
2079                                 new ServerChangeReply(ServerChangeStatus.NO_LEADER, null)), memberId);
2080
2081                 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
2082
2083                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
2084                 shardManager.tell(new ActorInitialized(), respondActor);
2085                 shardManager.tell(new RoleChangeNotification(memberId, null, RaftState.Follower.name()), respondActor);
2086
2087                 shardManager.tell(
2088                         new ChangeShardMembersVotingStatus("default", ImmutableMap.of("member-2", Boolean.TRUE)),
2089                         getRef());
2090
2091                 MessageCollectorActor.expectFirstMatching(respondActor, ChangeServersVotingStatus.class);
2092
2093                 Status.Failure resp = expectMsgClass(duration("5 seconds"), Status.Failure.class);
2094                 assertEquals("Failure resposnse", true, resp.cause() instanceof NoShardLeaderException);
2095             }
2096         };
2097     }
2098
2099     public static class TestShardManager extends ShardManager {
2100         private final CountDownLatch recoveryComplete = new CountDownLatch(1);
2101         private final CountDownLatch snapshotPersist = new CountDownLatch(1);
2102         private ShardManagerSnapshot snapshot;
2103         private final Map<String, ActorRef> shardActors;
2104         private final ActorRef shardActor;
2105         private CountDownLatch findPrimaryMessageReceived = new CountDownLatch(1);
2106         private CountDownLatch memberUpReceived = new CountDownLatch(1);
2107         private CountDownLatch memberRemovedReceived = new CountDownLatch(1);
2108         private CountDownLatch memberUnreachableReceived = new CountDownLatch(1);
2109         private CountDownLatch memberReachableReceived = new CountDownLatch(1);
2110         private volatile MessageInterceptor messageInterceptor;
2111
2112         private TestShardManager(Builder builder) {
2113             super(builder);
2114             shardActor = builder.shardActor;
2115             shardActors = builder.shardActors;
2116         }
2117
2118         @Override
2119         protected void handleRecover(Object message) throws Exception {
2120             try {
2121                 super.handleRecover(message);
2122             } finally {
2123                 if (message instanceof RecoveryCompleted) {
2124                     recoveryComplete.countDown();
2125                 }
2126             }
2127         }
2128
2129         private void countDownIfOther(final Member member, CountDownLatch latch) {
2130             if (!getCluster().getCurrentMemberName().equals(memberToName(member))) {
2131                 latch.countDown();
2132             }
2133         }
2134
2135         @Override
2136         public void handleCommand(Object message) throws Exception {
2137             try {
2138                 if (messageInterceptor != null && messageInterceptor.canIntercept(message)) {
2139                     getSender().tell(messageInterceptor.apply(message), getSelf());
2140                 } else {
2141                     super.handleCommand(message);
2142                 }
2143             } finally {
2144                 if (message instanceof FindPrimary) {
2145                     findPrimaryMessageReceived.countDown();
2146                 } else if (message instanceof ClusterEvent.MemberUp) {
2147                     countDownIfOther(((ClusterEvent.MemberUp) message).member(), memberUpReceived);
2148                 } else if (message instanceof ClusterEvent.MemberRemoved) {
2149                     countDownIfOther(((ClusterEvent.MemberRemoved) message).member(), memberRemovedReceived);
2150                 } else if (message instanceof ClusterEvent.UnreachableMember) {
2151                     countDownIfOther(((ClusterEvent.UnreachableMember) message).member(), memberUnreachableReceived);
2152                 } else if (message instanceof ClusterEvent.ReachableMember) {
2153                     countDownIfOther(((ClusterEvent.ReachableMember) message).member(), memberReachableReceived);
2154                 }
2155             }
2156         }
2157
2158         void setMessageInterceptor(MessageInterceptor messageInterceptor) {
2159             this.messageInterceptor = messageInterceptor;
2160         }
2161
2162         void waitForRecoveryComplete() {
2163             assertEquals("Recovery complete", true,
2164                     Uninterruptibles.awaitUninterruptibly(recoveryComplete, 5, TimeUnit.SECONDS));
2165         }
2166
2167         public void waitForMemberUp() {
2168             assertEquals("MemberUp received", true,
2169                     Uninterruptibles.awaitUninterruptibly(memberUpReceived, 5, TimeUnit.SECONDS));
2170             memberUpReceived = new CountDownLatch(1);
2171         }
2172
2173         void waitForMemberRemoved() {
2174             assertEquals("MemberRemoved received", true,
2175                     Uninterruptibles.awaitUninterruptibly(memberRemovedReceived, 5, TimeUnit.SECONDS));
2176             memberRemovedReceived = new CountDownLatch(1);
2177         }
2178
2179         void waitForUnreachableMember() {
2180             assertEquals("UnreachableMember received", true,
2181                 Uninterruptibles.awaitUninterruptibly(memberUnreachableReceived, 5, TimeUnit.SECONDS
2182                 ));
2183             memberUnreachableReceived = new CountDownLatch(1);
2184         }
2185
2186         void waitForReachableMember() {
2187             assertEquals("ReachableMember received", true,
2188                 Uninterruptibles.awaitUninterruptibly(memberReachableReceived, 5, TimeUnit.SECONDS));
2189             memberReachableReceived = new CountDownLatch(1);
2190         }
2191
2192         void verifyFindPrimary() {
2193             assertEquals("FindPrimary received", true,
2194                     Uninterruptibles.awaitUninterruptibly(findPrimaryMessageReceived, 5, TimeUnit.SECONDS));
2195             findPrimaryMessageReceived = new CountDownLatch(1);
2196         }
2197
2198         public static Builder builder(DatastoreContext.Builder datastoreContextBuilder) {
2199             return new Builder(datastoreContextBuilder);
2200         }
2201
2202         public static class Builder extends AbstractGenericCreator<Builder, TestShardManager> {
2203             private ActorRef shardActor;
2204             private final Map<String, ActorRef> shardActors = new HashMap<>();
2205
2206             Builder(DatastoreContext.Builder datastoreContextBuilder) {
2207                 super(TestShardManager.class);
2208                 datastoreContextFactory(newDatastoreContextFactory(datastoreContextBuilder.build()));
2209             }
2210
2211             Builder shardActor(ActorRef newShardActor) {
2212                 this.shardActor = newShardActor;
2213                 return this;
2214             }
2215
2216             Builder addShardActor(String shardName, ActorRef actorRef) {
2217                 shardActors.put(shardName, actorRef);
2218                 return this;
2219             }
2220         }
2221
2222         @Override
2223         public void saveSnapshot(Object obj) {
2224             snapshot = (ShardManagerSnapshot) obj;
2225             snapshotPersist.countDown();
2226             super.saveSnapshot(obj);
2227         }
2228
2229         void verifySnapshotPersisted(Set<String> shardList) {
2230             assertEquals("saveSnapshot invoked", true,
2231                     Uninterruptibles.awaitUninterruptibly(snapshotPersist, 5, TimeUnit.SECONDS));
2232             assertEquals("Shard Persisted", shardList, Sets.newHashSet(snapshot.getShardList()));
2233         }
2234
2235         @Override
2236         protected ActorRef newShardActor(SchemaContext schemaContext, ShardInformation info) {
2237             if (shardActors.get(info.getShardName()) != null) {
2238                 return shardActors.get(info.getShardName());
2239             }
2240
2241             if (shardActor != null) {
2242                 return shardActor;
2243             }
2244
2245             return super.newShardActor(schemaContext, info);
2246         }
2247     }
2248
2249     private abstract static class AbstractGenericCreator<T extends AbstractGenericCreator<T, ?>, C extends ShardManager>
2250                                                      extends AbstractShardManagerCreator<T> {
2251         private final Class<C> shardManagerClass;
2252
2253         AbstractGenericCreator(Class<C> shardManagerClass) {
2254             this.shardManagerClass = shardManagerClass;
2255             cluster(new MockClusterWrapper()).configuration(new MockConfiguration()).waitTillReadyCountDownLatch(ready)
2256                     .primaryShardInfoCache(new PrimaryShardInfoFutureCache());
2257         }
2258
2259         @Override
2260         public Props props() {
2261             verify();
2262             return Props.create(shardManagerClass, this);
2263         }
2264     }
2265
2266     private static class GenericCreator<C extends ShardManager> extends AbstractGenericCreator<GenericCreator<C>, C> {
2267         GenericCreator(Class<C> shardManagerClass) {
2268             super(shardManagerClass);
2269         }
2270     }
2271
2272     private static class DelegatingShardManagerCreator implements Creator<ShardManager> {
2273         private static final long serialVersionUID = 1L;
2274         private final Creator<ShardManager> delegate;
2275
2276         DelegatingShardManagerCreator(Creator<ShardManager> delegate) {
2277             this.delegate = delegate;
2278         }
2279
2280         @Override
2281         public ShardManager create() throws Exception {
2282             return delegate.create();
2283         }
2284     }
2285
2286     interface MessageInterceptor extends Function<Object, Object> {
2287         boolean canIntercept(Object message);
2288     }
2289
2290     private static MessageInterceptor newFindPrimaryInterceptor(final ActorRef primaryActor) {
2291         return new MessageInterceptor() {
2292             @Override
2293             public Object apply(Object message) {
2294                 return new RemotePrimaryShardFound(Serialization.serializedActorPath(primaryActor), (short) 1);
2295             }
2296
2297             @Override
2298             public boolean canIntercept(Object message) {
2299                 return message instanceof FindPrimary;
2300             }
2301         };
2302     }
2303
2304     private static class MockRespondActor extends MessageCollectorActor {
2305         static final String CLEAR_RESPONSE = "clear-response";
2306
2307         private Object responseMsg;
2308         private final Class<?> requestClass;
2309
2310         @SuppressWarnings("unused")
2311         MockRespondActor(Class<?> requestClass, Object responseMsg) {
2312             this.requestClass = requestClass;
2313             this.responseMsg = responseMsg;
2314         }
2315
2316         @Override
2317         public void onReceive(Object message) throws Exception {
2318             if (message.equals(CLEAR_RESPONSE)) {
2319                 responseMsg = null;
2320             } else {
2321                 super.onReceive(message);
2322                 if (message.getClass().equals(requestClass) && responseMsg != null) {
2323                     getSender().tell(responseMsg, getSelf());
2324                 }
2325             }
2326         }
2327     }
2328 }