opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/shardmanager/ShardManagerTest.java

   1 /*
   2  * Copyright (c) 2014, 2015 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.controller.cluster.datastore.shardmanager;
  10
  11 import static org.junit.Assert.assertEquals;
  12 import static org.junit.Assert.assertNotNull;
  13 import static org.junit.Assert.assertNull;
  14 import static org.junit.Assert.assertSame;
  15 import static org.junit.Assert.assertTrue;
  16 import static org.junit.Assert.fail;
  17 import static org.mockito.Mockito.mock;
  18 import static org.mockito.Mockito.never;
  19 import static org.mockito.Mockito.times;
  20 import static org.mockito.Mockito.verify;
  21
  22 import akka.actor.ActorRef;
  23 import akka.actor.ActorSystem;
  24 import akka.actor.AddressFromURIString;
  25 import akka.actor.Props;
  26 import akka.actor.Status;
  27 import akka.actor.Status.Failure;
  28 import akka.actor.Status.Success;
  29 import akka.cluster.Cluster;
  30 import akka.cluster.ClusterEvent;
  31 import akka.cluster.Member;
  32 import akka.dispatch.Dispatchers;
  33 import akka.japi.Creator;
  34 import akka.pattern.Patterns;
  35 import akka.persistence.RecoveryCompleted;
  36 import akka.serialization.Serialization;
  37 import akka.testkit.JavaTestKit;
  38 import akka.testkit.TestActorRef;
  39 import akka.util.Timeout;
  40 import com.google.common.base.Function;
  41 import com.google.common.base.Stopwatch;
  42 import com.google.common.collect.ImmutableMap;
  43 import com.google.common.collect.ImmutableSet;
  44 import com.google.common.collect.Lists;
  45 import com.google.common.collect.Sets;
  46 import com.google.common.util.concurrent.Uninterruptibles;
  47 import java.net.URI;
  48 import java.util.AbstractMap;
  49 import java.util.Arrays;
  50 import java.util.Collection;
  51 import java.util.Collections;
  52 import java.util.HashMap;
  53 import java.util.List;
  54 import java.util.Map;
  55 import java.util.Map.Entry;
  56 import java.util.Set;
  57 import java.util.concurrent.CountDownLatch;
  58 import java.util.concurrent.TimeUnit;
  59 import java.util.concurrent.TimeoutException;
  60 import java.util.stream.Collectors;
  61 import org.junit.Test;
  62 import org.mockito.Mockito;
  63 import org.opendaylight.controller.cluster.access.concepts.MemberName;
  64 import org.opendaylight.controller.cluster.datastore.AbstractShardManagerTest;
  65 import org.opendaylight.controller.cluster.datastore.ClusterWrapperImpl;
  66 import org.opendaylight.controller.cluster.datastore.DataStoreVersions;
  67 import org.opendaylight.controller.cluster.datastore.DatastoreContext;
  68 import org.opendaylight.controller.cluster.datastore.DatastoreContextFactory;
  69 import org.opendaylight.controller.cluster.datastore.DistributedDataStore;
  70 import org.opendaylight.controller.cluster.datastore.Shard;
  71 import org.opendaylight.controller.cluster.datastore.ShardManager.SchemaContextModules;
  72 import org.opendaylight.controller.cluster.datastore.config.ConfigurationImpl;
  73 import org.opendaylight.controller.cluster.datastore.config.EmptyModuleShardConfigProvider;
  74 import org.opendaylight.controller.cluster.datastore.config.ModuleShardConfiguration;
  75 import org.opendaylight.controller.cluster.datastore.exceptions.AlreadyExistsException;
  76 import org.opendaylight.controller.cluster.datastore.exceptions.NoShardLeaderException;
  77 import org.opendaylight.controller.cluster.datastore.exceptions.NotInitializedException;
  78 import org.opendaylight.controller.cluster.datastore.exceptions.PrimaryNotFoundException;
  79 import org.opendaylight.controller.cluster.datastore.identifiers.ShardIdentifier;
  80 import org.opendaylight.controller.cluster.datastore.identifiers.ShardManagerIdentifier;
  81 import org.opendaylight.controller.cluster.datastore.messages.ActorInitialized;
  82 import org.opendaylight.controller.cluster.datastore.messages.AddShardReplica;
  83 import org.opendaylight.controller.cluster.datastore.messages.ChangeShardMembersVotingStatus;
  84 import org.opendaylight.controller.cluster.datastore.messages.CreateShard;
  85 import org.opendaylight.controller.cluster.datastore.messages.FindLocalShard;
  86 import org.opendaylight.controller.cluster.datastore.messages.FindPrimary;
  87 import org.opendaylight.controller.cluster.datastore.messages.LocalPrimaryShardFound;
  88 import org.opendaylight.controller.cluster.datastore.messages.LocalShardFound;
  89 import org.opendaylight.controller.cluster.datastore.messages.LocalShardNotFound;
  90 import org.opendaylight.controller.cluster.datastore.messages.PeerDown;
  91 import org.opendaylight.controller.cluster.datastore.messages.PeerUp;
  92 import org.opendaylight.controller.cluster.datastore.messages.PrimaryShardInfo;
  93 import org.opendaylight.controller.cluster.datastore.messages.RemotePrimaryShardFound;
  94 import org.opendaylight.controller.cluster.datastore.messages.RemoveShardReplica;
  95 import org.opendaylight.controller.cluster.datastore.messages.ShardLeaderStateChanged;
  96 import org.opendaylight.controller.cluster.datastore.messages.UpdateSchemaContext;
  97 import org.opendaylight.controller.cluster.datastore.persisted.DatastoreSnapshot;
  98 import org.opendaylight.controller.cluster.datastore.persisted.DatastoreSnapshot.ShardSnapshot;
  99 import org.opendaylight.controller.cluster.datastore.persisted.ShardManagerSnapshot;
 100 import org.opendaylight.controller.cluster.datastore.utils.ForwardingActor;
 101 import org.opendaylight.controller.cluster.datastore.utils.MockClusterWrapper;
 102 import org.opendaylight.controller.cluster.datastore.utils.MockConfiguration;
 103 import org.opendaylight.controller.cluster.datastore.utils.PrimaryShardInfoFutureCache;
 104 import org.opendaylight.controller.cluster.notifications.LeaderStateChanged;
 105 import org.opendaylight.controller.cluster.notifications.RegisterRoleChangeListener;
 106 import org.opendaylight.controller.cluster.notifications.RoleChangeNotification;
 107 import org.opendaylight.controller.cluster.raft.RaftState;
 108 import org.opendaylight.controller.cluster.raft.base.messages.FollowerInitialSyncUpStatus;
 109 import org.opendaylight.controller.cluster.raft.base.messages.SwitchBehavior;
 110 import org.opendaylight.controller.cluster.raft.client.messages.GetSnapshot;
 111 import org.opendaylight.controller.cluster.raft.client.messages.Shutdown;
 112 import org.opendaylight.controller.cluster.raft.messages.AddServer;
 113 import org.opendaylight.controller.cluster.raft.messages.AddServerReply;
 114 import org.opendaylight.controller.cluster.raft.messages.ChangeServersVotingStatus;
 115 import org.opendaylight.controller.cluster.raft.messages.RemoveServer;
 116 import org.opendaylight.controller.cluster.raft.messages.RemoveServerReply;
 117 import org.opendaylight.controller.cluster.raft.messages.ServerChangeReply;
 118 import org.opendaylight.controller.cluster.raft.messages.ServerChangeStatus;
 119 import org.opendaylight.controller.cluster.raft.messages.ServerRemoved;
 120 import org.opendaylight.controller.cluster.raft.policy.DisableElectionsRaftPolicy;
 121 import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal;
 122 import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore;
 123 import org.opendaylight.controller.cluster.raft.utils.MessageCollectorActor;
 124 import org.opendaylight.controller.md.cluster.datastore.model.TestModel;
 125 import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTree;
 126 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
 127 import org.slf4j.Logger;
 128 import org.slf4j.LoggerFactory;
 129 import scala.concurrent.Await;
 130 import scala.concurrent.Future;
 131 import scala.concurrent.duration.FiniteDuration;
 132
 133 public class ShardManagerTest extends AbstractShardManagerTest {
 134     private static final Logger LOG = LoggerFactory.getLogger(ShardManagerTest.class);
 135     private static final MemberName MEMBER_2 = MemberName.forName("member-2");
 136     private static final MemberName MEMBER_3 = MemberName.forName("member-3");
 137
 138     private final String shardMgrID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
 139
 140     private ActorSystem newActorSystem(String config) {
 141         return newActorSystem("cluster-test", config);
 142     }
 143
 144     private ActorRef newMockShardActor(ActorSystem system, String shardName, String memberName) {
 145         String name = ShardIdentifier.create(shardName, MemberName.forName(memberName), "config").toString();
 146         if (system == getSystem()) {
 147             return actorFactory.createTestActor(Props.create(MessageCollectorActor.class), name);
 148         }
 149
 150         return TestActorRef.create(system, Props.create(MessageCollectorActor.class), name);
 151     }
 152
 153     private Props newShardMgrProps() {
 154         return newShardMgrProps(new MockConfiguration());
 155     }
 156
 157     private static DatastoreContextFactory newDatastoreContextFactory(DatastoreContext datastoreContext) {
 158         DatastoreContextFactory mockFactory = mock(DatastoreContextFactory.class);
 159         Mockito.doReturn(datastoreContext).when(mockFactory).getBaseDatastoreContext();
 160         Mockito.doReturn(datastoreContext).when(mockFactory).getShardDatastoreContext(Mockito.anyString());
 161         return mockFactory;
 162     }
 163
 164     private TestShardManager.Builder newTestShardMgrBuilderWithMockShardActor() {
 165         return newTestShardMgrBuilderWithMockShardActor(mockShardActor);
 166     }
 167
 168     private TestShardManager.Builder newTestShardMgrBuilderWithMockShardActor(ActorRef shardActor) {
 169         return TestShardManager.builder(datastoreContextBuilder).shardActor(shardActor)
 170                 .distributedDataStore(mock(DistributedDataStore.class));
 171     }
 172
 173
 174     private Props newPropsShardMgrWithMockShardActor() {
 175         return newTestShardMgrBuilderWithMockShardActor().props().withDispatcher(
 176                 Dispatchers.DefaultDispatcherId());
 177     }
 178
 179     private Props newPropsShardMgrWithMockShardActor(ActorRef shardActor) {
 180         return newTestShardMgrBuilderWithMockShardActor(shardActor).props();
 181     }
 182
 183
 184     private TestShardManager newTestShardManager() {
 185         return newTestShardManager(newShardMgrProps());
 186     }
 187
 188     private TestShardManager newTestShardManager(Props props) {
 189         TestActorRef<TestShardManager> shardManagerActor = actorFactory.createTestActor(props);
 190         TestShardManager shardManager = shardManagerActor.underlyingActor();
 191         shardManager.waitForRecoveryComplete();
 192         return shardManager;
 193     }
 194
 195     private static void waitForShardInitialized(ActorRef shardManager, String shardName, JavaTestKit kit) {
 196         AssertionError last = null;
 197         Stopwatch sw = Stopwatch.createStarted();
 198         while (sw.elapsed(TimeUnit.SECONDS) <= 5) {
 199             try {
 200                 shardManager.tell(new FindLocalShard(shardName, true), kit.getRef());
 201                 kit.expectMsgClass(LocalShardFound.class);
 202                 return;
 203             } catch (AssertionError e) {
 204                 last = e;
 205             }
 206
 207             Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
 208         }
 209
 210         throw last;
 211     }
 212
 213     @SuppressWarnings("unchecked")
 214     private static <T> T expectMsgClassOrFailure(Class<T> msgClass, JavaTestKit kit, String msg) {
 215         Object reply = kit.expectMsgAnyClassOf(JavaTestKit.duration("5 sec"), msgClass, Failure.class);
 216         if (reply instanceof Failure) {
 217             throw new AssertionError(msg + " failed", ((Failure)reply).cause());
 218         }
 219
 220         return (T)reply;
 221     }
 222
 223     @Test
 224     public void testPerShardDatastoreContext() throws Exception {
 225         LOG.info("testPerShardDatastoreContext starting");
 226         final DatastoreContextFactory mockFactory = newDatastoreContextFactory(
 227                 datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
 228
 229         Mockito.doReturn(
 230                 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(6).build())
 231                 .when(mockFactory).getShardDatastoreContext("default");
 232
 233         Mockito.doReturn(
 234                 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(7).build())
 235                 .when(mockFactory).getShardDatastoreContext("topology");
 236
 237         final MockConfiguration mockConfig = new MockConfiguration() {
 238             @Override
 239             public Collection<String> getMemberShardNames(MemberName memberName) {
 240                 return Arrays.asList("default", "topology");
 241             }
 242
 243             @Override
 244             public Collection<MemberName> getMembersFromShardName(String shardName) {
 245                 return members("member-1");
 246             }
 247         };
 248
 249         final TestActorRef<MessageCollectorActor> defaultShardActor = actorFactory.createTestActor(
 250                 Props.create(MessageCollectorActor.class), actorFactory.generateActorId("default"));
 251         final TestActorRef<MessageCollectorActor> topologyShardActor = actorFactory.createTestActor(
 252                 Props.create(MessageCollectorActor.class), actorFactory.generateActorId("topology"));
 253
 254         final Map<String, Entry<ActorRef, DatastoreContext>> shardInfoMap = Collections.synchronizedMap(
 255                 new HashMap<String, Entry<ActorRef, DatastoreContext>>());
 256         shardInfoMap.put("default", new AbstractMap.SimpleEntry<>(defaultShardActor, null));
 257         shardInfoMap.put("topology", new AbstractMap.SimpleEntry<>(topologyShardActor, null));
 258
 259         final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
 260         final CountDownLatch newShardActorLatch = new CountDownLatch(2);
 261         class LocalShardManager extends ShardManager {
 262             LocalShardManager(AbstractShardManagerCreator<?> creator) {
 263                 super(creator);
 264             }
 265
 266             @Override
 267             protected ActorRef newShardActor(SchemaContext schemaContext, ShardInformation info) {
 268                 Entry<ActorRef, DatastoreContext> entry = shardInfoMap.get(info.getShardName());
 269                 ActorRef ref = null;
 270                 if (entry != null) {
 271                     ref = entry.getKey();
 272                     entry.setValue(info.getDatastoreContext());
 273                 }
 274
 275                 newShardActorLatch.countDown();
 276                 return ref;
 277             }
 278         }
 279
 280         final Creator<ShardManager> creator = new Creator<ShardManager>() {
 281             private static final long serialVersionUID = 1L;
 282             @Override
 283             public ShardManager create() throws Exception {
 284                 return new LocalShardManager(
 285                         new GenericCreator<>(LocalShardManager.class).datastoreContextFactory(mockFactory)
 286                                 .primaryShardInfoCache(primaryShardInfoCache).configuration(mockConfig));
 287             }
 288         };
 289
 290         JavaTestKit kit = new JavaTestKit(getSystem());
 291
 292         final ActorRef shardManager = actorFactory.createActor(Props.create(
 293                 new DelegatingShardManagerCreator(creator)).withDispatcher(Dispatchers.DefaultDispatcherId()));
 294
 295         shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), kit.getRef());
 296
 297         assertEquals("Shard actors created", true, newShardActorLatch.await(5, TimeUnit.SECONDS));
 298         assertEquals("getShardElectionTimeoutFactor", 6,
 299                 shardInfoMap.get("default").getValue().getShardElectionTimeoutFactor());
 300         assertEquals("getShardElectionTimeoutFactor", 7,
 301                 shardInfoMap.get("topology").getValue().getShardElectionTimeoutFactor());
 302
 303         DatastoreContextFactory newMockFactory = newDatastoreContextFactory(
 304                 datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
 305         Mockito.doReturn(
 306                 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(66).build())
 307                 .when(newMockFactory).getShardDatastoreContext("default");
 308
 309         Mockito.doReturn(
 310                 DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).shardElectionTimeoutFactor(77).build())
 311                 .when(newMockFactory).getShardDatastoreContext("topology");
 312
 313         shardManager.tell(newMockFactory, kit.getRef());
 314
 315         DatastoreContext newContext = MessageCollectorActor.expectFirstMatching(defaultShardActor,
 316                 DatastoreContext.class);
 317         assertEquals("getShardElectionTimeoutFactor", 66, newContext.getShardElectionTimeoutFactor());
 318
 319         newContext = MessageCollectorActor.expectFirstMatching(topologyShardActor, DatastoreContext.class);
 320         assertEquals("getShardElectionTimeoutFactor", 77, newContext.getShardElectionTimeoutFactor());
 321
 322         LOG.info("testPerShardDatastoreContext ending");
 323     }
 324
 325     @Test
 326     public void testOnReceiveFindPrimaryForNonExistentShard() throws Exception {
 327         new JavaTestKit(getSystem()) {
 328             {
 329                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 330
 331                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 332
 333                 shardManager.tell(new FindPrimary("non-existent", false), getRef());
 334
 335                 expectMsgClass(duration("5 seconds"), PrimaryNotFoundException.class);
 336             }
 337         };
 338     }
 339
 340     @Test
 341     public void testOnReceiveFindPrimaryForLocalLeaderShard() throws Exception {
 342         LOG.info("testOnReceiveFindPrimaryForLocalLeaderShard starting");
 343         new JavaTestKit(getSystem()) {
 344             {
 345                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
 346
 347                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 348
 349                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 350                 shardManager.tell(new ActorInitialized(), mockShardActor);
 351
 352                 DataTree mockDataTree = mock(DataTree.class);
 353                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
 354                         DataStoreVersions.CURRENT_VERSION), getRef());
 355
 356                 MessageCollectorActor.expectFirstMatching(mockShardActor, RegisterRoleChangeListener.class);
 357                 shardManager.tell(
 358                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
 359                         mockShardActor);
 360
 361                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 362
 363                 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
 364                         LocalPrimaryShardFound.class);
 365                 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
 366                         primaryFound.getPrimaryPath().contains("member-1-shard-default"));
 367                 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
 368             }
 369         };
 370
 371         LOG.info("testOnReceiveFindPrimaryForLocalLeaderShard ending");
 372     }
 373
 374     @Test
 375     public void testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp() throws Exception {
 376         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp starting");
 377         new JavaTestKit(getSystem()) {
 378             {
 379                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 380
 381                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 382                 shardManager.tell(new ActorInitialized(), mockShardActor);
 383
 384                 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
 385                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
 386                 shardManager.tell(
 387                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
 388                         mockShardActor);
 389                 shardManager.tell(new LeaderStateChanged(memberId1, memberId2, DataStoreVersions.CURRENT_VERSION),
 390                         mockShardActor);
 391
 392                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 393
 394                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 395             }
 396         };
 397
 398         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp ending");
 399     }
 400
 401     @Test
 402     public void testOnReceiveFindPrimaryForNonLocalLeaderShard() throws Exception {
 403         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShard starting");
 404         new JavaTestKit(getSystem()) {
 405             {
 406                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 407
 408                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 409                 shardManager.tell(new ActorInitialized(), mockShardActor);
 410
 411                 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
 412                 MockClusterWrapper.sendMemberUp(shardManager, "member-2", getRef().path().toString());
 413
 414                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
 415                 shardManager.tell(
 416                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
 417                         mockShardActor);
 418                 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
 419                 shardManager.tell(new ShardLeaderStateChanged(memberId1, memberId2, leaderVersion), mockShardActor);
 420
 421                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 422
 423                 RemotePrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
 424                         RemotePrimaryShardFound.class);
 425                 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
 426                         primaryFound.getPrimaryPath().contains("member-2-shard-default"));
 427                 assertEquals("getPrimaryVersion", leaderVersion, primaryFound.getPrimaryVersion());
 428             }
 429         };
 430
 431         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShard ending");
 432     }
 433
 434     @Test
 435     public void testOnReceiveFindPrimaryForUninitializedShard() throws Exception {
 436         new JavaTestKit(getSystem()) {
 437             {
 438                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 439
 440                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 441
 442                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
 443             }
 444         };
 445     }
 446
 447     @Test
 448     public void testOnReceiveFindPrimaryForInitializedShardWithNoRole() throws Exception {
 449         new JavaTestKit(getSystem()) {
 450             {
 451                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 452
 453                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 454                 shardManager.tell(new ActorInitialized(), mockShardActor);
 455
 456                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 457
 458                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 459             }
 460         };
 461     }
 462
 463     @Test
 464     public void testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId() throws Exception {
 465         LOG.info("testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId starting");
 466         new JavaTestKit(getSystem()) {
 467             {
 468                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 469
 470                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 471                 shardManager.tell(new ActorInitialized(), mockShardActor);
 472
 473                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
 474                 shardManager.tell(
 475                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Follower.name()),
 476                         mockShardActor);
 477
 478                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 479
 480                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 481
 482                 DataTree mockDataTree = mock(DataTree.class);
 483                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
 484                         DataStoreVersions.CURRENT_VERSION), mockShardActor);
 485
 486                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 487
 488                 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
 489                         LocalPrimaryShardFound.class);
 490                 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
 491                         primaryFound.getPrimaryPath().contains("member-1-shard-default"));
 492                 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
 493             }
 494         };
 495
 496         LOG.info("testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId starting");
 497     }
 498
 499     @Test
 500     public void testOnReceiveFindPrimaryWaitForShardLeader() throws Exception {
 501         LOG.info("testOnReceiveFindPrimaryWaitForShardLeader starting");
 502         datastoreContextBuilder.shardInitializationTimeout(10, TimeUnit.SECONDS);
 503         new JavaTestKit(getSystem()) {
 504             {
 505                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 506
 507                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 508
 509                 // We're passing waitUntilInitialized = true to FindPrimary so
 510                 // the response should be
 511                 // delayed until we send ActorInitialized and
 512                 // RoleChangeNotification.
 513                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
 514
 515                 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
 516
 517                 shardManager.tell(new ActorInitialized(), mockShardActor);
 518
 519                 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
 520
 521                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
 522                 shardManager.tell(
 523                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
 524                         mockShardActor);
 525
 526                 expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
 527
 528                 DataTree mockDataTree = mock(DataTree.class);
 529                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mockDataTree,
 530                         DataStoreVersions.CURRENT_VERSION), mockShardActor);
 531
 532                 LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"),
 533                         LocalPrimaryShardFound.class);
 534                 assertTrue("Unexpected primary path " + primaryFound.getPrimaryPath(),
 535                         primaryFound.getPrimaryPath().contains("member-1-shard-default"));
 536                 assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
 537
 538                 expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
 539             }
 540         };
 541
 542         LOG.info("testOnReceiveFindPrimaryWaitForShardLeader ending");
 543     }
 544
 545     @Test
 546     public void testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard() throws Exception {
 547         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard starting");
 548         new JavaTestKit(getSystem()) {
 549             {
 550                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 551
 552                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 553
 554                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
 555
 556                 expectMsgClass(duration("2 seconds"), NotInitializedException.class);
 557
 558                 shardManager.tell(new ActorInitialized(), mockShardActor);
 559
 560                 expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
 561             }
 562         };
 563
 564         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard ending");
 565     }
 566
 567     @Test
 568     public void testOnReceiveFindPrimaryWaitForReadyWithCandidateShard() throws Exception {
 569         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithCandidateShard starting");
 570         new JavaTestKit(getSystem()) {
 571             {
 572                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 573
 574                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 575                 shardManager.tell(new ActorInitialized(), mockShardActor);
 576                 shardManager.tell(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix, null,
 577                         RaftState.Candidate.name()), mockShardActor);
 578
 579                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
 580
 581                 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
 582             }
 583         };
 584
 585         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithCandidateShard ending");
 586     }
 587
 588     @Test
 589     public void testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard() throws Exception {
 590         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard starting");
 591         new JavaTestKit(getSystem()) {
 592             {
 593                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 594
 595                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 596                 shardManager.tell(new ActorInitialized(), mockShardActor);
 597                 shardManager.tell(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix, null,
 598                         RaftState.IsolatedLeader.name()), mockShardActor);
 599
 600                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
 601
 602                 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
 603             }
 604         };
 605
 606         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard ending");
 607     }
 608
 609     @Test
 610     public void testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard() throws Exception {
 611         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard starting");
 612         new JavaTestKit(getSystem()) {
 613             {
 614                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 615
 616                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 617                 shardManager.tell(new ActorInitialized(), mockShardActor);
 618
 619                 shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
 620
 621                 expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
 622             }
 623         };
 624
 625         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard ending");
 626     }
 627
 628     @Test
 629     public void testOnReceiveFindPrimaryForRemoteShard() throws Exception {
 630         LOG.info("testOnReceiveFindPrimaryForRemoteShard starting");
 631         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
 632
 633         // Create an ActorSystem ShardManager actor for member-1.
 634
 635         final ActorSystem system1 = newActorSystem("Member1");
 636         Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
 637
 638         final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
 639                 newTestShardMgrBuilderWithMockShardActor().cluster(
 640                         new ClusterWrapperImpl(system1)).props().withDispatcher(
 641                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
 642
 643         // Create an ActorSystem ShardManager actor for member-2.
 644
 645         final ActorSystem system2 = newActorSystem("Member2");
 646
 647         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
 648
 649         final ActorRef mockShardActor2 = newMockShardActor(system2, "astronauts", "member-2");
 650
 651         MockConfiguration mockConfig2 = new MockConfiguration(
 652                 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
 653                         .put("astronauts", Arrays.asList("member-2")).build());
 654
 655         final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
 656                 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
 657                         new ClusterWrapperImpl(system2)).props().withDispatcher(
 658                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
 659
 660         new JavaTestKit(system1) {
 661             {
 662                 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 663                 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 664
 665                 shardManager2.tell(new ActorInitialized(), mockShardActor2);
 666
 667                 String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
 668                 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
 669                 shardManager2.tell(
 670                         new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
 671                         mockShardActor2);
 672                 shardManager2.tell(
 673                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
 674                         mockShardActor2);
 675
 676                 shardManager1.underlyingActor().waitForMemberUp();
 677                 shardManager1.tell(new FindPrimary("astronauts", false), getRef());
 678
 679                 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 680                 String path = found.getPrimaryPath();
 681                 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-astronauts-config"));
 682                 assertEquals("getPrimaryVersion", leaderVersion, found.getPrimaryVersion());
 683
 684                 shardManager2.underlyingActor().verifyFindPrimary();
 685
 686                 Cluster.get(system2).down(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
 687
 688                 shardManager1.underlyingActor().waitForMemberRemoved();
 689
 690                 shardManager1.tell(new FindPrimary("astronauts", false), getRef());
 691
 692                 expectMsgClass(duration("5 seconds"), PrimaryNotFoundException.class);
 693             }
 694         };
 695
 696         LOG.info("testOnReceiveFindPrimaryForRemoteShard ending");
 697     }
 698
 699     @Test
 700     public void testShardAvailabilityOnChangeOfMemberReachability() throws Exception {
 701         LOG.info("testShardAvailabilityOnChangeOfMemberReachability starting");
 702         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
 703
 704         // Create an ActorSystem ShardManager actor for member-1.
 705
 706         final ActorSystem system1 = newActorSystem("Member1");
 707         Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
 708
 709         final ActorRef mockShardActor1 = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
 710
 711         final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
 712                 newTestShardMgrBuilder().shardActor(mockShardActor1).cluster(
 713                         new ClusterWrapperImpl(system1)).props().withDispatcher(
 714                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
 715
 716         // Create an ActorSystem ShardManager actor for member-2.
 717
 718         final ActorSystem system2 = newActorSystem("Member2");
 719
 720         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
 721
 722         final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
 723
 724         MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
 725                 .put("default", Arrays.asList("member-1", "member-2")).build());
 726
 727         final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
 728                 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
 729                         new ClusterWrapperImpl(system2)).props().withDispatcher(
 730                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
 731
 732         new JavaTestKit(system1) {
 733             {
 734                 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 735                 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 736                 shardManager1.tell(new ActorInitialized(), mockShardActor1);
 737                 shardManager2.tell(new ActorInitialized(), mockShardActor2);
 738
 739                 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
 740                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
 741                 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class),
 742                         DataStoreVersions.CURRENT_VERSION), mockShardActor1);
 743                 shardManager1.tell(
 744                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
 745                         mockShardActor1);
 746                 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class),
 747                         DataStoreVersions.CURRENT_VERSION), mockShardActor2);
 748                 shardManager2.tell(
 749                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
 750                         mockShardActor2);
 751                 shardManager1.underlyingActor().waitForMemberUp();
 752
 753                 shardManager1.tell(new FindPrimary("default", true), getRef());
 754
 755                 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 756                 String path = found.getPrimaryPath();
 757                 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
 758
 759                 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
 760                         "akka://cluster-test@127.0.0.1:2558"), getRef());
 761
 762                 shardManager1.underlyingActor().waitForUnreachableMember();
 763
 764                 PeerDown peerDown = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
 765                 assertEquals("getMemberName", MEMBER_2, peerDown.getMemberName());
 766                 MessageCollectorActor.clearMessages(mockShardActor1);
 767
 768                 shardManager1.tell(
 769                         MockClusterWrapper.createMemberRemoved("member-2", "akka://cluster-test@127.0.0.1:2558"),
 770                         getRef());
 771
 772                 MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
 773
 774                 shardManager1.tell(new FindPrimary("default", true), getRef());
 775
 776                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 777
 778                 shardManager1.tell(
 779                         MockClusterWrapper.createReachableMember("member-2", "akka://cluster-test@127.0.0.1:2558"),
 780                         getRef());
 781
 782                 shardManager1.underlyingActor().waitForReachableMember();
 783
 784                 PeerUp peerUp = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
 785                 assertEquals("getMemberName", MEMBER_2, peerUp.getMemberName());
 786                 MessageCollectorActor.clearMessages(mockShardActor1);
 787
 788                 shardManager1.tell(new FindPrimary("default", true), getRef());
 789
 790                 RemotePrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 791                 String path1 = found1.getPrimaryPath();
 792                 assertTrue("Unexpected primary path " + path1, path1.contains("member-2-shard-default-config"));
 793
 794                 shardManager1.tell(
 795                         MockClusterWrapper.createMemberUp("member-2", "akka://cluster-test@127.0.0.1:2558"),
 796                         getRef());
 797
 798                 MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
 799
 800                 // Test FindPrimary wait succeeds after reachable member event.
 801
 802                 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
 803                         "akka://cluster-test@127.0.0.1:2558"), getRef());
 804                 shardManager1.underlyingActor().waitForUnreachableMember();
 805
 806                 shardManager1.tell(new FindPrimary("default", true), getRef());
 807
 808                 shardManager1.tell(
 809                         MockClusterWrapper.createReachableMember("member-2", "akka://cluster-test@127.0.0.1:2558"),
 810                         getRef());
 811
 812                 RemotePrimaryShardFound found2 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 813                 String path2 = found2.getPrimaryPath();
 814                 assertTrue("Unexpected primary path " + path2, path2.contains("member-2-shard-default-config"));
 815             }
 816         };
 817
 818         LOG.info("testShardAvailabilityOnChangeOfMemberReachability ending");
 819     }
 820
 821     @Test
 822     public void testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange() throws Exception {
 823         LOG.info("testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange starting");
 824         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
 825
 826         // Create an ActorSystem ShardManager actor for member-1.
 827
 828         final ActorSystem system1 = newActorSystem("Member1");
 829         Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
 830
 831         final ActorRef mockShardActor1 = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
 832
 833         final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
 834         final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
 835                 newTestShardMgrBuilder().shardActor(mockShardActor1).cluster(new ClusterWrapperImpl(system1))
 836                         .primaryShardInfoCache(primaryShardInfoCache).props()
 837                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
 838                 shardManagerID);
 839
 840         // Create an ActorSystem ShardManager actor for member-2.
 841
 842         final ActorSystem system2 = newActorSystem("Member2");
 843
 844         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
 845
 846         final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
 847
 848         MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
 849                 .put("default", Arrays.asList("member-1", "member-2")).build());
 850
 851         final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
 852                 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
 853                         new ClusterWrapperImpl(system2)).props().withDispatcher(
 854                                 Dispatchers.DefaultDispatcherId()), shardManagerID);
 855
 856         new JavaTestKit(system1) {
 857             {
 858                 shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 859                 shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 860                 shardManager1.tell(new ActorInitialized(), mockShardActor1);
 861                 shardManager2.tell(new ActorInitialized(), mockShardActor2);
 862
 863                 String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
 864                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
 865                 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class),
 866                         DataStoreVersions.CURRENT_VERSION), mockShardActor1);
 867                 shardManager1.tell(
 868                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
 869                         mockShardActor1);
 870                 shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class),
 871                         DataStoreVersions.CURRENT_VERSION), mockShardActor2);
 872                 shardManager2.tell(
 873                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
 874                         mockShardActor2);
 875                 shardManager1.underlyingActor().waitForMemberUp();
 876
 877                 shardManager1.tell(new FindPrimary("default", true), getRef());
 878
 879                 RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 880                 String path = found.getPrimaryPath();
 881                 assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
 882
 883                 primaryShardInfoCache.putSuccessful("default", new PrimaryShardInfo(
 884                         system1.actorSelection(mockShardActor1.path()), DataStoreVersions.CURRENT_VERSION));
 885
 886                 shardManager1.tell(MockClusterWrapper.createUnreachableMember("member-2",
 887                         "akka://cluster-test@127.0.0.1:2558"), getRef());
 888
 889                 shardManager1.underlyingActor().waitForUnreachableMember();
 890
 891                 shardManager1.tell(new FindPrimary("default", true), getRef());
 892
 893                 expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 894
 895                 assertNull("Expected primaryShardInfoCache entry removed",
 896                         primaryShardInfoCache.getIfPresent("default"));
 897
 898                 shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId1, mock(DataTree.class),
 899                         DataStoreVersions.CURRENT_VERSION), mockShardActor1);
 900                 shardManager1.tell(
 901                         new RoleChangeNotification(memberId1, RaftState.Follower.name(), RaftState.Leader.name()),
 902                         mockShardActor1);
 903
 904                 shardManager1.tell(new FindPrimary("default", true), getRef());
 905
 906                 LocalPrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
 907                 String path1 = found1.getPrimaryPath();
 908                 assertTrue("Unexpected primary path " + path1, path1.contains("member-1-shard-default-config"));
 909
 910             }
 911         };
 912
 913         LOG.info("testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange ending");
 914     }
 915
 916
 917     @Test
 918     public void testOnReceiveFindLocalShardForNonExistentShard() throws Exception {
 919         new JavaTestKit(getSystem()) {
 920             {
 921                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 922
 923                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 924
 925                 shardManager.tell(new FindLocalShard("non-existent", false), getRef());
 926
 927                 LocalShardNotFound notFound = expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
 928
 929                 assertEquals("getShardName", "non-existent", notFound.getShardName());
 930             }
 931         };
 932     }
 933
 934     @Test
 935     public void testOnReceiveFindLocalShardForExistentShard() throws Exception {
 936         new JavaTestKit(getSystem()) {
 937             {
 938                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 939
 940                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 941                 shardManager.tell(new ActorInitialized(), mockShardActor);
 942
 943                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
 944
 945                 LocalShardFound found = expectMsgClass(duration("5 seconds"), LocalShardFound.class);
 946
 947                 assertTrue("Found path contains " + found.getPath().path().toString(),
 948                         found.getPath().path().toString().contains("member-1-shard-default-config"));
 949             }
 950         };
 951     }
 952
 953     @Test
 954     public void testOnReceiveFindLocalShardForNotInitializedShard() throws Exception {
 955         new JavaTestKit(getSystem()) {
 956             {
 957                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 958
 959                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
 960
 961                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
 962             }
 963         };
 964     }
 965
 966     @Test
 967     public void testOnReceiveFindLocalShardWaitForShardInitialized() throws Exception {
 968         LOG.info("testOnReceiveFindLocalShardWaitForShardInitialized starting");
 969         new JavaTestKit(getSystem()) {
 970             {
 971                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 972
 973                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 974
 975                 // We're passing waitUntilInitialized = true to FindLocalShard
 976                 // so the response should be
 977                 // delayed until we send ActorInitialized.
 978                 Future<Object> future = Patterns.ask(shardManager, new FindLocalShard(Shard.DEFAULT_NAME, true),
 979                         new Timeout(5, TimeUnit.SECONDS));
 980
 981                 shardManager.tell(new ActorInitialized(), mockShardActor);
 982
 983                 Object resp = Await.result(future, duration("5 seconds"));
 984                 assertTrue("Expected: LocalShardFound, Actual: " + resp, resp instanceof LocalShardFound);
 985             }
 986         };
 987
 988         LOG.info("testOnReceiveFindLocalShardWaitForShardInitialized starting");
 989     }
 990
 991     @Test
 992     public void testOnRecoveryJournalIsCleaned() {
 993         String persistenceID = "shard-manager-" + shardMrgIDSuffix;
 994         InMemoryJournal.addEntry(persistenceID, 1L, new SchemaContextModules(ImmutableSet.of("foo")));
 995         InMemoryJournal.addEntry(persistenceID, 2L, new SchemaContextModules(ImmutableSet.of("bar")));
 996         InMemoryJournal.addDeleteMessagesCompleteLatch(persistenceID);
 997
 998         newTestShardManager();
 999
1000         InMemoryJournal.waitForDeleteMessagesComplete(persistenceID);
1001
1002         // Journal entries up to the last one should've been deleted
1003         Map<Long, Object> journal = InMemoryJournal.get(persistenceID);
1004         synchronized (journal) {
1005             assertEquals("Journal size", 0, journal.size());
1006         }
1007     }
1008
1009     @Test
1010     public void testRoleChangeNotificationAndShardLeaderStateChangedReleaseReady() throws Exception {
1011         TestShardManager shardManager = newTestShardManager();
1012
1013         String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1014         shardManager.onReceiveCommand(new RoleChangeNotification(
1015                 memberId, RaftState.Candidate.name(), RaftState.Leader.name()));
1016
1017         verify(ready, never()).countDown();
1018
1019         shardManager.onReceiveCommand(new ShardLeaderStateChanged(memberId, memberId,
1020                 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1021
1022         verify(ready, times(1)).countDown();
1023     }
1024
1025     @Test
1026     public void testRoleChangeNotificationToFollowerWithShardLeaderStateChangedReleaseReady() throws Exception {
1027         new JavaTestKit(getSystem()) {
1028             {
1029                 TestShardManager shardManager = newTestShardManager();
1030
1031                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1032                 shardManager.onReceiveCommand(new RoleChangeNotification(memberId, null, RaftState.Follower.name()));
1033
1034                 verify(ready, never()).countDown();
1035
1036                 shardManager
1037                         .onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
1038
1039                 shardManager.onReceiveCommand(
1040                         new ShardLeaderStateChanged(memberId, "member-2-shard-default-" + shardMrgIDSuffix,
1041                                 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1042
1043                 verify(ready, times(1)).countDown();
1044             }
1045         };
1046     }
1047
1048     @Test
1049     public void testReadyCountDownForMemberUpAfterLeaderStateChanged() throws Exception {
1050         new JavaTestKit(getSystem()) {
1051             {
1052                 TestShardManager shardManager = newTestShardManager();
1053
1054                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1055                 shardManager.onReceiveCommand(new RoleChangeNotification(memberId, null, RaftState.Follower.name()));
1056
1057                 verify(ready, never()).countDown();
1058
1059                 shardManager.onReceiveCommand(
1060                         new ShardLeaderStateChanged(memberId, "member-2-shard-default-" + shardMrgIDSuffix,
1061                                 mock(DataTree.class), DataStoreVersions.CURRENT_VERSION));
1062
1063                 shardManager
1064                         .onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
1065
1066                 verify(ready, times(1)).countDown();
1067             }
1068         };
1069     }
1070
1071     @Test
1072     public void testRoleChangeNotificationDoNothingForUnknownShard() throws Exception {
1073         TestShardManager shardManager = newTestShardManager();
1074
1075         shardManager.onReceiveCommand(new RoleChangeNotification(
1076                 "unknown", RaftState.Candidate.name(), RaftState.Leader.name()));
1077
1078         verify(ready, never()).countDown();
1079     }
1080
1081     @Test
1082     public void testByDefaultSyncStatusIsFalse() throws Exception {
1083         TestShardManager shardManager = newTestShardManager();
1084
1085         assertEquals(false, shardManager.getMBean().getSyncStatus());
1086     }
1087
1088     @Test
1089     public void testWhenShardIsLeaderSyncStatusIsTrue() throws Exception {
1090         TestShardManager shardManager = newTestShardManager();
1091
1092         shardManager.onReceiveCommand(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix,
1093                 RaftState.Follower.name(), RaftState.Leader.name()));
1094
1095         assertEquals(true, shardManager.getMBean().getSyncStatus());
1096     }
1097
1098     @Test
1099     public void testWhenShardIsCandidateSyncStatusIsFalse() throws Exception {
1100         TestShardManager shardManager = newTestShardManager();
1101
1102         String shardId = "member-1-shard-default-" + shardMrgIDSuffix;
1103         shardManager.onReceiveCommand(new RoleChangeNotification(shardId,
1104                 RaftState.Follower.name(), RaftState.Candidate.name()));
1105
1106         assertEquals(false, shardManager.getMBean().getSyncStatus());
1107
1108         // Send a FollowerInitialSyncStatus with status = true for the replica whose current state is candidate
1109         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(
1110                 true, shardId));
1111
1112         assertEquals(false, shardManager.getMBean().getSyncStatus());
1113     }
1114
1115     @Test
1116     public void testWhenShardIsFollowerSyncStatusDependsOnFollowerInitialSyncStatus() throws Exception {
1117         TestShardManager shardManager = newTestShardManager();
1118
1119         String shardId = "member-1-shard-default-" + shardMrgIDSuffix;
1120         shardManager.onReceiveCommand(new RoleChangeNotification(shardId,
1121                 RaftState.Candidate.name(), RaftState.Follower.name()));
1122
1123         // Initially will be false
1124         assertEquals(false, shardManager.getMBean().getSyncStatus());
1125
1126         // Send status true will make sync status true
1127         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(true, shardId));
1128
1129         assertEquals(true, shardManager.getMBean().getSyncStatus());
1130
1131         // Send status false will make sync status false
1132         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(false, shardId));
1133
1134         assertEquals(false, shardManager.getMBean().getSyncStatus());
1135
1136     }
1137
1138     @Test
1139     public void testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards() throws Exception {
1140         LOG.info("testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards starting");
1141         TestShardManager shardManager = newTestShardManager(newShardMgrProps(new MockConfiguration() {
1142             @Override
1143             public List<String> getMemberShardNames(MemberName memberName) {
1144                 return Arrays.asList("default", "astronauts");
1145             }
1146         }));
1147
1148         // Initially will be false
1149         assertEquals(false, shardManager.getMBean().getSyncStatus());
1150
1151         // Make default shard leader
1152         String defaultShardId = "member-1-shard-default-" + shardMrgIDSuffix;
1153         shardManager.onReceiveCommand(new RoleChangeNotification(defaultShardId,
1154                 RaftState.Follower.name(), RaftState.Leader.name()));
1155
1156         // default = Leader, astronauts is unknown so sync status remains false
1157         assertEquals(false, shardManager.getMBean().getSyncStatus());
1158
1159         // Make astronauts shard leader as well
1160         String astronautsShardId = "member-1-shard-astronauts-" + shardMrgIDSuffix;
1161         shardManager.onReceiveCommand(new RoleChangeNotification(astronautsShardId,
1162                 RaftState.Follower.name(), RaftState.Leader.name()));
1163
1164         // Now sync status should be true
1165         assertEquals(true, shardManager.getMBean().getSyncStatus());
1166
1167         // Make astronauts a Follower
1168         shardManager.onReceiveCommand(new RoleChangeNotification(astronautsShardId,
1169                 RaftState.Leader.name(), RaftState.Follower.name()));
1170
1171         // Sync status is not true
1172         assertEquals(false, shardManager.getMBean().getSyncStatus());
1173
1174         // Make the astronauts follower sync status true
1175         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(true, astronautsShardId));
1176
1177         // Sync status is now true
1178         assertEquals(true, shardManager.getMBean().getSyncStatus());
1179
1180         LOG.info("testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards ending");
1181     }
1182
1183     @Test
1184     public void testOnReceiveSwitchShardBehavior() throws Exception {
1185         new JavaTestKit(getSystem()) {
1186             {
1187                 final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1188
1189                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1190                 shardManager.tell(new ActorInitialized(), mockShardActor);
1191
1192                 shardManager.tell(new SwitchShardBehavior(mockShardName, RaftState.Leader, 1000), getRef());
1193
1194                 SwitchBehavior switchBehavior = MessageCollectorActor.expectFirstMatching(mockShardActor,
1195                         SwitchBehavior.class);
1196
1197                 assertEquals(RaftState.Leader, switchBehavior.getNewState());
1198                 assertEquals(1000, switchBehavior.getNewTerm());
1199             }
1200         };
1201     }
1202
1203     private static List<MemberName> members(String... names) {
1204         return Arrays.asList(names).stream().map(MemberName::forName).collect(Collectors.toList());
1205     }
1206
1207     @Test
1208     public void testOnCreateShard() {
1209         LOG.info("testOnCreateShard starting");
1210         new JavaTestKit(getSystem()) {
1211             {
1212                 datastoreContextBuilder.shardInitializationTimeout(1, TimeUnit.MINUTES).persistent(true);
1213
1214                 ActorRef shardManager = actorFactory
1215                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1216
1217                 SchemaContext schemaContext = TestModel.createTestContext();
1218                 shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
1219
1220                 DatastoreContext datastoreContext = DatastoreContext.newBuilder().shardElectionTimeoutFactor(100)
1221                         .persistent(false).build();
1222                 Shard.Builder shardBuilder = Shard.builder();
1223
1224                 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1225                         "foo", null, members("member-1", "member-5", "member-6"));
1226                 shardManager.tell(new CreateShard(config, shardBuilder, datastoreContext), getRef());
1227
1228                 expectMsgClass(duration("5 seconds"), Success.class);
1229
1230                 shardManager.tell(new FindLocalShard("foo", true), getRef());
1231
1232                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1233
1234                 assertEquals("isRecoveryApplicable", false, shardBuilder.getDatastoreContext().isPersistent());
1235                 assertTrue("Epxected ShardPeerAddressResolver", shardBuilder.getDatastoreContext().getShardRaftConfig()
1236                         .getPeerAddressResolver() instanceof ShardPeerAddressResolver);
1237                 assertEquals("peerMembers", Sets.newHashSet(
1238                         ShardIdentifier.create("foo", MemberName.forName("member-5"), shardMrgIDSuffix).toString(),
1239                         ShardIdentifier.create("foo", MemberName.forName("member-6"), shardMrgIDSuffix).toString()),
1240                         shardBuilder.getPeerAddresses().keySet());
1241                 assertEquals("ShardIdentifier", ShardIdentifier.create("foo", MEMBER_1, shardMrgIDSuffix),
1242                         shardBuilder.getId());
1243                 assertSame("schemaContext", schemaContext, shardBuilder.getSchemaContext());
1244
1245                 // Send CreateShard with same name - should return Success with
1246                 // a message.
1247
1248                 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1249
1250                 Success success = expectMsgClass(duration("5 seconds"), Success.class);
1251                 assertNotNull("Success status is null", success.status());
1252             }
1253         };
1254
1255         LOG.info("testOnCreateShard ending");
1256     }
1257
1258     @Test
1259     public void testOnCreateShardWithLocalMemberNotInShardConfig() {
1260         LOG.info("testOnCreateShardWithLocalMemberNotInShardConfig starting");
1261         new JavaTestKit(getSystem()) {
1262             {
1263                 datastoreContextBuilder.shardInitializationTimeout(1, TimeUnit.MINUTES).persistent(true);
1264
1265                 ActorRef shardManager = actorFactory
1266                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1267
1268                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1269
1270                 Shard.Builder shardBuilder = Shard.builder();
1271                 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1272                         "foo", null, members("member-5", "member-6"));
1273
1274                 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1275                 expectMsgClass(duration("5 seconds"), Success.class);
1276
1277                 shardManager.tell(new FindLocalShard("foo", true), getRef());
1278                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1279
1280                 assertEquals("peerMembers size", 0, shardBuilder.getPeerAddresses().size());
1281                 assertEquals("schemaContext", DisableElectionsRaftPolicy.class.getName(), shardBuilder
1282                         .getDatastoreContext().getShardRaftConfig().getCustomRaftPolicyImplementationClass());
1283             }
1284         };
1285
1286         LOG.info("testOnCreateShardWithLocalMemberNotInShardConfig ending");
1287     }
1288
1289     @Test
1290     public void testOnCreateShardWithNoInitialSchemaContext() {
1291         LOG.info("testOnCreateShardWithNoInitialSchemaContext starting");
1292         new JavaTestKit(getSystem()) {
1293             {
1294                 ActorRef shardManager = actorFactory
1295                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1296
1297                 Shard.Builder shardBuilder = Shard.builder();
1298
1299                 ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1300                         "foo", null, members("member-1"));
1301                 shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1302
1303                 expectMsgClass(duration("5 seconds"), Success.class);
1304
1305                 SchemaContext schemaContext = TestModel.createTestContext();
1306                 shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
1307
1308                 shardManager.tell(new FindLocalShard("foo", true), getRef());
1309
1310                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1311
1312                 assertSame("schemaContext", schemaContext, shardBuilder.getSchemaContext());
1313                 assertNotNull("schemaContext is null", shardBuilder.getDatastoreContext());
1314             }
1315         };
1316
1317         LOG.info("testOnCreateShardWithNoInitialSchemaContext ending");
1318     }
1319
1320     @Test
1321     public void testGetSnapshot() throws Exception {
1322         LOG.info("testGetSnapshot starting");
1323         JavaTestKit kit = new JavaTestKit(getSystem());
1324
1325         MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1326                 .put("shard1", Arrays.asList("member-1")).put("shard2", Arrays.asList("member-1"))
1327                 .put("astronauts", Collections.<String>emptyList()).build());
1328
1329         TestActorRef<TestShardManager> shardManager = actorFactory
1330                 .createTestActor(newShardMgrProps(mockConfig).withDispatcher(Dispatchers.DefaultDispatcherId()));
1331
1332         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1333         Failure failure = kit.expectMsgClass(Failure.class);
1334         assertEquals("Failure cause type", IllegalStateException.class, failure.cause().getClass());
1335
1336         shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1337
1338         waitForShardInitialized(shardManager, "shard1", kit);
1339         waitForShardInitialized(shardManager, "shard2", kit);
1340
1341         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1342
1343         DatastoreSnapshot datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1344
1345         assertEquals("getType", shardMrgIDSuffix, datastoreSnapshot.getType());
1346         assertNull("Expected null ShardManagerSnapshot", datastoreSnapshot.getShardManagerSnapshot());
1347
1348         Function<ShardSnapshot, String> shardNameTransformer = s -> s.getName();
1349
1350         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2"), Sets.newHashSet(
1351                 Lists.transform(datastoreSnapshot.getShardSnapshots(), shardNameTransformer)));
1352
1353         // Add a new replica
1354
1355         JavaTestKit mockShardLeaderKit = new JavaTestKit(getSystem());
1356
1357         TestShardManager shardManagerInstance = shardManager.underlyingActor();
1358         shardManagerInstance.setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1359
1360         shardManager.tell(new AddShardReplica("astronauts"), kit.getRef());
1361         mockShardLeaderKit.expectMsgClass(AddServer.class);
1362         mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.OK, ""));
1363         kit.expectMsgClass(Status.Success.class);
1364         waitForShardInitialized(shardManager, "astronauts", kit);
1365
1366         // Send another GetSnapshot and verify
1367
1368         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1369         datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1370
1371         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"), Sets.newHashSet(
1372                 Lists.transform(datastoreSnapshot.getShardSnapshots(), shardNameTransformer)));
1373
1374         ShardManagerSnapshot snapshot = datastoreSnapshot.getShardManagerSnapshot();
1375         assertNotNull("Expected ShardManagerSnapshot", snapshot);
1376         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"),
1377                 Sets.newHashSet(snapshot.getShardList()));
1378
1379         LOG.info("testGetSnapshot ending");
1380     }
1381
1382     @Test
1383     public void testRestoreFromSnapshot() throws Exception {
1384         LOG.info("testRestoreFromSnapshot starting");
1385
1386         datastoreContextBuilder.shardInitializationTimeout(3, TimeUnit.SECONDS);
1387
1388         JavaTestKit kit = new JavaTestKit(getSystem());
1389
1390         MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1391                 .put("shard1", Collections.<String>emptyList()).put("shard2", Collections.<String>emptyList())
1392                 .put("astronauts", Collections.<String>emptyList()).build());
1393
1394         ShardManagerSnapshot snapshot =
1395                 new ShardManagerSnapshot(Arrays.asList("shard1", "shard2", "astronauts"), Collections.emptyMap());
1396         DatastoreSnapshot restoreFromSnapshot = new DatastoreSnapshot(shardMrgIDSuffix, snapshot,
1397                 Collections.<ShardSnapshot>emptyList());
1398         TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(newTestShardMgrBuilder(mockConfig)
1399                 .restoreFromSnapshot(restoreFromSnapshot).props().withDispatcher(Dispatchers.DefaultDispatcherId()));
1400
1401         shardManager.underlyingActor().waitForRecoveryComplete();
1402
1403         shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1404
1405         waitForShardInitialized(shardManager, "shard1", kit);
1406         waitForShardInitialized(shardManager, "shard2", kit);
1407         waitForShardInitialized(shardManager, "astronauts", kit);
1408
1409         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1410
1411         DatastoreSnapshot datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1412
1413         assertEquals("getType", shardMrgIDSuffix, datastoreSnapshot.getType());
1414
1415         assertNotNull("Expected ShardManagerSnapshot", datastoreSnapshot.getShardManagerSnapshot());
1416         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"),
1417                 Sets.newHashSet(datastoreSnapshot.getShardManagerSnapshot().getShardList()));
1418
1419         LOG.info("testRestoreFromSnapshot ending");
1420     }
1421
1422     @Test
1423     public void testAddShardReplicaForNonExistentShardConfig() throws Exception {
1424         new JavaTestKit(getSystem()) {
1425             {
1426                 ActorRef shardManager = actorFactory
1427                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1428
1429                 shardManager.tell(new AddShardReplica("model-inventory"), getRef());
1430                 Status.Failure resp = expectMsgClass(duration("2 seconds"), Status.Failure.class);
1431
1432                 assertEquals("Failure obtained", true, resp.cause() instanceof IllegalArgumentException);
1433             }
1434         };
1435     }
1436
1437     @Test
1438     public void testAddShardReplica() throws Exception {
1439         LOG.info("testAddShardReplica starting");
1440         MockConfiguration mockConfig = new MockConfiguration(
1441                 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
1442                         .put("astronauts", Arrays.asList("member-2")).build());
1443
1444         final String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
1445         datastoreContextBuilder.shardManagerPersistenceId(shardManagerID);
1446
1447         // Create an ActorSystem ShardManager actor for member-1.
1448         final ActorSystem system1 = newActorSystem("Member1");
1449         Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1450         ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
1451         final TestActorRef<TestShardManager> newReplicaShardManager = TestActorRef.create(system1,
1452                 newTestShardMgrBuilder(mockConfig).shardActor(mockDefaultShardActor)
1453                         .cluster(new ClusterWrapperImpl(system1)).props()
1454                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
1455                 shardManagerID);
1456
1457         // Create an ActorSystem ShardManager actor for member-2.
1458         final ActorSystem system2 = newActorSystem("Member2");
1459         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1460
1461         String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
1462         String name = ShardIdentifier.create("astronauts", MEMBER_2, "config").toString();
1463         final TestActorRef<MockRespondActor> mockShardLeaderActor = TestActorRef.create(system2,
1464                 Props.create(MockRespondActor.class, AddServer.class,
1465                         new AddServerReply(ServerChangeStatus.OK, memberId2))
1466                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
1467                 name);
1468         final TestActorRef<TestShardManager> leaderShardManager = TestActorRef.create(system2,
1469                 newTestShardMgrBuilder(mockConfig).shardActor(mockShardLeaderActor)
1470                         .cluster(new ClusterWrapperImpl(system2)).props()
1471                         .withDispatcher(Dispatchers.DefaultDispatcherId()),
1472                 shardManagerID);
1473
1474         new JavaTestKit(system1) {
1475             {
1476                 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1477                 leaderShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1478
1479                 leaderShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1480
1481                 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
1482                 leaderShardManager.tell(
1483                         new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
1484                         mockShardLeaderActor);
1485                 leaderShardManager.tell(
1486                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
1487                         mockShardLeaderActor);
1488
1489                 newReplicaShardManager.underlyingActor().waitForMemberUp();
1490                 leaderShardManager.underlyingActor().waitForMemberUp();
1491
1492                 // Have a dummy snapshot to be overwritten by the new data
1493                 // persisted.
1494                 String[] restoredShards = { "default", "people" };
1495                 ShardManagerSnapshot snapshot =
1496                         new ShardManagerSnapshot(Arrays.asList(restoredShards), Collections.emptyMap());
1497                 InMemorySnapshotStore.addSnapshot(shardManagerID, snapshot);
1498                 Uninterruptibles.sleepUninterruptibly(2, TimeUnit.MILLISECONDS);
1499
1500                 InMemorySnapshotStore.addSnapshotSavedLatch(shardManagerID);
1501                 InMemorySnapshotStore.addSnapshotDeletedLatch(shardManagerID);
1502
1503                 // construct a mock response message
1504                 newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
1505                 AddServer addServerMsg = MessageCollectorActor.expectFirstMatching(mockShardLeaderActor,
1506                         AddServer.class);
1507                 String addServerId = "member-1-shard-astronauts-" + shardMrgIDSuffix;
1508                 assertEquals("AddServer serverId", addServerId, addServerMsg.getNewServerId());
1509                 expectMsgClass(duration("5 seconds"), Status.Success.class);
1510
1511                 InMemorySnapshotStore.waitForSavedSnapshot(shardManagerID, ShardManagerSnapshot.class);
1512                 InMemorySnapshotStore.waitForDeletedSnapshot(shardManagerID);
1513                 List<ShardManagerSnapshot> persistedSnapshots = InMemorySnapshotStore.getSnapshots(shardManagerID,
1514                         ShardManagerSnapshot.class);
1515                 assertEquals("Number of snapshots persisted", 1, persistedSnapshots.size());
1516                 ShardManagerSnapshot shardManagerSnapshot = persistedSnapshots.get(0);
1517                 assertEquals("Persisted local shards", Sets.newHashSet("default", "astronauts"),
1518                         Sets.newHashSet(shardManagerSnapshot.getShardList()));
1519             }
1520         };
1521         LOG.info("testAddShardReplica ending");
1522     }
1523
1524     @Test
1525     public void testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader() throws Exception {
1526         LOG.info("testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader starting");
1527         new JavaTestKit(getSystem()) {
1528             {
1529                 TestActorRef<TestShardManager> shardManager = actorFactory
1530                         .createTestActor(newPropsShardMgrWithMockShardActor(), shardMgrID);
1531
1532                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1533                 shardManager.tell(new ActorInitialized(), mockShardActor);
1534
1535                 String leaderId = "leader-member-shard-default-" + shardMrgIDSuffix;
1536                 AddServerReply addServerReply = new AddServerReply(ServerChangeStatus.ALREADY_EXISTS, null);
1537                 ActorRef leaderShardActor = shardManager.underlyingActor().getContext()
1538                         .actorOf(Props.create(MockRespondActor.class, AddServer.class, addServerReply), leaderId);
1539
1540                 MockClusterWrapper.sendMemberUp(shardManager, "leader-member", leaderShardActor.path().toString());
1541
1542                 String newReplicaId = "member-1-shard-default-" + shardMrgIDSuffix;
1543                 shardManager.tell(
1544                         new RoleChangeNotification(newReplicaId, RaftState.Candidate.name(), RaftState.Follower.name()),
1545                         mockShardActor);
1546                 shardManager.tell(
1547                         new ShardLeaderStateChanged(newReplicaId, leaderId, DataStoreVersions.CURRENT_VERSION),
1548                         mockShardActor);
1549
1550                 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1551
1552                 MessageCollectorActor.expectFirstMatching(leaderShardActor, AddServer.class);
1553
1554                 Failure resp = expectMsgClass(duration("5 seconds"), Failure.class);
1555                 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1556
1557                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1558                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1559
1560                 // Send message again to verify previous in progress state is
1561                 // cleared
1562
1563                 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1564                 resp = expectMsgClass(duration("5 seconds"), Failure.class);
1565                 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1566
1567                 // Send message again with an AddServer timeout to verify the
1568                 // pre-existing shard actor isn't terminated.
1569
1570                 shardManager.tell(
1571                         newDatastoreContextFactory(
1572                                 datastoreContextBuilder.shardLeaderElectionTimeout(100, TimeUnit.MILLISECONDS).build()),
1573                         getRef());
1574                 leaderShardActor.tell(MockRespondActor.CLEAR_RESPONSE, ActorRef.noSender());
1575                 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1576                 expectMsgClass(duration("5 seconds"), Failure.class);
1577
1578                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1579                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1580             }
1581         };
1582
1583         LOG.info("testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader ending");
1584     }
1585
1586     @Test
1587     public void testAddShardReplicaWithPreExistingLocalReplicaLeader() throws Exception {
1588         LOG.info("testAddShardReplicaWithPreExistingLocalReplicaLeader starting");
1589         new JavaTestKit(getSystem()) {
1590             {
1591                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1592                 ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1593
1594                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1595                 shardManager.tell(new ActorInitialized(), mockShardActor);
1596                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
1597                         DataStoreVersions.CURRENT_VERSION), getRef());
1598                 shardManager.tell(
1599                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
1600                         mockShardActor);
1601
1602                 shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1603                 Failure resp = expectMsgClass(duration("5 seconds"), Failure.class);
1604                 assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1605
1606                 shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1607                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1608             }
1609         };
1610
1611         LOG.info("testAddShardReplicaWithPreExistingLocalReplicaLeader ending");
1612     }
1613
1614     @Test
1615     public void testAddShardReplicaWithAddServerReplyFailure() throws Exception {
1616         LOG.info("testAddShardReplicaWithAddServerReplyFailure starting");
1617         new JavaTestKit(getSystem()) {
1618             {
1619                 JavaTestKit mockShardLeaderKit = new JavaTestKit(getSystem());
1620
1621                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1622                         .put("astronauts", Arrays.asList("member-2")).build());
1623
1624                 ActorRef mockNewReplicaShardActor = newMockShardActor(getSystem(), "astronauts", "member-1");
1625                 final TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(
1626                         newTestShardMgrBuilder(mockConfig).shardActor(mockNewReplicaShardActor).props(), shardMgrID);
1627                 shardManager.underlyingActor()
1628                         .setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1629
1630                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1631
1632                 JavaTestKit terminateWatcher = new JavaTestKit(getSystem());
1633                 terminateWatcher.watch(mockNewReplicaShardActor);
1634
1635                 shardManager.tell(new AddShardReplica("astronauts"), getRef());
1636
1637                 AddServer addServerMsg = mockShardLeaderKit.expectMsgClass(AddServer.class);
1638                 assertEquals("AddServer serverId", "member-1-shard-astronauts-" + shardMrgIDSuffix,
1639                         addServerMsg.getNewServerId());
1640                 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.TIMEOUT, null));
1641
1642                 Failure failure = expectMsgClass(duration("5 seconds"), Failure.class);
1643                 assertEquals("Failure cause", TimeoutException.class, failure.cause().getClass());
1644
1645                 shardManager.tell(new FindLocalShard("astronauts", false), getRef());
1646                 expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
1647
1648                 terminateWatcher.expectTerminated(mockNewReplicaShardActor);
1649
1650                 shardManager.tell(new AddShardReplica("astronauts"), getRef());
1651                 mockShardLeaderKit.expectMsgClass(AddServer.class);
1652                 mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.NO_LEADER, null));
1653                 failure = expectMsgClass(duration("5 seconds"), Failure.class);
1654                 assertEquals("Failure cause", NoShardLeaderException.class, failure.cause().getClass());
1655             }
1656         };
1657
1658         LOG.info("testAddShardReplicaWithAddServerReplyFailure ending");
1659     }
1660
1661     @Test
1662     public void testAddShardReplicaWithAlreadyInProgress() throws Exception {
1663         testServerChangeWhenAlreadyInProgress("astronauts", new AddShardReplica("astronauts"),
1664                 AddServer.class, new AddShardReplica("astronauts"));
1665     }
1666
1667     @Test
1668     public void testAddShardReplicaWithFindPrimaryTimeout() throws Exception {
1669         LOG.info("testAddShardReplicaWithFindPrimaryTimeout starting");
1670         datastoreContextBuilder.shardInitializationTimeout(100, TimeUnit.MILLISECONDS);
1671         new JavaTestKit(getSystem()) {
1672             {
1673                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1674                         .put("astronauts", Arrays.asList("member-2")).build());
1675
1676                 final ActorRef newReplicaShardManager = actorFactory
1677                         .createActor(newTestShardMgrBuilder(mockConfig).shardActor(mockShardActor).props(), shardMgrID);
1678
1679                 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1680                 MockClusterWrapper.sendMemberUp(newReplicaShardManager, "member-2",
1681                         AddressFromURIString.parse("akka://non-existent@127.0.0.1:5").toString());
1682
1683                 newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
1684                 Status.Failure resp = expectMsgClass(duration("5 seconds"), Status.Failure.class);
1685                 assertEquals("Failure obtained", true, resp.cause() instanceof RuntimeException);
1686             }
1687         };
1688
1689         LOG.info("testAddShardReplicaWithFindPrimaryTimeout ending");
1690     }
1691
1692     @Test
1693     public void testRemoveShardReplicaForNonExistentShard() throws Exception {
1694         new JavaTestKit(getSystem()) {
1695             {
1696                 ActorRef shardManager = actorFactory
1697                         .createActor(newShardMgrProps(new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1698
1699                 shardManager.tell(new RemoveShardReplica("model-inventory", MEMBER_1), getRef());
1700                 Status.Failure resp = expectMsgClass(duration("10 seconds"), Status.Failure.class);
1701                 assertEquals("Failure obtained", true, resp.cause() instanceof PrimaryNotFoundException);
1702             }
1703         };
1704     }
1705
1706     @Test
1707     /**
1708      * Primary is Local
1709      */
1710     public void testRemoveShardReplicaLocal() throws Exception {
1711         new JavaTestKit(getSystem()) {
1712             {
1713                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1714
1715                 final TestActorRef<MockRespondActor> respondActor = actorFactory
1716                         .createTestActor(Props.create(MockRespondActor.class, RemoveServer.class,
1717                                 new RemoveServerReply(ServerChangeStatus.OK, null)), memberId);
1718
1719                 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
1720
1721                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1722                 shardManager.tell(new ActorInitialized(), respondActor);
1723                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
1724                         DataStoreVersions.CURRENT_VERSION), getRef());
1725                 shardManager.tell(
1726                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
1727                         respondActor);
1728
1729                 shardManager.tell(new RemoveShardReplica(Shard.DEFAULT_NAME, MEMBER_1), getRef());
1730                 final RemoveServer removeServer = MessageCollectorActor.expectFirstMatching(respondActor,
1731                         RemoveServer.class);
1732                 assertEquals(ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString(),
1733                         removeServer.getServerId());
1734                 expectMsgClass(duration("5 seconds"), Success.class);
1735             }
1736         };
1737     }
1738
1739     @Test
1740     public void testRemoveShardReplicaRemote() throws Exception {
1741         MockConfiguration mockConfig = new MockConfiguration(
1742                 ImmutableMap.<String, List<String>>builder().put("default", Arrays.asList("member-1", "member-2"))
1743                         .put("astronauts", Arrays.asList("member-1")).build());
1744
1745         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
1746
1747         // Create an ActorSystem ShardManager actor for member-1.
1748         final ActorSystem system1 = newActorSystem("Member1");
1749         Cluster.get(system1).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1750         ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
1751
1752         final TestActorRef<TestShardManager> newReplicaShardManager = TestActorRef.create(system1,
1753                 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockDefaultShardActor).cluster(
1754                         new ClusterWrapperImpl(system1)).props().withDispatcher(Dispatchers.DefaultDispatcherId()),
1755                 shardManagerID);
1756
1757         // Create an ActorSystem ShardManager actor for member-2.
1758         final ActorSystem system2 = newActorSystem("Member2");
1759         Cluster.get(system2).join(AddressFromURIString.parse("akka://cluster-test@127.0.0.1:2558"));
1760
1761         String name = ShardIdentifier.create("default", MEMBER_2, shardMrgIDSuffix).toString();
1762         String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
1763         final TestActorRef<MockRespondActor> mockShardLeaderActor =
1764                 TestActorRef.create(system2, Props.create(MockRespondActor.class, RemoveServer.class,
1765                         new RemoveServerReply(ServerChangeStatus.OK, memberId2)), name);
1766
1767         LOG.error("Mock Shard Leader Actor : {}", mockShardLeaderActor);
1768
1769         final TestActorRef<TestShardManager> leaderShardManager = TestActorRef.create(system2,
1770                 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockShardLeaderActor).cluster(
1771                         new ClusterWrapperImpl(system2)).props().withDispatcher(Dispatchers.DefaultDispatcherId()),
1772                 shardManagerID);
1773
1774         // Because mockShardLeaderActor is created at the top level of the actor system it has an address like so,
1775         //    akka://cluster-test@127.0.0.1:2559/user/member-2-shard-default-config1
1776         // However when a shard manager has a local shard which is a follower and a leader that is remote it will
1777         // try to compute an address for the remote shard leader using the ShardPeerAddressResolver. This address will
1778         // look like so,
1779         //    akka://cluster-test@127.0.0.1:2559/user/shardmanager-config1/member-2-shard-default-config1
1780         // In this specific case if we did a FindPrimary for shard default from member-1 we would come up
1781         // with the address of an actor which does not exist, therefore any message sent to that actor would go to
1782         // dead letters.
1783         // To work around this problem we create a ForwardingActor with the right address and pass to it the
1784         // mockShardLeaderActor. The ForwardingActor simply forwards all messages to the mockShardLeaderActor and every
1785         // thing works as expected
1786         final ActorRef actorRef = leaderShardManager.underlyingActor().context()
1787                 .actorOf(Props.create(ForwardingActor.class, mockShardLeaderActor),
1788                         "member-2-shard-default-" + shardMrgIDSuffix);
1789
1790         LOG.error("Forwarding actor : {}", actorRef);
1791
1792         new JavaTestKit(system1) {
1793             {
1794                 newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1795                 leaderShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1796
1797                 leaderShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1798                 newReplicaShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1799
1800                 short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
1801                 leaderShardManager.tell(
1802                         new ShardLeaderStateChanged(memberId2, memberId2, mock(DataTree.class), leaderVersion),
1803                         mockShardLeaderActor);
1804                 leaderShardManager.tell(
1805                         new RoleChangeNotification(memberId2, RaftState.Candidate.name(), RaftState.Leader.name()),
1806                         mockShardLeaderActor);
1807
1808                 String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
1809                 newReplicaShardManager.tell(
1810                         new ShardLeaderStateChanged(memberId1, memberId2, mock(DataTree.class), leaderVersion),
1811                         mockShardActor);
1812                 newReplicaShardManager.tell(
1813                         new RoleChangeNotification(memberId1, RaftState.Candidate.name(), RaftState.Follower.name()),
1814                         mockShardActor);
1815
1816                 newReplicaShardManager.underlyingActor().waitForMemberUp();
1817                 leaderShardManager.underlyingActor().waitForMemberUp();
1818
1819                 // construct a mock response message
1820                 newReplicaShardManager.tell(new RemoveShardReplica("default", MEMBER_1), getRef());
1821                 RemoveServer removeServer = MessageCollectorActor.expectFirstMatching(mockShardLeaderActor,
1822                         RemoveServer.class);
1823                 String removeServerId = ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString();
1824                 assertEquals("RemoveServer serverId", removeServerId, removeServer.getServerId());
1825                 expectMsgClass(duration("5 seconds"), Status.Success.class);
1826             }
1827         };
1828     }
1829
1830     @Test
1831     public void testRemoveShardReplicaWhenAnotherRemoveShardReplicaAlreadyInProgress() throws Exception {
1832         testServerChangeWhenAlreadyInProgress("astronauts", new RemoveShardReplica("astronauts", MEMBER_2),
1833                 RemoveServer.class, new RemoveShardReplica("astronauts", MEMBER_3));
1834     }
1835
1836     @Test
1837     public void testRemoveShardReplicaWhenAddShardReplicaAlreadyInProgress() throws Exception {
1838         testServerChangeWhenAlreadyInProgress("astronauts", new AddShardReplica("astronauts"),
1839                 AddServer.class, new RemoveShardReplica("astronauts", MEMBER_2));
1840     }
1841
1842
1843     public void testServerChangeWhenAlreadyInProgress(final String shardName, final Object firstServerChange,
1844                                                       final Class<?> firstForwardedServerChangeClass,
1845                                                       final Object secondServerChange) throws Exception {
1846         new JavaTestKit(getSystem()) {
1847             {
1848                 JavaTestKit mockShardLeaderKit = new JavaTestKit(getSystem());
1849                 final JavaTestKit secondRequestKit = new JavaTestKit(getSystem());
1850
1851                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1852                         .put(shardName, Arrays.asList("member-2")).build());
1853
1854                 final TestActorRef<TestShardManager> shardManager = TestActorRef.create(getSystem(),
1855                         newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockShardActor)
1856                                 .cluster(new MockClusterWrapper()).props()
1857                                 .withDispatcher(Dispatchers.DefaultDispatcherId()),
1858                         shardMgrID);
1859
1860                 shardManager.underlyingActor()
1861                         .setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1862
1863                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1864
1865                 shardManager.tell(firstServerChange, getRef());
1866
1867                 mockShardLeaderKit.expectMsgClass(firstForwardedServerChangeClass);
1868
1869                 shardManager.tell(secondServerChange, secondRequestKit.getRef());
1870
1871                 secondRequestKit.expectMsgClass(duration("5 seconds"), Failure.class);
1872             }
1873         };
1874     }
1875
1876     @Test
1877     public void testServerRemovedShardActorNotRunning() throws Exception {
1878         LOG.info("testServerRemovedShardActorNotRunning starting");
1879         new JavaTestKit(getSystem()) {
1880             {
1881                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1882                         .put("default", Arrays.asList("member-1", "member-2"))
1883                         .put("astronauts", Arrays.asList("member-2"))
1884                         .put("people", Arrays.asList("member-1", "member-2")).build());
1885
1886                 TestActorRef<TestShardManager> shardManager = actorFactory
1887                         .createTestActor(newShardMgrProps(mockConfig));
1888
1889                 shardManager.underlyingActor().waitForRecoveryComplete();
1890                 shardManager.tell(new FindLocalShard("people", false), getRef());
1891                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1892
1893                 shardManager.tell(new FindLocalShard("default", false), getRef());
1894                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1895
1896                 // Removed the default shard replica from member-1
1897                 ShardIdentifier.Builder builder = new ShardIdentifier.Builder();
1898                 ShardIdentifier shardId = builder.shardName("default").memberName(MEMBER_1).type(shardMrgIDSuffix)
1899                         .build();
1900                 shardManager.tell(new ServerRemoved(shardId.toString()), getRef());
1901
1902                 shardManager.underlyingActor().verifySnapshotPersisted(Sets.newHashSet("people"));
1903             }
1904         };
1905
1906         LOG.info("testServerRemovedShardActorNotRunning ending");
1907     }
1908
1909     @Test
1910     public void testServerRemovedShardActorRunning() throws Exception {
1911         LOG.info("testServerRemovedShardActorRunning starting");
1912         new JavaTestKit(getSystem()) {
1913             {
1914                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1915                         .put("default", Arrays.asList("member-1", "member-2"))
1916                         .put("astronauts", Arrays.asList("member-2"))
1917                         .put("people", Arrays.asList("member-1", "member-2")).build());
1918
1919                 String shardId = ShardIdentifier.create("default", MEMBER_1, shardMrgIDSuffix).toString();
1920                 TestActorRef<MessageCollectorActor> shard = actorFactory.createTestActor(MessageCollectorActor.props(),
1921                         shardId);
1922
1923                 TestActorRef<TestShardManager> shardManager = actorFactory
1924                         .createTestActor(newTestShardMgrBuilder(mockConfig).addShardActor("default", shard).props());
1925
1926                 shardManager.underlyingActor().waitForRecoveryComplete();
1927
1928                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1929                 shardManager.tell(new ActorInitialized(), shard);
1930
1931                 waitForShardInitialized(shardManager, "people", this);
1932                 waitForShardInitialized(shardManager, "default", this);
1933
1934                 // Removed the default shard replica from member-1
1935                 shardManager.tell(new ServerRemoved(shardId), getRef());
1936
1937                 shardManager.underlyingActor().verifySnapshotPersisted(Sets.newHashSet("people"));
1938
1939                 MessageCollectorActor.expectFirstMatching(shard, Shutdown.class);
1940             }
1941         };
1942
1943         LOG.info("testServerRemovedShardActorRunning ending");
1944     }
1945
1946     @Test
1947     public void testShardPersistenceWithRestoredData() throws Exception {
1948         LOG.info("testShardPersistenceWithRestoredData starting");
1949         new JavaTestKit(getSystem()) {
1950             {
1951                 MockConfiguration mockConfig =
1952                     new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1953                             .put("default", Arrays.asList("member-1", "member-2"))
1954                             .put("astronauts", Arrays.asList("member-2"))
1955                             .put("people", Arrays.asList("member-1", "member-2")).build());
1956                 String[] restoredShards = {"default", "astronauts"};
1957                 ShardManagerSnapshot snapshot =
1958                         new ShardManagerSnapshot(Arrays.asList(restoredShards), Collections.emptyMap());
1959                 InMemorySnapshotStore.addSnapshot("shard-manager-" + shardMrgIDSuffix, snapshot);
1960
1961                 // create shardManager to come up with restored data
1962                 TestActorRef<TestShardManager> newRestoredShardManager = actorFactory
1963                         .createTestActor(newShardMgrProps(mockConfig));
1964
1965                 newRestoredShardManager.underlyingActor().waitForRecoveryComplete();
1966
1967                 newRestoredShardManager.tell(new FindLocalShard("people", false), getRef());
1968                 LocalShardNotFound notFound = expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
1969                 assertEquals("for uninitialized shard", "people", notFound.getShardName());
1970
1971                 // Verify a local shard is created for the restored shards,
1972                 // although we expect a NotInitializedException for the shards
1973                 // as the actor initialization
1974                 // message is not sent for them
1975                 newRestoredShardManager.tell(new FindLocalShard("default", false), getRef());
1976                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1977
1978                 newRestoredShardManager.tell(new FindLocalShard("astronauts", false), getRef());
1979                 expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1980             }
1981         };
1982
1983         LOG.info("testShardPersistenceWithRestoredData ending");
1984     }
1985
1986     @Test
1987     public void testShutDown() throws Exception {
1988         LOG.info("testShutDown starting");
1989         new JavaTestKit(getSystem()) {
1990             {
1991                 MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder()
1992                         .put("shard1", Arrays.asList("member-1")).put("shard2", Arrays.asList("member-1")).build());
1993
1994                 String shardId1 = ShardIdentifier.create("shard1", MEMBER_1, shardMrgIDSuffix).toString();
1995                 TestActorRef<MessageCollectorActor> shard1 = actorFactory.createTestActor(
1996                         MessageCollectorActor.props().withDispatcher(Dispatchers.DefaultDispatcherId()), shardId1);
1997
1998                 String shardId2 = ShardIdentifier.create("shard2", MEMBER_1, shardMrgIDSuffix).toString();
1999                 TestActorRef<MessageCollectorActor> shard2 = actorFactory.createTestActor(
2000                         MessageCollectorActor.props().withDispatcher(Dispatchers.DefaultDispatcherId()), shardId2);
2001
2002                 TestActorRef<TestShardManager> shardManager = actorFactory
2003                         .createTestActor(newTestShardMgrBuilder(mockConfig).addShardActor("shard1", shard1)
2004                                 .addShardActor("shard2", shard2).props()
2005                                 .withDispatcher(Dispatchers.DefaultDispatcherId()));
2006
2007                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
2008                 shardManager.tell(new ActorInitialized(), shard1);
2009                 shardManager.tell(new ActorInitialized(), shard2);
2010
2011                 FiniteDuration duration = FiniteDuration.create(5, TimeUnit.SECONDS);
2012                 Future<Boolean> stopFuture = Patterns.gracefulStop(shardManager, duration, Shutdown.INSTANCE);
2013
2014                 MessageCollectorActor.expectFirstMatching(shard1, Shutdown.class);
2015                 MessageCollectorActor.expectFirstMatching(shard2, Shutdown.class);
2016
2017                 try {
2018                     Await.ready(stopFuture, FiniteDuration.create(500, TimeUnit.MILLISECONDS));
2019                     fail("ShardManager actor stopped without waiting for the Shards to be stopped");
2020                 } catch (TimeoutException e) {
2021                     // expected
2022                 }
2023
2024                 actorFactory.killActor(shard1, this);
2025                 actorFactory.killActor(shard2, this);
2026
2027                 Boolean stopped = Await.result(stopFuture, duration);
2028                 assertEquals("Stopped", Boolean.TRUE, stopped);
2029             }
2030         };
2031
2032         LOG.info("testShutDown ending");
2033     }
2034
2035     @Test
2036     public void testChangeServersVotingStatus() throws Exception {
2037         new JavaTestKit(getSystem()) {
2038             {
2039                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
2040
2041                 TestActorRef<MockRespondActor> respondActor = actorFactory
2042                         .createTestActor(Props.create(MockRespondActor.class, ChangeServersVotingStatus.class,
2043                                 new ServerChangeReply(ServerChangeStatus.OK, null)), memberId);
2044
2045                 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
2046
2047                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
2048                 shardManager.tell(new ActorInitialized(), respondActor);
2049                 shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, mock(DataTree.class),
2050                         DataStoreVersions.CURRENT_VERSION), getRef());
2051                 shardManager.tell(
2052                         new RoleChangeNotification(memberId, RaftState.Candidate.name(), RaftState.Leader.name()),
2053                         respondActor);
2054
2055                 shardManager.tell(
2056                         new ChangeShardMembersVotingStatus("default", ImmutableMap.of("member-2", Boolean.TRUE)),
2057                         getRef());
2058
2059                 ChangeServersVotingStatus actualChangeStatusMsg = MessageCollectorActor
2060                         .expectFirstMatching(respondActor, ChangeServersVotingStatus.class);
2061                 assertEquals("ChangeServersVotingStatus map", actualChangeStatusMsg.getServerVotingStatusMap(),
2062                         ImmutableMap.of(ShardIdentifier
2063                                 .create("default", MemberName.forName("member-2"), shardMrgIDSuffix).toString(),
2064                                 Boolean.TRUE));
2065
2066                 expectMsgClass(duration("5 seconds"), Success.class);
2067             }
2068         };
2069     }
2070
2071     @Test
2072     public void testChangeServersVotingStatusWithNoLeader() throws Exception {
2073         new JavaTestKit(getSystem()) {
2074             {
2075                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
2076
2077                 TestActorRef<MockRespondActor> respondActor = actorFactory
2078                         .createTestActor(Props.create(MockRespondActor.class, ChangeServersVotingStatus.class,
2079                                 new ServerChangeReply(ServerChangeStatus.NO_LEADER, null)), memberId);
2080
2081                 ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
2082
2083                 shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
2084                 shardManager.tell(new ActorInitialized(), respondActor);
2085                 shardManager.tell(new RoleChangeNotification(memberId, null, RaftState.Follower.name()), respondActor);
2086
2087                 shardManager.tell(
2088                         new ChangeShardMembersVotingStatus("default", ImmutableMap.of("member-2", Boolean.TRUE)),
2089                         getRef());
2090
2091                 MessageCollectorActor.expectFirstMatching(respondActor, ChangeServersVotingStatus.class);
2092
2093                 Status.Failure resp = expectMsgClass(duration("5 seconds"), Status.Failure.class);
2094                 assertEquals("Failure resposnse", true, resp.cause() instanceof NoShardLeaderException);
2095             }
2096         };
2097     }
2098
2099     public static class TestShardManager extends ShardManager {
2100         private final CountDownLatch recoveryComplete = new CountDownLatch(1);
2101         private final CountDownLatch snapshotPersist = new CountDownLatch(1);
2102         private ShardManagerSnapshot snapshot;
2103         private final Map<String, ActorRef> shardActors;
2104         private final ActorRef shardActor;
2105         private CountDownLatch findPrimaryMessageReceived = new CountDownLatch(1);
2106         private CountDownLatch memberUpReceived = new CountDownLatch(1);
2107         private CountDownLatch memberRemovedReceived = new CountDownLatch(1);
2108         private CountDownLatch memberUnreachableReceived = new CountDownLatch(1);
2109         private CountDownLatch memberReachableReceived = new CountDownLatch(1);
2110         private volatile MessageInterceptor messageInterceptor;
2111
2112         private TestShardManager(Builder builder) {
2113             super(builder);
2114             shardActor = builder.shardActor;
2115             shardActors = builder.shardActors;
2116         }
2117
2118         @Override
2119         protected void handleRecover(Object message) throws Exception {
2120             try {
2121                 super.handleRecover(message);
2122             } finally {
2123                 if (message instanceof RecoveryCompleted) {
2124                     recoveryComplete.countDown();
2125                 }
2126             }
2127         }
2128
2129         private void countDownIfOther(final Member member, CountDownLatch latch) {
2130             if (!getCluster().getCurrentMemberName().equals(memberToName(member))) {
2131                 latch.countDown();
2132             }
2133         }
2134
2135         @Override
2136         public void handleCommand(Object message) throws Exception {
2137             try {
2138                 if (messageInterceptor != null && messageInterceptor.canIntercept(message)) {
2139                     getSender().tell(messageInterceptor.apply(message), getSelf());
2140                 } else {
2141                     super.handleCommand(message);
2142                 }
2143             } finally {
2144                 if (message instanceof FindPrimary) {
2145                     findPrimaryMessageReceived.countDown();
2146                 } else if (message instanceof ClusterEvent.MemberUp) {
2147                     countDownIfOther(((ClusterEvent.MemberUp) message).member(), memberUpReceived);
2148                 } else if (message instanceof ClusterEvent.MemberRemoved) {
2149                     countDownIfOther(((ClusterEvent.MemberRemoved) message).member(), memberRemovedReceived);
2150                 } else if (message instanceof ClusterEvent.UnreachableMember) {
2151                     countDownIfOther(((ClusterEvent.UnreachableMember) message).member(), memberUnreachableReceived);
2152                 } else if (message instanceof ClusterEvent.ReachableMember) {
2153                     countDownIfOther(((ClusterEvent.ReachableMember) message).member(), memberReachableReceived);
2154                 }
2155             }
2156         }
2157
2158         void setMessageInterceptor(MessageInterceptor messageInterceptor) {
2159             this.messageInterceptor = messageInterceptor;
2160         }
2161
2162         void waitForRecoveryComplete() {
2163             assertEquals("Recovery complete", true,
2164                     Uninterruptibles.awaitUninterruptibly(recoveryComplete, 5, TimeUnit.SECONDS));
2165         }
2166
2167         public void waitForMemberUp() {
2168             assertEquals("MemberUp received", true,
2169                     Uninterruptibles.awaitUninterruptibly(memberUpReceived, 5, TimeUnit.SECONDS));
2170             memberUpReceived = new CountDownLatch(1);
2171         }
2172
2173         void waitForMemberRemoved() {
2174             assertEquals("MemberRemoved received", true,
2175                     Uninterruptibles.awaitUninterruptibly(memberRemovedReceived, 5, TimeUnit.SECONDS));
2176             memberRemovedReceived = new CountDownLatch(1);
2177         }
2178
2179         void waitForUnreachableMember() {
2180             assertEquals("UnreachableMember received", true,
2181                 Uninterruptibles.awaitUninterruptibly(memberUnreachableReceived, 5, TimeUnit.SECONDS
2182                 ));
2183             memberUnreachableReceived = new CountDownLatch(1);
2184         }
2185
2186         void waitForReachableMember() {
2187             assertEquals("ReachableMember received", true,
2188                 Uninterruptibles.awaitUninterruptibly(memberReachableReceived, 5, TimeUnit.SECONDS));
2189             memberReachableReceived = new CountDownLatch(1);
2190         }
2191
2192         void verifyFindPrimary() {
2193             assertEquals("FindPrimary received", true,
2194                     Uninterruptibles.awaitUninterruptibly(findPrimaryMessageReceived, 5, TimeUnit.SECONDS));
2195             findPrimaryMessageReceived = new CountDownLatch(1);
2196         }
2197
2198         public static Builder builder(DatastoreContext.Builder datastoreContextBuilder) {
2199             return new Builder(datastoreContextBuilder);
2200         }
2201
2202         public static class Builder extends AbstractGenericCreator<Builder, TestShardManager> {
2203             private ActorRef shardActor;
2204             private final Map<String, ActorRef> shardActors = new HashMap<>();
2205
2206             Builder(DatastoreContext.Builder datastoreContextBuilder) {
2207                 super(TestShardManager.class);
2208                 datastoreContextFactory(newDatastoreContextFactory(datastoreContextBuilder.build()));
2209             }
2210
2211             Builder shardActor(ActorRef newShardActor) {
2212                 this.shardActor = newShardActor;
2213                 return this;
2214             }
2215
2216             Builder addShardActor(String shardName, ActorRef actorRef) {
2217                 shardActors.put(shardName, actorRef);
2218                 return this;
2219             }
2220         }
2221
2222         @Override
2223         public void saveSnapshot(Object obj) {
2224             snapshot = (ShardManagerSnapshot) obj;
2225             snapshotPersist.countDown();
2226             super.saveSnapshot(obj);
2227         }
2228
2229         void verifySnapshotPersisted(Set<String> shardList) {
2230             assertEquals("saveSnapshot invoked", true,
2231                     Uninterruptibles.awaitUninterruptibly(snapshotPersist, 5, TimeUnit.SECONDS));
2232             assertEquals("Shard Persisted", shardList, Sets.newHashSet(snapshot.getShardList()));
2233         }
2234
2235         @Override
2236         protected ActorRef newShardActor(SchemaContext schemaContext, ShardInformation info) {
2237             if (shardActors.get(info.getShardName()) != null) {
2238                 return shardActors.get(info.getShardName());
2239             }
2240
2241             if (shardActor != null) {
2242                 return shardActor;
2243             }
2244
2245             return super.newShardActor(schemaContext, info);
2246         }
2247     }
2248
2249     private abstract static class AbstractGenericCreator<T extends AbstractGenericCreator<T, ?>, C extends ShardManager>
2250                                                      extends AbstractShardManagerCreator<T> {
2251         private final Class<C> shardManagerClass;
2252
2253         AbstractGenericCreator(Class<C> shardManagerClass) {
2254             this.shardManagerClass = shardManagerClass;
2255             cluster(new MockClusterWrapper()).configuration(new MockConfiguration()).waitTillReadyCountDownLatch(ready)
2256                     .primaryShardInfoCache(new PrimaryShardInfoFutureCache());
2257         }
2258
2259         @Override
2260         public Props props() {
2261             verify();
2262             return Props.create(shardManagerClass, this);
2263         }
2264     }
2265
2266     private static class GenericCreator<C extends ShardManager> extends AbstractGenericCreator<GenericCreator<C>, C> {
2267         GenericCreator(Class<C> shardManagerClass) {
2268             super(shardManagerClass);
2269         }
2270     }
2271
2272     private static class DelegatingShardManagerCreator implements Creator<ShardManager> {
2273         private static final long serialVersionUID = 1L;
2274         private final Creator<ShardManager> delegate;
2275
2276         DelegatingShardManagerCreator(Creator<ShardManager> delegate) {
2277             this.delegate = delegate;
2278         }
2279
2280         @Override
2281         public ShardManager create() throws Exception {
2282             return delegate.create();
2283         }
2284     }
2285
2286     interface MessageInterceptor extends Function<Object, Object> {
2287         boolean canIntercept(Object message);
2288     }
2289
2290     private static MessageInterceptor newFindPrimaryInterceptor(final ActorRef primaryActor) {
2291         return new MessageInterceptor() {
2292             @Override
2293             public Object apply(Object message) {
2294                 return new RemotePrimaryShardFound(Serialization.serializedActorPath(primaryActor), (short) 1);
2295             }
2296
2297             @Override
2298             public boolean canIntercept(Object message) {
2299                 return message instanceof FindPrimary;
2300             }
2301         };
2302     }
2303
2304     private static class MockRespondActor extends MessageCollectorActor {
2305         static final String CLEAR_RESPONSE = "clear-response";
2306
2307         private Object responseMsg;
2308         private final Class<?> requestClass;
2309
2310         @SuppressWarnings("unused")
2311         MockRespondActor(Class<?> requestClass, Object responseMsg) {
2312             this.requestClass = requestClass;
2313             this.responseMsg = responseMsg;
2314         }
2315
2316         @Override
2317         public void onReceive(Object message) throws Exception {
2318             if (message.equals(CLEAR_RESPONSE)) {
2319                 responseMsg = null;
2320             } else {
2321                 super.onReceive(message);
2322                 if (message.getClass().equals(requestClass) && responseMsg != null) {
2323                     getSender().tell(responseMsg, getSelf());
2324                 }
2325             }
2326         }
2327     }
2328 }