opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/ShardManagerTest.java

   1 /*
   2  * Copyright (c) 2014, 2015 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.controller.cluster.datastore;
  10
  11 import static org.junit.Assert.assertEquals;
  12 import static org.junit.Assert.assertNotNull;
  13 import static org.junit.Assert.assertNull;
  14 import static org.junit.Assert.assertSame;
  15 import static org.junit.Assert.assertTrue;
  16 import static org.junit.Assert.fail;
  17 import static org.mockito.Mockito.mock;
  18 import static org.mockito.Mockito.never;
  19 import static org.mockito.Mockito.times;
  20 import static org.mockito.Mockito.verify;
  21 import akka.actor.ActorRef;
  22 import akka.actor.ActorSystem;
  23 import akka.actor.AddressFromURIString;
  24 import akka.actor.Props;
  25 import akka.actor.Status;
  26 import akka.actor.Status.Failure;
  27 import akka.actor.Status.Success;
  28 import akka.actor.Terminated;
  29 import akka.cluster.Cluster;
  30 import akka.cluster.ClusterEvent;
  31 import akka.dispatch.Dispatchers;
  32 import akka.japi.Creator;
  33 import akka.pattern.Patterns;
  34 import akka.persistence.RecoveryCompleted;
  35 import akka.serialization.Serialization;
  36 import akka.testkit.JavaTestKit;
  37 import akka.testkit.TestActorRef;
  38 import akka.util.Timeout;
  39 import com.google.common.base.Function;
  40 import com.google.common.base.Optional;
  41 import com.google.common.base.Stopwatch;
  42 import com.google.common.collect.ImmutableMap;
  43 import com.google.common.collect.ImmutableSet;
  44 import com.google.common.collect.Lists;
  45 import com.google.common.collect.Sets;
  46 import com.google.common.util.concurrent.Uninterruptibles;
  47 import com.typesafe.config.ConfigFactory;
  48 import java.net.URI;
  49 import java.util.AbstractMap;
  50 import java.util.ArrayList;
  51 import java.util.Arrays;
  52 import java.util.Collection;
  53 import java.util.Collections;
  54 import java.util.HashMap;
  55 import java.util.List;
  56 import java.util.Map;
  57 import java.util.Map.Entry;
  58 import java.util.Set;
  59 import java.util.concurrent.CountDownLatch;
  60 import java.util.concurrent.TimeUnit;
  61 import java.util.concurrent.TimeoutException;
  62 import org.apache.commons.lang3.SerializationUtils;
  63 import org.junit.After;
  64 import org.junit.Before;
  65 import org.junit.Test;
  66 import org.mockito.Mock;
  67 import org.mockito.Mockito;
  68 import org.mockito.MockitoAnnotations;
  69 import org.opendaylight.controller.cluster.datastore.config.Configuration;
  70 import org.opendaylight.controller.cluster.datastore.config.ConfigurationImpl;
  71 import org.opendaylight.controller.cluster.datastore.config.EmptyModuleShardConfigProvider;
  72 import org.opendaylight.controller.cluster.datastore.config.ModuleShardConfiguration;
  73 import org.opendaylight.controller.cluster.datastore.exceptions.AlreadyExistsException;
  74 import org.opendaylight.controller.cluster.datastore.exceptions.NoShardLeaderException;
  75 import org.opendaylight.controller.cluster.datastore.exceptions.NotInitializedException;
  76 import org.opendaylight.controller.cluster.datastore.exceptions.PrimaryNotFoundException;
  77 import org.opendaylight.controller.cluster.datastore.identifiers.ShardIdentifier;
  78 import org.opendaylight.controller.cluster.datastore.identifiers.ShardManagerIdentifier;
  79 import org.opendaylight.controller.cluster.datastore.messages.ActorInitialized;
  80 import org.opendaylight.controller.cluster.datastore.messages.AddShardReplica;
  81 import org.opendaylight.controller.cluster.datastore.messages.CreateShard;
  82 import org.opendaylight.controller.cluster.datastore.messages.DatastoreSnapshot;
  83 import org.opendaylight.controller.cluster.datastore.messages.DatastoreSnapshot.ShardSnapshot;
  84 import org.opendaylight.controller.cluster.datastore.messages.FindLocalShard;
  85 import org.opendaylight.controller.cluster.datastore.messages.FindPrimary;
  86 import org.opendaylight.controller.cluster.datastore.messages.LocalPrimaryShardFound;
  87 import org.opendaylight.controller.cluster.datastore.messages.LocalShardFound;
  88 import org.opendaylight.controller.cluster.datastore.messages.LocalShardNotFound;
  89 import org.opendaylight.controller.cluster.datastore.messages.PeerDown;
  90 import org.opendaylight.controller.cluster.datastore.messages.PeerUp;
  91 import org.opendaylight.controller.cluster.datastore.messages.PrimaryShardInfo;
  92 import org.opendaylight.controller.cluster.datastore.messages.RemotePrimaryShardFound;
  93 import org.opendaylight.controller.cluster.datastore.messages.RemoveShardReplica;
  94 import org.opendaylight.controller.cluster.datastore.messages.ShardLeaderStateChanged;
  95 import org.opendaylight.controller.cluster.datastore.messages.SwitchShardBehavior;
  96 import org.opendaylight.controller.cluster.datastore.messages.UpdateSchemaContext;
  97 import org.opendaylight.controller.cluster.datastore.utils.ForwardingActor;
  98 import org.opendaylight.controller.cluster.datastore.utils.MockClusterWrapper;
  99 import org.opendaylight.controller.cluster.datastore.utils.MockConfiguration;
 100 import org.opendaylight.controller.cluster.datastore.utils.PrimaryShardInfoFutureCache;
 101 import org.opendaylight.controller.cluster.notifications.LeaderStateChanged;
 102 import org.opendaylight.controller.cluster.notifications.RegisterRoleChangeListener;
 103 import org.opendaylight.controller.cluster.notifications.RoleChangeNotification;
 104 import org.opendaylight.controller.cluster.raft.RaftState;
 105 import org.opendaylight.controller.cluster.raft.TestActorFactory;
 106 import org.opendaylight.controller.cluster.raft.base.messages.FollowerInitialSyncUpStatus;
 107 import org.opendaylight.controller.cluster.raft.base.messages.SwitchBehavior;
 108 import org.opendaylight.controller.cluster.raft.client.messages.GetSnapshot;
 109 import org.opendaylight.controller.cluster.raft.client.messages.Shutdown;
 110 import org.opendaylight.controller.cluster.raft.messages.AddServer;
 111 import org.opendaylight.controller.cluster.raft.messages.AddServerReply;
 112 import org.opendaylight.controller.cluster.raft.messages.RemoveServer;
 113 import org.opendaylight.controller.cluster.raft.messages.RemoveServerReply;
 114 import org.opendaylight.controller.cluster.raft.messages.ServerChangeStatus;
 115 import org.opendaylight.controller.cluster.raft.messages.ServerRemoved;
 116 import org.opendaylight.controller.cluster.raft.policy.DisableElectionsRaftPolicy;
 117 import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal;
 118 import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore;
 119 import org.opendaylight.controller.cluster.raft.utils.MessageCollectorActor;
 120 import org.opendaylight.controller.md.cluster.datastore.model.TestModel;
 121 import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTree;
 122 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
 123 import org.slf4j.Logger;
 124 import org.slf4j.LoggerFactory;
 125 import scala.concurrent.Await;
 126 import scala.concurrent.Future;
 127 import scala.concurrent.duration.FiniteDuration;
 128
 129 public class ShardManagerTest extends AbstractActorTest {
 130     private static final Logger LOG = LoggerFactory.getLogger(ShardManagerTest.class);
 131
 132     private static int ID_COUNTER = 1;
 133
 134     private final String shardMrgIDSuffix = "config" + ID_COUNTER++;
 135     private final String shardMgrID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
 136
 137     @Mock
 138     private static CountDownLatch ready;
 139
 140     private static TestActorRef<MessageCollectorActor> mockShardActor;
 141
 142     private static String mockShardName;
 143
 144     private final DatastoreContext.Builder datastoreContextBuilder = DatastoreContext.newBuilder().
 145             dataStoreName(shardMrgIDSuffix).shardInitializationTimeout(600, TimeUnit.MILLISECONDS)
 146                    .shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(6);
 147
 148     private final Collection<ActorSystem> actorSystems = new ArrayList<>();
 149
 150     private final TestActorFactory actorFactory = new TestActorFactory(getSystem());
 151
 152     @Before
 153     public void setUp() {
 154         MockitoAnnotations.initMocks(this);
 155
 156         InMemoryJournal.clear();
 157         InMemorySnapshotStore.clear();
 158
 159         if(mockShardActor == null) {
 160             mockShardName = new ShardIdentifier(Shard.DEFAULT_NAME, "member-1", "config").toString();
 161             mockShardActor = TestActorRef.create(getSystem(), Props.create(MessageCollectorActor.class), mockShardName);
 162         }
 163
 164         mockShardActor.underlyingActor().clear();
 165     }
 166
 167     @After
 168     public void tearDown() {
 169         InMemoryJournal.clear();
 170         InMemorySnapshotStore.clear();
 171
 172         for(ActorSystem system: actorSystems) {
 173             JavaTestKit.shutdownActorSystem(system, null, Boolean.TRUE);
 174         }
 175
 176         actorFactory.close();
 177     }
 178
 179     private ActorSystem newActorSystem(String config) {
 180         ActorSystem system = ActorSystem.create("cluster-test", ConfigFactory.load().getConfig(config));
 181         actorSystems.add(system);
 182         return system;
 183     }
 184
 185     private ActorRef newMockShardActor(ActorSystem system, String shardName, String memberName) {
 186         String name = new ShardIdentifier(shardName, memberName,"config").toString();
 187         if(system == getSystem()) {
 188             return actorFactory.createTestActor(Props.create(MessageCollectorActor.class), name);
 189         }
 190
 191         return TestActorRef.create(system, Props.create(MessageCollectorActor.class), name);
 192     }
 193
 194     private Props newShardMgrProps() {
 195         return newShardMgrProps(new MockConfiguration());
 196     }
 197
 198     private static DatastoreContextFactory newDatastoreContextFactory(DatastoreContext datastoreContext) {
 199         DatastoreContextFactory mockFactory = mock(DatastoreContextFactory.class);
 200         Mockito.doReturn(datastoreContext).when(mockFactory).getBaseDatastoreContext();
 201         Mockito.doReturn(datastoreContext).when(mockFactory).getShardDatastoreContext(Mockito.anyString());
 202         return mockFactory;
 203     }
 204
 205     private TestShardManager.Builder newTestShardMgrBuilder() {
 206         return TestShardManager.builder(datastoreContextBuilder);
 207     }
 208
 209     private TestShardManager.Builder newTestShardMgrBuilder(Configuration config) {
 210         return TestShardManager.builder(datastoreContextBuilder).configuration(config);
 211     }
 212
 213     private Props newShardMgrProps(Configuration config) {
 214         return newTestShardMgrBuilder(config).props();
 215     }
 216
 217     private TestShardManager.Builder newTestShardMgrBuilderWithMockShardActor() {
 218         return newTestShardMgrBuilderWithMockShardActor(mockShardActor);
 219     }
 220
 221     private TestShardManager.Builder newTestShardMgrBuilderWithMockShardActor(ActorRef shardActor) {
 222         return TestShardManager.builder(datastoreContextBuilder).shardActor(shardActor);
 223     }
 224
 225
 226     private Props newPropsShardMgrWithMockShardActor() {
 227         return newTestShardMgrBuilderWithMockShardActor().props();
 228     }
 229
 230     private Props newPropsShardMgrWithMockShardActor(ActorRef shardActor) {
 231         return newTestShardMgrBuilderWithMockShardActor(shardActor).props();
 232     }
 233
 234
 235     private TestShardManager newTestShardManager() {
 236         return newTestShardManager(newShardMgrProps());
 237     }
 238
 239     private TestShardManager newTestShardManager(Props props) {
 240         TestActorRef<TestShardManager> shardManagerActor = actorFactory.createTestActor(props);
 241         TestShardManager shardManager = shardManagerActor.underlyingActor();
 242         shardManager.waitForRecoveryComplete();
 243         return shardManager;
 244     }
 245
 246     private void waitForShardInitialized(ActorRef shardManager, String shardName, JavaTestKit kit) {
 247         AssertionError last = null;
 248         Stopwatch sw = Stopwatch.createStarted();
 249         while(sw.elapsed(TimeUnit.SECONDS) <= 5) {
 250             try {
 251                 shardManager.tell(new FindLocalShard(shardName, true), kit.getRef());
 252                 kit.expectMsgClass(LocalShardFound.class);
 253                 return;
 254             } catch(AssertionError e) {
 255                 last = e;
 256             }
 257
 258             Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
 259         }
 260
 261         throw last;
 262     }
 263
 264     private <T> T expectMsgClassOrFailure(Class<T> msgClass, JavaTestKit kit, String msg) {
 265         Object reply = kit.expectMsgAnyClassOf(JavaTestKit.duration("5 sec"), msgClass, Failure.class);
 266         if(reply instanceof Failure) {
 267             throw new AssertionError(msg + " failed", ((Failure)reply).cause());
 268         }
 269
 270         return (T)reply;
 271     }
 272
 273     @Test
 274     public void testPerShardDatastoreContext() throws Exception {
 275         LOG.info("testPerShardDatastoreContext starting");
 276         final DatastoreContextFactory mockFactory = newDatastoreContextFactory(
 277                 datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
 278
 279         Mockito.doReturn(DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).
 280                 shardElectionTimeoutFactor(6).build()).when(mockFactory).getShardDatastoreContext("default");
 281
 282         Mockito.doReturn(DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).
 283                 shardElectionTimeoutFactor(7).build()).when(mockFactory).getShardDatastoreContext("topology");
 284
 285         final MockConfiguration mockConfig = new MockConfiguration() {
 286             @Override
 287             public Collection<String> getMemberShardNames(String memberName) {
 288                 return Arrays.asList("default", "topology");
 289             }
 290
 291             @Override
 292             public Collection<String> getMembersFromShardName(String shardName) {
 293                 return Arrays.asList("member-1");
 294             }
 295         };
 296
 297         final TestActorRef<MessageCollectorActor> defaultShardActor = actorFactory.createTestActor(
 298                 Props.create(MessageCollectorActor.class), actorFactory.generateActorId("default"));
 299         final TestActorRef<MessageCollectorActor> topologyShardActor = actorFactory.createTestActor(
 300                 Props.create(MessageCollectorActor.class), actorFactory.generateActorId("topology"));
 301
 302         final Map<String, Entry<ActorRef, DatastoreContext>> shardInfoMap = Collections.synchronizedMap(
 303                 new HashMap<String, Entry<ActorRef, DatastoreContext>>());
 304         shardInfoMap.put("default", new AbstractMap.SimpleEntry<ActorRef, DatastoreContext>(defaultShardActor, null));
 305         shardInfoMap.put("topology", new AbstractMap.SimpleEntry<ActorRef, DatastoreContext>(topologyShardActor, null));
 306
 307         final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
 308         final CountDownLatch newShardActorLatch = new CountDownLatch(2);
 309         class LocalShardManager extends ShardManager {
 310             public LocalShardManager(AbstractBuilder<?> builder) {
 311                 super(builder);
 312             }
 313
 314             @Override
 315             protected ActorRef newShardActor(SchemaContext schemaContext, ShardInformation info) {
 316                 Entry<ActorRef, DatastoreContext> entry = shardInfoMap.get(info.getShardName());
 317                 ActorRef ref = null;
 318                 if(entry != null) {
 319                     ref = entry.getKey();
 320                     entry.setValue(info.getDatastoreContext());
 321                 }
 322
 323                 newShardActorLatch.countDown();
 324                 return ref;
 325             }
 326         }
 327
 328         final Creator<ShardManager> creator = new Creator<ShardManager>() {
 329             private static final long serialVersionUID = 1L;
 330             @Override
 331             public ShardManager create() throws Exception {
 332                 return new LocalShardManager(new GenericBuilder<LocalShardManager>(LocalShardManager.class).
 333                         datastoreContextFactory(mockFactory).primaryShardInfoCache(primaryShardInfoCache).
 334                         configuration(mockConfig));
 335             }
 336         };
 337
 338         JavaTestKit kit = new JavaTestKit(getSystem());
 339
 340         final ActorRef shardManager = actorFactory.createActor(Props.create(
 341                 new DelegatingShardManagerCreator(creator)).withDispatcher(Dispatchers.DefaultDispatcherId()));
 342
 343         shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), kit.getRef());
 344
 345         assertEquals("Shard actors created", true, newShardActorLatch.await(5, TimeUnit.SECONDS));
 346         assertEquals("getShardElectionTimeoutFactor", 6, shardInfoMap.get("default").getValue().
 347                 getShardElectionTimeoutFactor());
 348         assertEquals("getShardElectionTimeoutFactor", 7, shardInfoMap.get("topology").getValue().
 349                 getShardElectionTimeoutFactor());
 350
 351         DatastoreContextFactory newMockFactory = newDatastoreContextFactory(
 352                 datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
 353         Mockito.doReturn(DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).
 354                 shardElectionTimeoutFactor(66).build()).when(newMockFactory).getShardDatastoreContext("default");
 355
 356         Mockito.doReturn(DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).
 357                 shardElectionTimeoutFactor(77).build()).when(newMockFactory).getShardDatastoreContext("topology");
 358
 359         shardManager.tell(newMockFactory, kit.getRef());
 360
 361         DatastoreContext newContext = MessageCollectorActor.expectFirstMatching(defaultShardActor, DatastoreContext.class);
 362         assertEquals("getShardElectionTimeoutFactor", 66, newContext.getShardElectionTimeoutFactor());
 363
 364         newContext = MessageCollectorActor.expectFirstMatching(topologyShardActor, DatastoreContext.class);
 365         assertEquals("getShardElectionTimeoutFactor", 77, newContext.getShardElectionTimeoutFactor());
 366
 367         LOG.info("testPerShardDatastoreContext ending");
 368     }
 369
 370     @Test
 371     public void testOnReceiveFindPrimaryForNonExistentShard() throws Exception {
 372         new JavaTestKit(getSystem()) {{
 373             final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 374
 375             shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 376
 377             shardManager.tell(new FindPrimary("non-existent", false), getRef());
 378
 379             expectMsgClass(duration("5 seconds"), PrimaryNotFoundException.class);
 380         }};
 381     }
 382
 383     @Test
 384     public void testOnReceiveFindPrimaryForLocalLeaderShard() throws Exception {
 385         LOG.info("testOnReceiveFindPrimaryForLocalLeaderShard starting");
 386         new JavaTestKit(getSystem()) {{
 387             String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
 388
 389             final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 390
 391             shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 392             shardManager.tell(new ActorInitialized(), mockShardActor);
 393
 394             DataTree mockDataTree = mock(DataTree.class);
 395             shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, Optional.of(mockDataTree),
 396                     DataStoreVersions.CURRENT_VERSION), getRef());
 397
 398             MessageCollectorActor.expectFirstMatching(mockShardActor, RegisterRoleChangeListener.class);
 399             shardManager.tell((new RoleChangeNotification(memberId, RaftState.Candidate.name(),
 400                     RaftState.Leader.name())), mockShardActor);
 401
 402             shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 403
 404             LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
 405             assertTrue("Unexpected primary path " +  primaryFound.getPrimaryPath(),
 406                     primaryFound.getPrimaryPath().contains("member-1-shard-default"));
 407             assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
 408         }};
 409
 410         LOG.info("testOnReceiveFindPrimaryForLocalLeaderShard ending");
 411     }
 412
 413     @Test
 414     public void testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp() throws Exception {
 415         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp starting");
 416         new JavaTestKit(getSystem()) {{
 417             final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 418
 419             shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 420             shardManager.tell(new ActorInitialized(), mockShardActor);
 421
 422             String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
 423             String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
 424             shardManager.tell(new RoleChangeNotification(memberId1,
 425                     RaftState.Candidate.name(), RaftState.Follower.name()), mockShardActor);
 426             shardManager.tell(new LeaderStateChanged(memberId1, memberId2, DataStoreVersions.CURRENT_VERSION), mockShardActor);
 427
 428             shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 429
 430             expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 431         }};
 432
 433         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShardBeforeMemberUp ending");
 434     }
 435
 436     @Test
 437     public void testOnReceiveFindPrimaryForNonLocalLeaderShard() throws Exception {
 438         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShard starting");
 439         new JavaTestKit(getSystem()) {{
 440             final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 441
 442             shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 443             shardManager.tell(new ActorInitialized(), mockShardActor);
 444
 445             String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
 446             MockClusterWrapper.sendMemberUp(shardManager, "member-2", getRef().path().toString());
 447
 448             String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
 449             shardManager.tell(new RoleChangeNotification(memberId1,
 450                     RaftState.Candidate.name(), RaftState.Follower.name()), mockShardActor);
 451             short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
 452             shardManager.tell(new ShardLeaderStateChanged(memberId1, memberId2, Optional.<DataTree>absent(),
 453                     leaderVersion), mockShardActor);
 454
 455             shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 456
 457             RemotePrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 458             assertTrue("Unexpected primary path " +  primaryFound.getPrimaryPath(),
 459                     primaryFound.getPrimaryPath().contains("member-2-shard-default"));
 460             assertEquals("getPrimaryVersion", leaderVersion, primaryFound.getPrimaryVersion());
 461         }};
 462
 463         LOG.info("testOnReceiveFindPrimaryForNonLocalLeaderShard ending");
 464     }
 465
 466     @Test
 467     public void testOnReceiveFindPrimaryForUninitializedShard() throws Exception {
 468         new JavaTestKit(getSystem()) {{
 469             final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 470
 471             shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 472
 473             expectMsgClass(duration("5 seconds"), NotInitializedException.class);
 474         }};
 475     }
 476
 477     @Test
 478     public void testOnReceiveFindPrimaryForInitializedShardWithNoRole() throws Exception {
 479         new JavaTestKit(getSystem()) {{
 480             final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 481
 482             shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 483             shardManager.tell(new ActorInitialized(), mockShardActor);
 484
 485             shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 486
 487             expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 488         }};
 489     }
 490
 491     @Test
 492     public void testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId() throws Exception {
 493         LOG.info("testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId starting");
 494         new JavaTestKit(getSystem()) {{
 495             final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 496
 497             shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 498             shardManager.tell(new ActorInitialized(), mockShardActor);
 499
 500             String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
 501             shardManager.tell(new RoleChangeNotification(memberId,
 502                     RaftState.Candidate.name(), RaftState.Follower.name()), mockShardActor);
 503
 504             shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 505
 506             expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 507
 508             DataTree mockDataTree = mock(DataTree.class);
 509             shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, Optional.of(mockDataTree),
 510                     DataStoreVersions.CURRENT_VERSION), mockShardActor);
 511
 512             shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 513
 514             LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
 515             assertTrue("Unexpected primary path " +  primaryFound.getPrimaryPath(),
 516                     primaryFound.getPrimaryPath().contains("member-1-shard-default"));
 517             assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree());
 518         }};
 519
 520         LOG.info("testOnReceiveFindPrimaryForFollowerShardWithNoInitialLeaderId starting");
 521     }
 522
 523     @Test
 524     public void testOnReceiveFindPrimaryWaitForShardLeader() throws Exception {
 525         LOG.info("testOnReceiveFindPrimaryWaitForShardLeader starting");
 526         new JavaTestKit(getSystem()) {{
 527             final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 528
 529             shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 530
 531             // We're passing waitUntilInitialized = true to FindPrimary so the response should be
 532             // delayed until we send ActorInitialized and RoleChangeNotification.
 533             shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
 534
 535             expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
 536
 537             shardManager.tell(new ActorInitialized(), mockShardActor);
 538
 539             expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
 540
 541             String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
 542             shardManager.tell(new RoleChangeNotification(memberId,
 543                     RaftState.Candidate.name(), RaftState.Leader.name()), mockShardActor);
 544
 545             expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
 546
 547             DataTree mockDataTree = mock(DataTree.class);
 548             shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, Optional.of(mockDataTree),
 549                     DataStoreVersions.CURRENT_VERSION), mockShardActor);
 550
 551             LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
 552             assertTrue("Unexpected primary path " +  primaryFound.getPrimaryPath(),
 553                     primaryFound.getPrimaryPath().contains("member-1-shard-default"));
 554             assertSame("getLocalShardDataTree", mockDataTree, primaryFound.getLocalShardDataTree() );
 555
 556             expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
 557         }};
 558
 559         LOG.info("testOnReceiveFindPrimaryWaitForShardLeader ending");
 560     }
 561
 562     @Test
 563     public void testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard() throws Exception {
 564         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard starting");
 565         new JavaTestKit(getSystem()) {{
 566             final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 567
 568             shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 569
 570             shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
 571
 572             expectMsgClass(duration("2 seconds"), NotInitializedException.class);
 573
 574             shardManager.tell(new ActorInitialized(), mockShardActor);
 575
 576             expectNoMsg(FiniteDuration.create(200, TimeUnit.MILLISECONDS));
 577         }};
 578
 579         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithUninitializedShard ending");
 580     }
 581
 582     @Test
 583     public void testOnReceiveFindPrimaryWaitForReadyWithCandidateShard() throws Exception {
 584         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithCandidateShard starting");
 585         new JavaTestKit(getSystem()) {{
 586             final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 587
 588             shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 589             shardManager.tell(new ActorInitialized(), mockShardActor);
 590             shardManager.tell(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix,
 591                     null, RaftState.Candidate.name()), mockShardActor);
 592
 593             shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
 594
 595             expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
 596         }};
 597
 598         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithCandidateShard ending");
 599     }
 600
 601     @Test
 602     public void testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard() throws Exception {
 603         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard starting");
 604         new JavaTestKit(getSystem()) {{
 605             final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 606
 607             shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 608             shardManager.tell(new ActorInitialized(), mockShardActor);
 609             shardManager.tell(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix,
 610                     null, RaftState.IsolatedLeader.name()), mockShardActor);
 611
 612             shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
 613
 614             expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
 615         }};
 616
 617         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard ending");
 618     }
 619
 620     @Test
 621     public void testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard() throws Exception {
 622         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard starting");
 623         new JavaTestKit(getSystem()) {{
 624             final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 625
 626             shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 627             shardManager.tell(new ActorInitialized(), mockShardActor);
 628
 629             shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
 630
 631             expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
 632         }};
 633
 634         LOG.info("testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard ending");
 635     }
 636
 637     @Test
 638     public void testOnReceiveFindPrimaryForRemoteShard() throws Exception {
 639         LOG.info("testOnReceiveFindPrimaryForRemoteShard starting");
 640         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
 641
 642         // Create an ActorSystem ShardManager actor for member-1.
 643
 644         final ActorSystem system1 = newActorSystem("Member1");
 645         Cluster.get(system1).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
 646
 647         final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
 648                 newTestShardMgrBuilderWithMockShardActor().cluster(
 649                         new ClusterWrapperImpl(system1)).props(), shardManagerID);
 650
 651         // Create an ActorSystem ShardManager actor for member-2.
 652
 653         final ActorSystem system2 = newActorSystem("Member2");
 654
 655         Cluster.get(system2).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
 656
 657         final ActorRef mockShardActor2 = newMockShardActor(system2, "astronauts", "member-2");
 658
 659         MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder().
 660                 put("default", Arrays.asList("member-1", "member-2")).
 661                 put("astronauts", Arrays.asList("member-2")).build());
 662
 663         final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
 664                 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
 665                         new ClusterWrapperImpl(system2)).props(), shardManagerID);
 666
 667         new JavaTestKit(system1) {{
 668
 669             shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 670             shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 671
 672             shardManager2.tell(new ActorInitialized(), mockShardActor2);
 673
 674             String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
 675             short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
 676             shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2,
 677                     Optional.of(mock(DataTree.class)), leaderVersion), mockShardActor2);
 678             shardManager2.tell(new RoleChangeNotification(memberId2,
 679                     RaftState.Candidate.name(), RaftState.Leader.name()), mockShardActor2);
 680
 681             shardManager1.underlyingActor().waitForMemberUp();
 682
 683             shardManager1.tell(new FindPrimary("astronauts", false), getRef());
 684
 685             RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 686             String path = found.getPrimaryPath();
 687             assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-astronauts-config"));
 688             assertEquals("getPrimaryVersion", leaderVersion, found.getPrimaryVersion());
 689
 690             shardManager2.underlyingActor().verifyFindPrimary();
 691
 692             Cluster.get(system2).down(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
 693
 694             shardManager1.underlyingActor().waitForMemberRemoved();
 695
 696             shardManager1.tell(new FindPrimary("astronauts", false), getRef());
 697
 698             expectMsgClass(duration("5 seconds"), PrimaryNotFoundException.class);
 699         }};
 700
 701         LOG.info("testOnReceiveFindPrimaryForRemoteShard ending");
 702     }
 703
 704     @Test
 705     public void testShardAvailabilityOnChangeOfMemberReachability() throws Exception {
 706         LOG.info("testShardAvailabilityOnChangeOfMemberReachability starting");
 707         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
 708
 709         // Create an ActorSystem ShardManager actor for member-1.
 710
 711         final ActorSystem system1 = newActorSystem("Member1");
 712         Cluster.get(system1).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
 713
 714         final ActorRef mockShardActor1 = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
 715
 716         final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
 717                 newTestShardMgrBuilder().shardActor(mockShardActor1).cluster(
 718                         new ClusterWrapperImpl(system1)).props(), shardManagerID);
 719
 720         // Create an ActorSystem ShardManager actor for member-2.
 721
 722         final ActorSystem system2 = newActorSystem("Member2");
 723
 724         Cluster.get(system2).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
 725
 726         final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
 727
 728         MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder().
 729             put("default", Arrays.asList("member-1", "member-2")).build());
 730
 731         final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
 732                 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
 733                         new ClusterWrapperImpl(system2)).props(), shardManagerID);
 734
 735         new JavaTestKit(system1) {{
 736
 737             shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 738             shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 739             shardManager1.tell(new ActorInitialized(), mockShardActor1);
 740             shardManager2.tell(new ActorInitialized(), mockShardActor2);
 741
 742             String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
 743             String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
 744             shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2,
 745                 Optional.of(mock(DataTree.class)), DataStoreVersions.CURRENT_VERSION), mockShardActor1);
 746             shardManager1.tell(new RoleChangeNotification(memberId1,
 747                 RaftState.Candidate.name(), RaftState.Follower.name()), mockShardActor1);
 748             shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, Optional.of(mock(DataTree.class)),
 749                     DataStoreVersions.CURRENT_VERSION),
 750                 mockShardActor2);
 751             shardManager2.tell(new RoleChangeNotification(memberId2,
 752                 RaftState.Candidate.name(), RaftState.Leader.name()), mockShardActor2);
 753             shardManager1.underlyingActor().waitForMemberUp();
 754
 755             shardManager1.tell(new FindPrimary("default", true), getRef());
 756
 757             RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 758             String path = found.getPrimaryPath();
 759             assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
 760
 761             shardManager1.underlyingActor().onReceiveCommand(MockClusterWrapper.
 762                 createUnreachableMember("member-2", "akka.tcp://cluster-test@127.0.0.1:2558"));
 763
 764             shardManager1.underlyingActor().waitForUnreachableMember();
 765
 766             PeerDown peerDown = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
 767             assertEquals("getMemberName", "member-2", peerDown.getMemberName());
 768             MessageCollectorActor.clearMessages(mockShardActor1);
 769
 770             shardManager1.underlyingActor().onReceiveCommand(MockClusterWrapper.
 771                     createMemberRemoved("member-2", "akka.tcp://cluster-test@127.0.0.1:2558"));
 772
 773             MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
 774
 775             shardManager1.tell(new FindPrimary("default", true), getRef());
 776
 777             expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 778
 779             shardManager1.underlyingActor().onReceiveCommand(MockClusterWrapper.
 780                 createReachableMember("member-2", "akka.tcp://cluster-test@127.0.0.1:2558"));
 781
 782             shardManager1.underlyingActor().waitForReachableMember();
 783
 784             PeerUp peerUp = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
 785             assertEquals("getMemberName", "member-2", peerUp.getMemberName());
 786             MessageCollectorActor.clearMessages(mockShardActor1);
 787
 788             shardManager1.tell(new FindPrimary("default", true), getRef());
 789
 790             RemotePrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 791             String path1 = found1.getPrimaryPath();
 792             assertTrue("Unexpected primary path " + path1, path1.contains("member-2-shard-default-config"));
 793
 794             shardManager1.underlyingActor().onReceiveCommand(MockClusterWrapper.
 795                     createMemberUp("member-2", "akka.tcp://cluster-test@127.0.0.1:2558"));
 796
 797             MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
 798
 799             // Test FindPrimary wait succeeds after reachable member event.
 800
 801             shardManager1.underlyingActor().onReceiveCommand(MockClusterWrapper.
 802                     createUnreachableMember("member-2", "akka.tcp://cluster-test@127.0.0.1:2558"));
 803             shardManager1.underlyingActor().waitForUnreachableMember();
 804
 805             shardManager1.tell(new FindPrimary("default", true), getRef());
 806
 807             shardManager1.underlyingActor().onReceiveCommand(MockClusterWrapper.
 808                     createReachableMember("member-2", "akka.tcp://cluster-test@127.0.0.1:2558"));
 809
 810             RemotePrimaryShardFound found2 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 811             String path2 = found2.getPrimaryPath();
 812             assertTrue("Unexpected primary path " + path2, path2.contains("member-2-shard-default-config"));
 813         }};
 814
 815         LOG.info("testShardAvailabilityOnChangeOfMemberReachability ending");
 816     }
 817
 818     @Test
 819     public void testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange() throws Exception {
 820         LOG.info("testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange starting");
 821         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
 822
 823         // Create an ActorSystem ShardManager actor for member-1.
 824
 825         final ActorSystem system1 = newActorSystem("Member1");
 826         Cluster.get(system1).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
 827
 828         final ActorRef mockShardActor1 = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
 829
 830         final PrimaryShardInfoFutureCache primaryShardInfoCache = new PrimaryShardInfoFutureCache();
 831         final TestActorRef<TestShardManager> shardManager1 = TestActorRef.create(system1,
 832                 newTestShardMgrBuilder().shardActor(mockShardActor1).cluster(
 833                         new ClusterWrapperImpl(system1)).primaryShardInfoCache(primaryShardInfoCache).props(),
 834                 shardManagerID);
 835
 836         // Create an ActorSystem ShardManager actor for member-2.
 837
 838         final ActorSystem system2 = newActorSystem("Member2");
 839
 840         Cluster.get(system2).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
 841
 842         final ActorRef mockShardActor2 = newMockShardActor(system2, Shard.DEFAULT_NAME, "member-2");
 843
 844         MockConfiguration mockConfig2 = new MockConfiguration(ImmutableMap.<String, List<String>>builder().
 845             put("default", Arrays.asList("member-1", "member-2")).build());
 846
 847         final TestActorRef<TestShardManager> shardManager2 = TestActorRef.create(system2,
 848                 newTestShardMgrBuilder(mockConfig2).shardActor(mockShardActor2).cluster(
 849                         new ClusterWrapperImpl(system2)).props(), shardManagerID);
 850
 851         new JavaTestKit(system1) {{
 852             shardManager1.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 853             shardManager2.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 854             shardManager1.tell(new ActorInitialized(), mockShardActor1);
 855             shardManager2.tell(new ActorInitialized(), mockShardActor2);
 856
 857             String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
 858             String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
 859             shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2,
 860                 Optional.of(mock(DataTree.class)), DataStoreVersions.CURRENT_VERSION), mockShardActor1);
 861             shardManager1.tell(new RoleChangeNotification(memberId1,
 862                 RaftState.Candidate.name(), RaftState.Follower.name()), mockShardActor1);
 863             shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, Optional.of(mock(DataTree.class)),
 864                     DataStoreVersions.CURRENT_VERSION),
 865                 mockShardActor2);
 866             shardManager2.tell(new RoleChangeNotification(memberId2,
 867                 RaftState.Candidate.name(), RaftState.Leader.name()), mockShardActor2);
 868             shardManager1.underlyingActor().waitForMemberUp();
 869
 870             shardManager1.tell(new FindPrimary("default", true), getRef());
 871
 872             RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
 873             String path = found.getPrimaryPath();
 874             assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
 875
 876             primaryShardInfoCache.putSuccessful("default", new PrimaryShardInfo(system1.actorSelection(
 877                     mockShardActor1.path()), DataStoreVersions.CURRENT_VERSION, Optional.<DataTree>absent()));
 878
 879             shardManager1.underlyingActor().onReceiveCommand(MockClusterWrapper.
 880                 createUnreachableMember("member-2", "akka.tcp://cluster-test@127.0.0.1:2558"));
 881
 882             shardManager1.underlyingActor().waitForUnreachableMember();
 883
 884             shardManager1.tell(new FindPrimary("default", true), getRef());
 885
 886             expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 887
 888             assertNull("Expected primaryShardInfoCache entry removed", primaryShardInfoCache.getIfPresent("default"));
 889
 890             shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId1, Optional.of(mock(DataTree.class)),
 891                     DataStoreVersions.CURRENT_VERSION), mockShardActor1);
 892             shardManager1.tell(new RoleChangeNotification(memberId1,
 893                 RaftState.Follower.name(), RaftState.Leader.name()), mockShardActor1);
 894
 895             shardManager1.tell(new FindPrimary("default", true), getRef());
 896
 897             LocalPrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
 898             String path1 = found1.getPrimaryPath();
 899             assertTrue("Unexpected primary path " + path1, path1.contains("member-1-shard-default-config"));
 900
 901         }};
 902
 903         LOG.info("testShardAvailabilityChangeOnMemberUnreachableAndLeadershipChange ending");
 904     }
 905
 906
 907     @Test
 908     public void testOnReceiveFindLocalShardForNonExistentShard() throws Exception {
 909         new JavaTestKit(getSystem()) {{
 910             final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 911
 912             shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 913
 914             shardManager.tell(new FindLocalShard("non-existent", false), getRef());
 915
 916             LocalShardNotFound notFound = expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
 917
 918             assertEquals("getShardName", "non-existent", notFound.getShardName());
 919         }};
 920     }
 921
 922     @Test
 923     public void testOnReceiveFindLocalShardForExistentShard() throws Exception {
 924         new JavaTestKit(getSystem()) {{
 925             final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 926
 927             shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 928             shardManager.tell(new ActorInitialized(), mockShardActor);
 929
 930             shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
 931
 932             LocalShardFound found = expectMsgClass(duration("5 seconds"), LocalShardFound.class);
 933
 934             assertTrue("Found path contains " + found.getPath().path().toString(),
 935                     found.getPath().path().toString().contains("member-1-shard-default-config"));
 936         }};
 937     }
 938
 939     @Test
 940     public void testOnReceiveFindLocalShardForNotInitializedShard() throws Exception {
 941         new JavaTestKit(getSystem()) {{
 942             final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 943
 944             shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
 945
 946             expectMsgClass(duration("5 seconds"), NotInitializedException.class);
 947         }};
 948     }
 949
 950     @Test
 951     public void testOnReceiveFindLocalShardWaitForShardInitialized() throws Exception {
 952         LOG.info("testOnReceiveFindLocalShardWaitForShardInitialized starting");
 953         new JavaTestKit(getSystem()) {{
 954             final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
 955
 956             shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
 957
 958             // We're passing waitUntilInitialized = true to FindLocalShard so the response should be
 959             // delayed until we send ActorInitialized.
 960             Future<Object> future = Patterns.ask(shardManager, new FindLocalShard(Shard.DEFAULT_NAME, true),
 961                     new Timeout(5, TimeUnit.SECONDS));
 962
 963             shardManager.tell(new ActorInitialized(), mockShardActor);
 964
 965             Object resp = Await.result(future, duration("5 seconds"));
 966             assertTrue("Expected: LocalShardFound, Actual: " + resp, resp instanceof LocalShardFound);
 967         }};
 968
 969         LOG.info("testOnReceiveFindLocalShardWaitForShardInitialized starting");
 970     }
 971
 972     @Test
 973     public void testOnRecoveryJournalIsCleaned() {
 974         String persistenceID = "shard-manager-" + shardMrgIDSuffix;
 975         InMemoryJournal.addEntry(persistenceID, 1L, new ShardManager.SchemaContextModules(
 976                 ImmutableSet.of("foo")));
 977         InMemoryJournal.addEntry(persistenceID, 2L, new ShardManager.SchemaContextModules(
 978                 ImmutableSet.of("bar")));
 979         InMemoryJournal.addDeleteMessagesCompleteLatch(persistenceID);
 980
 981         TestShardManager shardManager = newTestShardManager();
 982
 983         InMemoryJournal.waitForDeleteMessagesComplete(persistenceID);
 984
 985         // Journal entries up to the last one should've been deleted
 986         Map<Long, Object> journal = InMemoryJournal.get(persistenceID);
 987         synchronized (journal) {
 988             assertEquals("Journal size", 0, journal.size());
 989         }
 990     }
 991
 992     @Test
 993     public void testRoleChangeNotificationAndShardLeaderStateChangedReleaseReady() throws Exception {
 994         TestShardManager shardManager = newTestShardManager();
 995
 996         String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
 997         shardManager.onReceiveCommand(new RoleChangeNotification(
 998                 memberId, RaftState.Candidate.name(), RaftState.Leader.name()));
 999
1000         verify(ready, never()).countDown();
1001
1002         shardManager.onReceiveCommand(new ShardLeaderStateChanged(memberId, memberId,
1003                 Optional.of(mock(DataTree.class)), DataStoreVersions.CURRENT_VERSION));
1004
1005         verify(ready, times(1)).countDown();
1006     }
1007
1008     @Test
1009     public void testRoleChangeNotificationToFollowerWithShardLeaderStateChangedReleaseReady() throws Exception {
1010         new JavaTestKit(getSystem()) {{
1011             TestShardManager shardManager = newTestShardManager();
1012
1013             String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1014             shardManager.onReceiveCommand(new RoleChangeNotification(
1015                     memberId, null, RaftState.Follower.name()));
1016
1017             verify(ready, never()).countDown();
1018
1019             shardManager.onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
1020
1021             shardManager.onReceiveCommand(new ShardLeaderStateChanged(memberId,
1022                     "member-2-shard-default-" + shardMrgIDSuffix, Optional.of(mock(DataTree.class)),
1023                     DataStoreVersions.CURRENT_VERSION));
1024
1025             verify(ready, times(1)).countDown();
1026         }};
1027     }
1028
1029     @Test
1030     public void testReadyCountDownForMemberUpAfterLeaderStateChanged() throws Exception {
1031         new JavaTestKit(getSystem()) {{
1032             TestShardManager shardManager = newTestShardManager();
1033
1034             String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1035             shardManager.onReceiveCommand(new RoleChangeNotification(memberId, null, RaftState.Follower.name()));
1036
1037             verify(ready, never()).countDown();
1038
1039             shardManager.onReceiveCommand(new ShardLeaderStateChanged(memberId,
1040                     "member-2-shard-default-" + shardMrgIDSuffix, Optional.of(mock(DataTree.class)),
1041                     DataStoreVersions.CURRENT_VERSION));
1042
1043             shardManager.onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
1044
1045             verify(ready, times(1)).countDown();
1046         }};
1047     }
1048
1049     @Test
1050     public void testRoleChangeNotificationDoNothingForUnknownShard() throws Exception {
1051         TestShardManager shardManager = newTestShardManager();
1052
1053         shardManager.onReceiveCommand(new RoleChangeNotification(
1054                 "unknown", RaftState.Candidate.name(), RaftState.Leader.name()));
1055
1056         verify(ready, never()).countDown();
1057     }
1058
1059     @Test
1060     public void testByDefaultSyncStatusIsFalse() throws Exception{
1061         TestShardManager shardManager = newTestShardManager();
1062
1063         assertEquals(false, shardManager.getMBean().getSyncStatus());
1064     }
1065
1066     @Test
1067     public void testWhenShardIsLeaderSyncStatusIsTrue() throws Exception{
1068         TestShardManager shardManager = newTestShardManager();
1069
1070         shardManager.onReceiveCommand(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix,
1071                 RaftState.Follower.name(), RaftState.Leader.name()));
1072
1073         assertEquals(true, shardManager.getMBean().getSyncStatus());
1074     }
1075
1076     @Test
1077     public void testWhenShardIsCandidateSyncStatusIsFalse() throws Exception{
1078         TestShardManager shardManager = newTestShardManager();
1079
1080         String shardId = "member-1-shard-default-" + shardMrgIDSuffix;
1081         shardManager.onReceiveCommand(new RoleChangeNotification(shardId,
1082                 RaftState.Follower.name(), RaftState.Candidate.name()));
1083
1084         assertEquals(false, shardManager.getMBean().getSyncStatus());
1085
1086         // Send a FollowerInitialSyncStatus with status = true for the replica whose current state is candidate
1087         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(
1088                 true, shardId));
1089
1090         assertEquals(false, shardManager.getMBean().getSyncStatus());
1091     }
1092
1093     @Test
1094     public void testWhenShardIsFollowerSyncStatusDependsOnFollowerInitialSyncStatus() throws Exception{
1095         TestShardManager shardManager = newTestShardManager();
1096
1097         String shardId = "member-1-shard-default-" + shardMrgIDSuffix;
1098         shardManager.onReceiveCommand(new RoleChangeNotification(shardId,
1099                 RaftState.Candidate.name(), RaftState.Follower.name()));
1100
1101         // Initially will be false
1102         assertEquals(false, shardManager.getMBean().getSyncStatus());
1103
1104         // Send status true will make sync status true
1105         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(true, shardId));
1106
1107         assertEquals(true, shardManager.getMBean().getSyncStatus());
1108
1109         // Send status false will make sync status false
1110         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(false, shardId));
1111
1112         assertEquals(false, shardManager.getMBean().getSyncStatus());
1113
1114     }
1115
1116     @Test
1117     public void testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards() throws Exception{
1118         LOG.info("testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards starting");
1119         TestShardManager shardManager = newTestShardManager(newShardMgrProps(new MockConfiguration() {
1120             @Override
1121             public List<String> getMemberShardNames(String memberName) {
1122                 return Arrays.asList("default", "astronauts");
1123             }
1124         }));
1125
1126         // Initially will be false
1127         assertEquals(false, shardManager.getMBean().getSyncStatus());
1128
1129         // Make default shard leader
1130         String defaultShardId = "member-1-shard-default-" + shardMrgIDSuffix;
1131         shardManager.onReceiveCommand(new RoleChangeNotification(defaultShardId,
1132                 RaftState.Follower.name(), RaftState.Leader.name()));
1133
1134         // default = Leader, astronauts is unknown so sync status remains false
1135         assertEquals(false, shardManager.getMBean().getSyncStatus());
1136
1137         // Make astronauts shard leader as well
1138         String astronautsShardId = "member-1-shard-astronauts-" + shardMrgIDSuffix;
1139         shardManager.onReceiveCommand(new RoleChangeNotification(astronautsShardId,
1140                 RaftState.Follower.name(), RaftState.Leader.name()));
1141
1142         // Now sync status should be true
1143         assertEquals(true, shardManager.getMBean().getSyncStatus());
1144
1145         // Make astronauts a Follower
1146         shardManager.onReceiveCommand(new RoleChangeNotification(astronautsShardId,
1147                 RaftState.Leader.name(), RaftState.Follower.name()));
1148
1149         // Sync status is not true
1150         assertEquals(false, shardManager.getMBean().getSyncStatus());
1151
1152         // Make the astronauts follower sync status true
1153         shardManager.onReceiveCommand(new FollowerInitialSyncUpStatus(true, astronautsShardId));
1154
1155         // Sync status is now true
1156         assertEquals(true, shardManager.getMBean().getSyncStatus());
1157
1158         LOG.info("testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards ending");
1159     }
1160
1161     @Test
1162     public void testOnReceiveSwitchShardBehavior() throws Exception {
1163         new JavaTestKit(getSystem()) {{
1164             final ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1165
1166             shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1167             shardManager.tell(new ActorInitialized(), mockShardActor);
1168
1169             shardManager.tell(new SwitchShardBehavior(mockShardName, "Leader", 1000), getRef());
1170
1171             SwitchBehavior switchBehavior = MessageCollectorActor.expectFirstMatching(mockShardActor, SwitchBehavior.class);
1172
1173             assertEquals(RaftState.Leader, switchBehavior.getNewState());
1174             assertEquals(1000, switchBehavior.getNewTerm());
1175         }};
1176     }
1177
1178     @Test
1179     public void testOnCreateShard() {
1180         LOG.info("testOnCreateShard starting");
1181         new JavaTestKit(getSystem()) {{
1182             datastoreContextBuilder.shardInitializationTimeout(1, TimeUnit.MINUTES).persistent(true);
1183
1184             ActorRef shardManager = actorFactory.createActor(newShardMgrProps(
1185                     new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1186
1187             SchemaContext schemaContext = TestModel.createTestContext();
1188             shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
1189
1190             DatastoreContext datastoreContext = DatastoreContext.newBuilder().shardElectionTimeoutFactor(100).
1191                     persistent(false).build();
1192             Shard.Builder shardBuilder = Shard.builder();
1193
1194             ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1195                     "foo", null, Arrays.asList("member-1", "member-5", "member-6"));
1196             shardManager.tell(new CreateShard(config, shardBuilder, datastoreContext), getRef());
1197
1198             expectMsgClass(duration("5 seconds"), Success.class);
1199
1200             shardManager.tell(new FindLocalShard("foo", true), getRef());
1201
1202             expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1203
1204             assertEquals("isRecoveryApplicable", false, shardBuilder.getDatastoreContext().isPersistent());
1205             assertTrue("Epxected ShardPeerAddressResolver", shardBuilder.getDatastoreContext().getShardRaftConfig().
1206                     getPeerAddressResolver() instanceof ShardPeerAddressResolver);
1207             assertEquals("peerMembers", Sets.newHashSet(new ShardIdentifier("foo", "member-5", shardMrgIDSuffix).toString(),
1208                     new ShardIdentifier("foo", "member-6", shardMrgIDSuffix).toString()),
1209                     shardBuilder.getPeerAddresses().keySet());
1210             assertEquals("ShardIdentifier", new ShardIdentifier("foo", "member-1", shardMrgIDSuffix),
1211                     shardBuilder.getId());
1212             assertSame("schemaContext", schemaContext, shardBuilder.getSchemaContext());
1213
1214             // Send CreateShard with same name - should return Success with a message.
1215
1216             shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1217
1218             Success success = expectMsgClass(duration("5 seconds"), Success.class);
1219             assertNotNull("Success status is null", success.status());
1220         }};
1221
1222         LOG.info("testOnCreateShard ending");
1223     }
1224
1225     @Test
1226     public void testOnCreateShardWithLocalMemberNotInShardConfig() {
1227         LOG.info("testOnCreateShardWithLocalMemberNotInShardConfig starting");
1228         new JavaTestKit(getSystem()) {{
1229             datastoreContextBuilder.shardInitializationTimeout(1, TimeUnit.MINUTES).persistent(true);
1230
1231             ActorRef shardManager = actorFactory.createActor(newShardMgrProps(
1232                     new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1233
1234             shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1235
1236             Shard.Builder shardBuilder = Shard.builder();
1237             ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1238                     "foo", null, Arrays.asList("member-5", "member-6"));
1239
1240             shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1241             expectMsgClass(duration("5 seconds"), Success.class);
1242
1243             shardManager.tell(new FindLocalShard("foo", true), getRef());
1244             expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1245
1246             assertEquals("peerMembers size", 0, shardBuilder.getPeerAddresses().size());
1247             assertEquals("schemaContext", DisableElectionsRaftPolicy.class.getName(),
1248                     shardBuilder.getDatastoreContext().getShardRaftConfig().getCustomRaftPolicyImplementationClass());
1249         }};
1250
1251         LOG.info("testOnCreateShardWithLocalMemberNotInShardConfig ending");
1252     }
1253
1254     @Test
1255     public void testOnCreateShardWithNoInitialSchemaContext() {
1256         LOG.info("testOnCreateShardWithNoInitialSchemaContext starting");
1257         new JavaTestKit(getSystem()) {{
1258             ActorRef shardManager = actorFactory.createActor(newShardMgrProps(
1259                     new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1260
1261             Shard.Builder shardBuilder = Shard.builder();
1262
1263             ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
1264                     "foo", null, Arrays.asList("member-1"));
1265             shardManager.tell(new CreateShard(config, shardBuilder, null), getRef());
1266
1267             expectMsgClass(duration("5 seconds"), Success.class);
1268
1269             SchemaContext schemaContext = TestModel.createTestContext();
1270             shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
1271
1272             shardManager.tell(new FindLocalShard("foo", true), getRef());
1273
1274             expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1275
1276             assertSame("schemaContext", schemaContext, shardBuilder.getSchemaContext());
1277             assertNotNull("schemaContext is null", shardBuilder.getDatastoreContext());
1278         }};
1279
1280         LOG.info("testOnCreateShardWithNoInitialSchemaContext ending");
1281     }
1282
1283     @Test
1284     public void testGetSnapshot() throws Throwable {
1285         LOG.info("testGetSnapshot starting");
1286         JavaTestKit kit = new JavaTestKit(getSystem());
1287
1288         MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder().
1289                    put("shard1", Arrays.asList("member-1")).
1290                    put("shard2", Arrays.asList("member-1")).
1291                    put("astronauts", Collections.<String>emptyList()).build());
1292
1293         TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(newShardMgrProps(mockConfig).
1294                 withDispatcher(Dispatchers.DefaultDispatcherId()));
1295
1296         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1297         Failure failure = kit.expectMsgClass(Failure.class);
1298         assertEquals("Failure cause type", IllegalStateException.class, failure.cause().getClass());
1299
1300         shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1301
1302         waitForShardInitialized(shardManager, "shard1", kit);
1303         waitForShardInitialized(shardManager, "shard2", kit);
1304
1305         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1306
1307         DatastoreSnapshot datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1308
1309         assertEquals("getType", shardMrgIDSuffix, datastoreSnapshot.getType());
1310         assertNull("Expected null ShardManagerSnapshot", datastoreSnapshot.getShardManagerSnapshot());
1311
1312         Function<ShardSnapshot, String> shardNameTransformer = new Function<ShardSnapshot, String>() {
1313             @Override
1314             public String apply(ShardSnapshot s) {
1315                 return s.getName();
1316             }
1317         };
1318
1319         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2"), Sets.newHashSet(
1320                 Lists.transform(datastoreSnapshot.getShardSnapshots(), shardNameTransformer)));
1321
1322         // Add a new replica
1323
1324         JavaTestKit mockShardLeaderKit = new JavaTestKit(getSystem());
1325
1326         TestShardManager shardManagerInstance = shardManager.underlyingActor();
1327         shardManagerInstance.setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1328
1329         shardManager.tell(new AddShardReplica("astronauts"), kit.getRef());
1330         mockShardLeaderKit.expectMsgClass(AddServer.class);
1331         mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.OK, ""));
1332         kit.expectMsgClass(Status.Success.class);
1333         waitForShardInitialized(shardManager, "astronauts", kit);
1334
1335         // Send another GetSnapshot and verify
1336
1337         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1338         datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1339
1340         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"), Sets.newHashSet(
1341                 Lists.transform(datastoreSnapshot.getShardSnapshots(), shardNameTransformer)));
1342
1343         byte[] snapshotBytes = datastoreSnapshot.getShardManagerSnapshot();
1344         assertNotNull("Expected ShardManagerSnapshot", snapshotBytes);
1345         ShardManagerSnapshot snapshot = SerializationUtils.deserialize(snapshotBytes);
1346         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"),
1347                 Sets.newHashSet(snapshot.getShardList()));
1348
1349         LOG.info("testGetSnapshot ending");
1350     }
1351
1352     @Test
1353     public void testRestoreFromSnapshot() throws Throwable {
1354         LOG.info("testRestoreFromSnapshot starting");
1355
1356         JavaTestKit kit = new JavaTestKit(getSystem());
1357
1358         MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder().
1359                    put("shard1", Collections.<String>emptyList()).
1360                    put("shard2", Collections.<String>emptyList()).
1361                    put("astronauts", Collections.<String>emptyList()).build());
1362
1363
1364         ShardManagerSnapshot snapshot = new ShardManagerSnapshot(Arrays.asList("shard1", "shard2", "astronauts"));
1365         DatastoreSnapshot restoreFromSnapshot = new DatastoreSnapshot(shardMrgIDSuffix,
1366                 SerializationUtils.serialize(snapshot), Collections.<ShardSnapshot>emptyList());
1367         TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(newTestShardMgrBuilder(mockConfig).
1368                 restoreFromSnapshot(restoreFromSnapshot).props().withDispatcher(Dispatchers.DefaultDispatcherId()));
1369
1370         shardManager.underlyingActor().waitForRecoveryComplete();
1371
1372         shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), ActorRef.noSender());
1373
1374         waitForShardInitialized(shardManager, "shard1", kit);
1375         waitForShardInitialized(shardManager, "shard2", kit);
1376         waitForShardInitialized(shardManager, "astronauts", kit);
1377
1378         shardManager.tell(GetSnapshot.INSTANCE, kit.getRef());
1379
1380         DatastoreSnapshot datastoreSnapshot = expectMsgClassOrFailure(DatastoreSnapshot.class, kit, "GetSnapshot");
1381
1382         assertEquals("getType", shardMrgIDSuffix, datastoreSnapshot.getType());
1383
1384         byte[] snapshotBytes = datastoreSnapshot.getShardManagerSnapshot();
1385         assertNotNull("Expected ShardManagerSnapshot", snapshotBytes);
1386         snapshot = SerializationUtils.deserialize(snapshotBytes);
1387         assertEquals("Shard names", Sets.newHashSet("shard1", "shard2", "astronauts"),
1388                 Sets.newHashSet(snapshot.getShardList()));
1389
1390         LOG.info("testRestoreFromSnapshot ending");
1391     }
1392
1393     @Test
1394     public void testAddShardReplicaForNonExistentShardConfig() throws Exception {
1395         new JavaTestKit(getSystem()) {{
1396             ActorRef shardManager = actorFactory.createActor(newShardMgrProps(
1397                     new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1398
1399             shardManager.tell(new AddShardReplica("model-inventory"), getRef());
1400             Status.Failure resp = expectMsgClass(duration("2 seconds"), Status.Failure.class);
1401
1402             assertEquals("Failure obtained", true,
1403                           (resp.cause() instanceof IllegalArgumentException));
1404         }};
1405     }
1406
1407     @Test
1408     public void testAddShardReplica() throws Exception {
1409         MockConfiguration mockConfig =
1410                 new MockConfiguration(ImmutableMap.<String, List<String>>builder().
1411                    put("default", Arrays.asList("member-1", "member-2")).
1412                    put("astronauts", Arrays.asList("member-2")).build());
1413
1414         final String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
1415         datastoreContextBuilder.shardManagerPersistenceId(shardManagerID);
1416
1417         // Create an ActorSystem ShardManager actor for member-1.
1418         final ActorSystem system1 = newActorSystem("Member1");
1419         Cluster.get(system1).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
1420         ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
1421         final TestActorRef<TestShardManager> newReplicaShardManager = TestActorRef.create(system1,
1422                 newTestShardMgrBuilder(mockConfig).shardActor(mockDefaultShardActor).cluster(
1423                         new ClusterWrapperImpl(system1)).props(), shardManagerID);
1424
1425         // Create an ActorSystem ShardManager actor for member-2.
1426         final ActorSystem system2 = newActorSystem("Member2");
1427         Cluster.get(system2).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
1428
1429         String name = new ShardIdentifier("astronauts", "member-2", "config").toString();
1430         final TestActorRef<MockRespondActor> mockShardLeaderActor =
1431                 TestActorRef.create(system2, Props.create(MockRespondActor.class), name);
1432         final TestActorRef<TestShardManager> leaderShardManager = TestActorRef.create(system2,
1433                 newTestShardMgrBuilder(mockConfig).shardActor(mockShardLeaderActor).cluster(
1434                         new ClusterWrapperImpl(system2)).props(), shardManagerID);
1435
1436         new JavaTestKit(system1) {{
1437
1438             newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1439             leaderShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1440
1441             leaderShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1442
1443             String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
1444             short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
1445             leaderShardManager.tell(new ShardLeaderStateChanged(memberId2, memberId2,
1446                     Optional.of(mock(DataTree.class)), leaderVersion), mockShardLeaderActor);
1447             leaderShardManager.tell(new RoleChangeNotification(memberId2,
1448                     RaftState.Candidate.name(), RaftState.Leader.name()), mockShardLeaderActor);
1449
1450             newReplicaShardManager.underlyingActor().waitForMemberUp();
1451             leaderShardManager.underlyingActor().waitForMemberUp();
1452
1453             //Have a dummy snapshot to be overwritten by the new data persisted.
1454             String[] restoredShards = {"default", "people"};
1455             ShardManagerSnapshot snapshot = new ShardManagerSnapshot(Arrays.asList(restoredShards));
1456             InMemorySnapshotStore.addSnapshot(shardManagerID, snapshot);
1457             Uninterruptibles.sleepUninterruptibly(2, TimeUnit.MILLISECONDS);
1458
1459             InMemorySnapshotStore.addSnapshotSavedLatch(shardManagerID);
1460             InMemorySnapshotStore.addSnapshotDeletedLatch(shardManagerID);
1461
1462             //construct a mock response message
1463             AddServerReply response = new AddServerReply(ServerChangeStatus.OK, memberId2);
1464             mockShardLeaderActor.underlyingActor().updateResponse(response);
1465             newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
1466             AddServer addServerMsg = MessageCollectorActor.expectFirstMatching(mockShardLeaderActor,
1467                 AddServer.class);
1468             String addServerId = "member-1-shard-astronauts-" + shardMrgIDSuffix;
1469             assertEquals("AddServer serverId", addServerId, addServerMsg.getNewServerId());
1470             expectMsgClass(duration("5 seconds"), Status.Success.class);
1471
1472             InMemorySnapshotStore.waitForSavedSnapshot(shardManagerID, ShardManagerSnapshot.class);
1473             InMemorySnapshotStore.waitForDeletedSnapshot(shardManagerID);
1474             List<ShardManagerSnapshot> persistedSnapshots =
1475                 InMemorySnapshotStore.getSnapshots(shardManagerID, ShardManagerSnapshot.class);
1476             assertEquals("Number of snapshots persisted", 1, persistedSnapshots.size());
1477             ShardManagerSnapshot shardManagerSnapshot = persistedSnapshots.get(0);
1478             assertEquals("Persisted local shards", Sets.newHashSet("default", "astronauts"),
1479                     Sets.newHashSet(shardManagerSnapshot.getShardList()));
1480         }};
1481
1482         LOG.info("testAddShardReplica ending");
1483     }
1484
1485     @Test
1486     public void testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader() throws Exception {
1487         LOG.info("testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader starting");
1488         new JavaTestKit(getSystem()) {{
1489             TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(
1490                     newPropsShardMgrWithMockShardActor(), shardMgrID);
1491
1492             shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1493             shardManager.tell(new ActorInitialized(), mockShardActor);
1494
1495             String leaderId = "leader-member-shard-default-" + shardMrgIDSuffix;
1496             AddServerReply addServerReply = new AddServerReply(ServerChangeStatus.ALREADY_EXISTS, null);
1497             ActorRef leaderShardActor = shardManager.underlyingActor().getContext().actorOf(
1498                     Props.create(MockRespondActor.class, addServerReply), leaderId);
1499
1500             MockClusterWrapper.sendMemberUp(shardManager, "leader-member", leaderShardActor.path().toString());
1501
1502             String newReplicaId = "member-1-shard-default-" + shardMrgIDSuffix;
1503             shardManager.tell(new RoleChangeNotification(newReplicaId,
1504                     RaftState.Candidate.name(), RaftState.Follower.name()), mockShardActor);
1505             shardManager.tell(new ShardLeaderStateChanged(newReplicaId, leaderId, Optional.<DataTree>absent(),
1506                     DataStoreVersions.CURRENT_VERSION), mockShardActor);
1507
1508             shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1509
1510             MessageCollectorActor.expectFirstMatching(leaderShardActor, AddServer.class);
1511
1512             Failure resp = expectMsgClass(duration("5 seconds"), Failure.class);
1513             assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1514
1515             shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1516             expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1517
1518             // Send message again to verify previous in progress state is cleared
1519
1520             shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1521             resp = expectMsgClass(duration("5 seconds"), Failure.class);
1522             assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1523
1524             // Send message again with an AddServer timeout to verify the pre-existing shard actor isn't terminated.
1525
1526             shardManager.tell(newDatastoreContextFactory(datastoreContextBuilder.
1527                     shardLeaderElectionTimeout(100, TimeUnit.MILLISECONDS).build()), getRef());
1528             leaderShardActor.tell(MockRespondActor.CLEAR_RESPONSE, ActorRef.noSender());
1529             shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1530             expectMsgClass(duration("5 seconds"), Failure.class);
1531
1532             shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1533             expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1534         }};
1535
1536         LOG.info("testAddShardReplicaWithPreExistingReplicaInRemoteShardLeader ending");
1537     }
1538
1539     @Test
1540     public void testAddShardReplicaWithPreExistingLocalReplicaLeader() throws Exception {
1541         LOG.info("testAddShardReplicaWithPreExistingLocalReplicaLeader starting");
1542         new JavaTestKit(getSystem()) {{
1543             String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1544             ActorRef shardManager = actorFactory.createActor(newPropsShardMgrWithMockShardActor());
1545
1546             shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1547             shardManager.tell(new ActorInitialized(), mockShardActor);
1548             shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, Optional.of(mock(DataTree.class)),
1549                     DataStoreVersions.CURRENT_VERSION), getRef());
1550             shardManager.tell((new RoleChangeNotification(memberId, RaftState.Candidate.name(),
1551                     RaftState.Leader.name())), mockShardActor);
1552
1553             shardManager.tell(new AddShardReplica(Shard.DEFAULT_NAME), getRef());
1554             Failure resp = expectMsgClass(duration("5 seconds"), Failure.class);
1555             assertEquals("Failure cause", AlreadyExistsException.class, resp.cause().getClass());
1556
1557             shardManager.tell(new FindLocalShard(Shard.DEFAULT_NAME, false), getRef());
1558             expectMsgClass(duration("5 seconds"), LocalShardFound.class);
1559         }};
1560
1561         LOG.info("testAddShardReplicaWithPreExistingLocalReplicaLeader ending");
1562     }
1563
1564     @Test
1565     public void testAddShardReplicaWithAddServerReplyFailure() throws Exception {
1566         LOG.info("testAddShardReplicaWithAddServerReplyFailure starting");
1567         new JavaTestKit(getSystem()) {{
1568             JavaTestKit mockShardLeaderKit = new JavaTestKit(getSystem());
1569
1570             MockConfiguration mockConfig =
1571                     new MockConfiguration(ImmutableMap.<String, List<String>>builder().
1572                        put("astronauts", Arrays.asList("member-2")).build());
1573
1574             ActorRef mockNewReplicaShardActor = newMockShardActor(getSystem(), "astronauts", "member-1");
1575             final TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(
1576                     newTestShardMgrBuilder(mockConfig).shardActor(mockNewReplicaShardActor).props(), shardMgrID);
1577             shardManager.underlyingActor().setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1578
1579             shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1580
1581             JavaTestKit terminateWatcher = new JavaTestKit(getSystem());
1582             terminateWatcher.watch(mockNewReplicaShardActor);
1583
1584             shardManager.tell(new AddShardReplica("astronauts"), getRef());
1585
1586             AddServer addServerMsg = mockShardLeaderKit.expectMsgClass(AddServer.class);
1587             assertEquals("AddServer serverId", "member-1-shard-astronauts-" + shardMrgIDSuffix,
1588                     addServerMsg.getNewServerId());
1589             mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.TIMEOUT, null));
1590
1591             Failure failure = expectMsgClass(duration("5 seconds"), Failure.class);
1592             assertEquals("Failure cause", TimeoutException.class, failure.cause().getClass());
1593
1594             shardManager.tell(new FindLocalShard("astronauts", false), getRef());
1595             expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
1596
1597             terminateWatcher.expectTerminated(mockNewReplicaShardActor);
1598
1599             shardManager.tell(new AddShardReplica("astronauts"), getRef());
1600             mockShardLeaderKit.expectMsgClass(AddServer.class);
1601             mockShardLeaderKit.reply(new AddServerReply(ServerChangeStatus.NO_LEADER, null));
1602             failure = expectMsgClass(duration("5 seconds"), Failure.class);
1603             assertEquals("Failure cause", NoShardLeaderException.class, failure.cause().getClass());
1604         }};
1605
1606         LOG.info("testAddShardReplicaWithAddServerReplyFailure ending");
1607     }
1608
1609     @Test
1610     public void testAddShardReplicaWithAlreadyInProgress() throws Exception {
1611         testServerChangeWhenAlreadyInProgress("astronauts", new AddShardReplica("astronauts"),
1612                 AddServer.class, new AddShardReplica("astronauts"));
1613     }
1614
1615     @Test
1616     public void testAddShardReplicaWithFindPrimaryTimeout() throws Exception {
1617         LOG.info("testAddShardReplicaWithFindPrimaryTimeout starting");
1618         new JavaTestKit(getSystem()) {{
1619             MockConfiguration mockConfig = new MockConfiguration(ImmutableMap.<String, List<String>>builder().
1620                        put("astronauts", Arrays.asList("member-2")).build());
1621
1622             final ActorRef newReplicaShardManager = actorFactory.createActor(newTestShardMgrBuilder(mockConfig).
1623                     shardActor(mockShardActor).props(), shardMgrID);
1624
1625             newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1626             MockClusterWrapper.sendMemberUp(newReplicaShardManager, "member-2", getRef().path().toString());
1627
1628             newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
1629             Status.Failure resp = expectMsgClass(duration("5 seconds"), Status.Failure.class);
1630             assertEquals("Failure obtained", true,
1631                           (resp.cause() instanceof RuntimeException));
1632         }};
1633
1634         LOG.info("testAddShardReplicaWithFindPrimaryTimeout ending");
1635     }
1636
1637     @Test
1638     public void testRemoveShardReplicaForNonExistentShard() throws Exception {
1639         new JavaTestKit(getSystem()) {{
1640             ActorRef shardManager = actorFactory.createActor(newShardMgrProps(
1641                     new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
1642
1643             shardManager.tell(new RemoveShardReplica("model-inventory", "member-1"), getRef());
1644             Status.Failure resp = expectMsgClass(duration("10 seconds"), Status.Failure.class);
1645             assertEquals("Failure obtained", true,
1646                          (resp.cause() instanceof PrimaryNotFoundException));
1647         }};
1648     }
1649
1650     @Test
1651     /**
1652      * Primary is Local
1653      */
1654     public void testRemoveShardReplicaLocal() throws Exception {
1655         new JavaTestKit(getSystem()) {{
1656             String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
1657
1658             final TestActorRef<MockRespondActor> respondActor =
1659                     TestActorRef.create(getSystem(), Props.create(MockRespondActor.class), memberId);
1660
1661             ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor(respondActor));
1662
1663             shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1664             shardManager.tell(new ActorInitialized(), respondActor);
1665             shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, Optional.of(mock(DataTree.class)),
1666                     DataStoreVersions.CURRENT_VERSION), getRef());
1667             shardManager.tell((new RoleChangeNotification(memberId, RaftState.Candidate.name(),
1668                     RaftState.Leader.name())), respondActor);
1669
1670             respondActor.underlyingActor().updateResponse(new RemoveServerReply(ServerChangeStatus.OK, null));
1671             shardManager.tell(new RemoveShardReplica(Shard.DEFAULT_NAME, "member-1"), getRef());
1672             final RemoveServer removeServer = MessageCollectorActor.expectFirstMatching(respondActor, RemoveServer.class);
1673             assertEquals(new ShardIdentifier("default", "member-1", shardMrgIDSuffix).toString(),
1674                     removeServer.getServerId());
1675             expectMsgClass(duration("5 seconds"), Success.class);
1676         }};
1677     }
1678
1679     @Test
1680     public void testRemoveShardReplicaRemote() throws Exception {
1681         MockConfiguration mockConfig =
1682                 new MockConfiguration(ImmutableMap.<String, List<String>>builder().
1683                         put("default", Arrays.asList("member-1", "member-2")).
1684                         put("astronauts", Arrays.asList("member-1")).build());
1685
1686         String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
1687
1688         // Create an ActorSystem ShardManager actor for member-1.
1689         final ActorSystem system1 = newActorSystem("Member1");
1690         Cluster.get(system1).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
1691         ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
1692
1693         final TestActorRef<TestShardManager> newReplicaShardManager = TestActorRef.create(system1,
1694                 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockDefaultShardActor).cluster(
1695                         new ClusterWrapperImpl(system1)).props(),
1696                 shardManagerID);
1697
1698         // Create an ActorSystem ShardManager actor for member-2.
1699         final ActorSystem system2 = newActorSystem("Member2");
1700         Cluster.get(system2).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
1701
1702         String name = new ShardIdentifier("default", "member-2", shardMrgIDSuffix).toString();
1703         final TestActorRef<MockRespondActor> mockShardLeaderActor =
1704                 TestActorRef.create(system2, Props.create(MockRespondActor.class), name);
1705
1706         LOG.error("Mock Shard Leader Actor : {}", mockShardLeaderActor);
1707
1708         final TestActorRef<TestShardManager> leaderShardManager = TestActorRef.create(system2,
1709                 newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockShardLeaderActor).cluster(
1710                         new ClusterWrapperImpl(system2)).props(),
1711                 shardManagerID);
1712
1713         // Because mockShardLeaderActor is created at the top level of the actor system it has an address like so,
1714         //    akka.tcp://cluster-test@127.0.0.1:2559/user/member-2-shard-default-config1
1715         // However when a shard manager has a local shard which is a follower and a leader that is remote it will
1716         // try to compute an address for the remote shard leader using the ShardPeerAddressResolver. This address will
1717         // look like so,
1718         //    akka.tcp://cluster-test@127.0.0.1:2559/user/shardmanager-config1/member-2-shard-default-config1
1719         // In this specific case if we did a FindPrimary for shard default from member-1 we would come up
1720         // with the address of an actor which does not exist, therefore any message sent to that actor would go to
1721         // dead letters.
1722         // To work around this problem we create a ForwardingActor with the right address and pass to it the
1723         // mockShardLeaderActor. The ForwardingActor simply forwards all messages to the mockShardLeaderActor and every
1724         // thing works as expected
1725         final ActorRef actorRef = leaderShardManager.underlyingActor().context()
1726                 .actorOf(Props.create(ForwardingActor.class, mockShardLeaderActor), "member-2-shard-default-" + shardMrgIDSuffix);
1727
1728         LOG.error("Forwarding actor : {}", actorRef);
1729
1730         new JavaTestKit(system1) {{
1731
1732             newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1733             leaderShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1734
1735             leaderShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1736             newReplicaShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
1737
1738             String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
1739             short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
1740             leaderShardManager.tell(new ShardLeaderStateChanged(memberId2, memberId2,
1741                     Optional.of(mock(DataTree.class)), leaderVersion), mockShardLeaderActor);
1742             leaderShardManager.tell(new RoleChangeNotification(memberId2,
1743                     RaftState.Candidate.name(), RaftState.Leader.name()), mockShardLeaderActor);
1744
1745             String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
1746             newReplicaShardManager.tell(new ShardLeaderStateChanged(memberId1, memberId2,
1747                     Optional.of(mock(DataTree.class)), leaderVersion), mockShardActor);
1748             newReplicaShardManager.tell(new RoleChangeNotification(memberId1,
1749                     RaftState.Candidate.name(), RaftState.Follower.name()), mockShardActor);
1750
1751             newReplicaShardManager.underlyingActor().waitForMemberUp();
1752             leaderShardManager.underlyingActor().waitForMemberUp();
1753
1754             //construct a mock response message
1755             RemoveServerReply response = new RemoveServerReply(ServerChangeStatus.OK, memberId2);
1756             mockShardLeaderActor.underlyingActor().updateResponse(response);
1757             newReplicaShardManager.tell(new RemoveShardReplica("default", "member-1"), getRef());
1758             RemoveServer removeServer = MessageCollectorActor.expectFirstMatching(mockShardLeaderActor,
1759                     RemoveServer.class);
1760             String removeServerId = new ShardIdentifier("default", "member-1", shardMrgIDSuffix).toString();
1761             assertEquals("RemoveServer serverId", removeServerId, removeServer.getServerId());
1762             expectMsgClass(duration("5 seconds"), Status.Success.class);
1763         }};
1764
1765     }
1766
1767     @Test
1768     public void testRemoveShardReplicaWhenAnotherRemoveShardReplicaAlreadyInProgress() throws Exception {
1769         testServerChangeWhenAlreadyInProgress("astronauts", new RemoveShardReplica("astronauts", "member-2"),
1770                 RemoveServer.class, new RemoveShardReplica("astronauts", "member-3"));
1771     }
1772
1773     @Test
1774     public void testRemoveShardReplicaWhenAddShardReplicaAlreadyInProgress() throws Exception {
1775         testServerChangeWhenAlreadyInProgress("astronauts", new AddShardReplica("astronauts"),
1776                 AddServer.class, new RemoveShardReplica("astronauts", "member-2"));
1777     }
1778
1779
1780     public void testServerChangeWhenAlreadyInProgress(final String shardName, final Object firstServerChange,
1781                                                       final Class firstForwardedServerChangeClass,
1782                                                       final Object secondServerChange) throws Exception {
1783         new JavaTestKit(getSystem()) {{
1784             JavaTestKit mockShardLeaderKit = new JavaTestKit(getSystem());
1785             JavaTestKit secondRequestKit = new JavaTestKit(getSystem());
1786
1787             MockConfiguration mockConfig =
1788                     new MockConfiguration(ImmutableMap.<String, List<String>>builder().
1789                             put(shardName, Arrays.asList("member-2")).build());
1790
1791             final TestActorRef<TestShardManager> shardManager = TestActorRef.create(getSystem(),
1792                     newTestShardMgrBuilder().configuration(mockConfig).shardActor(mockShardActor).cluster(
1793                             new MockClusterWrapper()).props(),
1794                     shardMgrID);
1795
1796             shardManager.underlyingActor().setMessageInterceptor(newFindPrimaryInterceptor(mockShardLeaderKit.getRef()));
1797
1798             shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1799
1800             shardManager.tell(firstServerChange, getRef());
1801
1802             mockShardLeaderKit.expectMsgClass(firstForwardedServerChangeClass);
1803
1804             shardManager.tell(secondServerChange, secondRequestKit.getRef());
1805
1806             secondRequestKit.expectMsgClass(duration("5 seconds"), Failure.class);
1807         }};
1808     }
1809
1810     @Test
1811     public void testServerRemovedShardActorNotRunning() throws Exception {
1812         LOG.info("testServerRemovedShardActorNotRunning starting");
1813         new JavaTestKit(getSystem()) {{
1814             MockConfiguration mockConfig =
1815                     new MockConfiguration(ImmutableMap.<String, List<String>>builder().
1816                             put("default", Arrays.asList("member-1", "member-2")).
1817                             put("astronauts", Arrays.asList("member-2")).
1818                             put("people", Arrays.asList("member-1", "member-2")).build());
1819
1820             TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(newShardMgrProps(mockConfig));
1821
1822             shardManager.underlyingActor().waitForRecoveryComplete();
1823             shardManager.tell(new FindLocalShard("people", false), getRef());
1824             expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1825
1826             shardManager.tell(new FindLocalShard("default", false), getRef());
1827             expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1828
1829             // Removed the default shard replica from member-1
1830             ShardIdentifier.Builder builder = new ShardIdentifier.Builder();
1831             ShardIdentifier shardId = builder.shardName("default").memberName("member-1").type(shardMrgIDSuffix).build();
1832             shardManager.tell(new ServerRemoved(shardId.toString()), getRef());
1833
1834             shardManager.underlyingActor().verifySnapshotPersisted(Sets.newHashSet("people"));
1835         }};
1836
1837         LOG.info("testServerRemovedShardActorNotRunning ending");
1838     }
1839
1840     @Test
1841     public void testServerRemovedShardActorRunning() throws Exception {
1842         LOG.info("testServerRemovedShardActorRunning starting");
1843         new JavaTestKit(getSystem()) {{
1844             MockConfiguration mockConfig =
1845                     new MockConfiguration(ImmutableMap.<String, List<String>>builder().
1846                             put("default", Arrays.asList("member-1", "member-2")).
1847                             put("astronauts", Arrays.asList("member-2")).
1848                             put("people", Arrays.asList("member-1", "member-2")).build());
1849
1850             String shardId = ShardIdentifier.builder().shardName("default").memberName("member-1").
1851                     type(shardMrgIDSuffix).build().toString();
1852             TestActorRef<MessageCollectorActor> shard = actorFactory.createTestActor(
1853                     MessageCollectorActor.props(), shardId);
1854
1855             TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(
1856                     newTestShardMgrBuilder(mockConfig).addShardActor("default", shard).props());
1857
1858             watch(shard);
1859
1860             shardManager.underlyingActor().waitForRecoveryComplete();
1861
1862             shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1863             shardManager.tell(new ActorInitialized(), shard);
1864
1865             waitForShardInitialized(shardManager, "people", this);
1866             waitForShardInitialized(shardManager, "default", this);
1867
1868             // Removed the default shard replica from member-1
1869             shardManager.tell(new ServerRemoved(shardId), getRef());
1870
1871             shardManager.underlyingActor().verifySnapshotPersisted(Sets.newHashSet("people"));
1872
1873             expectMsgClass(duration("5 seconds"), Terminated.class);
1874         }};
1875
1876         LOG.info("testServerRemovedShardActorRunning ending");
1877     }
1878
1879
1880     @Test
1881     public void testShardPersistenceWithRestoredData() throws Exception {
1882         LOG.info("testShardPersistenceWithRestoredData starting");
1883         new JavaTestKit(getSystem()) {{
1884             MockConfiguration mockConfig =
1885                 new MockConfiguration(ImmutableMap.<String, List<String>>builder().
1886                    put("default", Arrays.asList("member-1", "member-2")).
1887                    put("astronauts", Arrays.asList("member-2")).
1888                    put("people", Arrays.asList("member-1", "member-2")).build());
1889             String[] restoredShards = {"default", "astronauts"};
1890             ShardManagerSnapshot snapshot = new ShardManagerSnapshot(Arrays.asList(restoredShards));
1891             InMemorySnapshotStore.addSnapshot("shard-manager-" + shardMrgIDSuffix, snapshot);
1892
1893             //create shardManager to come up with restored data
1894             TestActorRef<TestShardManager> newRestoredShardManager = actorFactory.createTestActor(
1895                     newShardMgrProps(mockConfig));
1896
1897             newRestoredShardManager.underlyingActor().waitForRecoveryComplete();
1898
1899             newRestoredShardManager.tell(new FindLocalShard("people", false), getRef());
1900             LocalShardNotFound notFound = expectMsgClass(duration("5 seconds"), LocalShardNotFound.class);
1901             assertEquals("for uninitialized shard", "people", notFound.getShardName());
1902
1903             //Verify a local shard is created for the restored shards,
1904             //although we expect a NotInitializedException for the shards as the actor initialization
1905             //message is not sent for them
1906             newRestoredShardManager.tell(new FindLocalShard("default", false), getRef());
1907             expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1908
1909             newRestoredShardManager.tell(new FindLocalShard("astronauts", false), getRef());
1910             expectMsgClass(duration("5 seconds"), NotInitializedException.class);
1911         }};
1912
1913         LOG.info("testShardPersistenceWithRestoredData ending");
1914     }
1915
1916     @Test
1917     public void testShutDown() throws Exception {
1918         LOG.info("testShutDown starting");
1919         new JavaTestKit(getSystem()) {{
1920             MockConfiguration mockConfig =
1921                     new MockConfiguration(ImmutableMap.<String, List<String>>builder().
1922                             put("shard1", Arrays.asList("member-1")).
1923                             put("shard2", Arrays.asList("member-1")).build());
1924
1925             String shardId1 = ShardIdentifier.builder().shardName("shard1").memberName("member-1").
1926                     type(shardMrgIDSuffix).build().toString();
1927             TestActorRef<MessageCollectorActor> shard1 = actorFactory.createTestActor(
1928                     MessageCollectorActor.props(), shardId1);
1929
1930             String shardId2 = ShardIdentifier.builder().shardName("shard2").memberName("member-1").
1931                     type(shardMrgIDSuffix).build().toString();
1932             TestActorRef<MessageCollectorActor> shard2 = actorFactory.createTestActor(
1933                     MessageCollectorActor.props(), shardId2);
1934
1935             TestActorRef<TestShardManager> shardManager = actorFactory.createTestActor(newTestShardMgrBuilder(
1936                     mockConfig).addShardActor("shard1", shard1).addShardActor("shard2", shard2).props());
1937
1938             shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
1939             shardManager.tell(new ActorInitialized(), shard1);
1940             shardManager.tell(new ActorInitialized(), shard2);
1941
1942             FiniteDuration duration = FiniteDuration.create(5, TimeUnit.SECONDS);
1943             Future<Boolean> stopFuture = Patterns.gracefulStop(shardManager, duration, new Shutdown());
1944
1945             MessageCollectorActor.expectFirstMatching(shard1, Shutdown.class);
1946             MessageCollectorActor.expectFirstMatching(shard2, Shutdown.class);
1947
1948             try {
1949                 Await.ready(stopFuture, FiniteDuration.create(500, TimeUnit.MILLISECONDS));
1950                 fail("ShardManager actor stopped without waiting for the Shards to be stopped");
1951             } catch(TimeoutException e) {
1952                 // expected
1953             }
1954
1955             actorFactory.killActor(shard1, this);
1956             actorFactory.killActor(shard2, this);
1957
1958             Boolean stopped = Await.result(stopFuture, duration);
1959             assertEquals("Stopped", Boolean.TRUE, stopped);
1960         }};
1961
1962         LOG.info("testShutDown ending");
1963     }
1964
1965     private static class TestShardManager extends ShardManager {
1966         private final CountDownLatch recoveryComplete = new CountDownLatch(1);
1967         private final CountDownLatch snapshotPersist = new CountDownLatch(1);
1968         private ShardManagerSnapshot snapshot;
1969         private final Map<String, ActorRef> shardActors;
1970         private final ActorRef shardActor;
1971         private CountDownLatch findPrimaryMessageReceived = new CountDownLatch(1);
1972         private CountDownLatch memberUpReceived = new CountDownLatch(1);
1973         private CountDownLatch memberRemovedReceived = new CountDownLatch(1);
1974         private CountDownLatch memberUnreachableReceived = new CountDownLatch(1);
1975         private CountDownLatch memberReachableReceived = new CountDownLatch(1);
1976         private volatile MessageInterceptor messageInterceptor;
1977
1978         private TestShardManager(Builder builder) {
1979             super(builder);
1980             shardActor = builder.shardActor;
1981             shardActors = builder.shardActors;
1982         }
1983
1984         @Override
1985         public void handleRecover(Object message) throws Exception {
1986             try {
1987                 super.handleRecover(message);
1988             } finally {
1989                 if(message instanceof RecoveryCompleted) {
1990                     recoveryComplete.countDown();
1991                 }
1992             }
1993         }
1994
1995         @Override
1996         public void handleCommand(Object message) throws Exception {
1997             try{
1998                 if(messageInterceptor != null && messageInterceptor.canIntercept(message)) {
1999                     getSender().tell(messageInterceptor.apply(message), getSelf());
2000                 } else {
2001                     super.handleCommand(message);
2002                 }
2003             } finally {
2004                 if(message instanceof FindPrimary) {
2005                     findPrimaryMessageReceived.countDown();
2006                 } else if(message instanceof ClusterEvent.MemberUp) {
2007                     String role = ((ClusterEvent.MemberUp)message).member().roles().head();
2008                     if(!getCluster().getCurrentMemberName().equals(role)) {
2009                         memberUpReceived.countDown();
2010                     }
2011                 } else if(message instanceof ClusterEvent.MemberRemoved) {
2012                     String role = ((ClusterEvent.MemberRemoved)message).member().roles().head();
2013                     if(!getCluster().getCurrentMemberName().equals(role)) {
2014                         memberRemovedReceived.countDown();
2015                     }
2016                 } else if(message instanceof ClusterEvent.UnreachableMember) {
2017                     String role = ((ClusterEvent.UnreachableMember)message).member().roles().head();
2018                     if(!getCluster().getCurrentMemberName().equals(role)) {
2019                         memberUnreachableReceived.countDown();
2020                     }
2021                 } else if(message instanceof ClusterEvent.ReachableMember) {
2022                     String role = ((ClusterEvent.ReachableMember)message).member().roles().head();
2023                     if(!getCluster().getCurrentMemberName().equals(role)) {
2024                         memberReachableReceived.countDown();
2025                     }
2026                 }
2027             }
2028         }
2029
2030         void setMessageInterceptor(MessageInterceptor messageInterceptor) {
2031             this.messageInterceptor = messageInterceptor;
2032         }
2033
2034         void waitForRecoveryComplete() {
2035             assertEquals("Recovery complete", true,
2036                     Uninterruptibles.awaitUninterruptibly(recoveryComplete, 5, TimeUnit.SECONDS));
2037         }
2038
2039         void waitForMemberUp() {
2040             assertEquals("MemberUp received", true,
2041                     Uninterruptibles.awaitUninterruptibly(memberUpReceived, 5, TimeUnit.SECONDS));
2042             memberUpReceived = new CountDownLatch(1);
2043         }
2044
2045         void waitForMemberRemoved() {
2046             assertEquals("MemberRemoved received", true,
2047                     Uninterruptibles.awaitUninterruptibly(memberRemovedReceived, 5, TimeUnit.SECONDS));
2048             memberRemovedReceived = new CountDownLatch(1);
2049         }
2050
2051         void waitForUnreachableMember() {
2052             assertEquals("UnreachableMember received", true,
2053                 Uninterruptibles.awaitUninterruptibly(memberUnreachableReceived, 5, TimeUnit.SECONDS
2054                 ));
2055             memberUnreachableReceived = new CountDownLatch(1);
2056         }
2057
2058         void waitForReachableMember() {
2059             assertEquals("ReachableMember received", true,
2060                 Uninterruptibles.awaitUninterruptibly(memberReachableReceived, 5, TimeUnit.SECONDS));
2061             memberReachableReceived = new CountDownLatch(1);
2062         }
2063
2064         void verifyFindPrimary() {
2065             assertEquals("FindPrimary received", true,
2066                     Uninterruptibles.awaitUninterruptibly(findPrimaryMessageReceived, 5, TimeUnit.SECONDS));
2067             findPrimaryMessageReceived = new CountDownLatch(1);
2068         }
2069
2070         public static Builder builder(DatastoreContext.Builder datastoreContextBuilder) {
2071             return new Builder(datastoreContextBuilder);
2072         }
2073
2074         private static class Builder extends AbstractGenericBuilder<Builder, TestShardManager> {
2075             private ActorRef shardActor;
2076             private final Map<String, ActorRef> shardActors = new HashMap<>();
2077
2078             Builder(DatastoreContext.Builder datastoreContextBuilder) {
2079                 super(TestShardManager.class);
2080                 datastoreContextFactory(newDatastoreContextFactory(datastoreContextBuilder.build()));
2081             }
2082
2083             Builder shardActor(ActorRef shardActor) {
2084                 this.shardActor = shardActor;
2085                 return this;
2086             }
2087
2088             Builder addShardActor(String shardName, ActorRef actorRef){
2089                 shardActors.put(shardName, actorRef);
2090                 return this;
2091             }
2092         }
2093
2094         @Override
2095         public void saveSnapshot(Object obj) {
2096             snapshot = (ShardManagerSnapshot) obj;
2097             snapshotPersist.countDown();
2098             super.saveSnapshot(obj);
2099         }
2100
2101         void verifySnapshotPersisted(Set<String> shardList) {
2102             assertEquals("saveSnapshot invoked", true,
2103                     Uninterruptibles.awaitUninterruptibly(snapshotPersist, 5, TimeUnit.SECONDS));
2104             assertEquals("Shard Persisted", shardList, Sets.newHashSet(snapshot.getShardList()));
2105         }
2106
2107         @Override
2108         protected ActorRef newShardActor(SchemaContext schemaContext, ShardInformation info) {
2109             if(shardActors.get(info.getShardName()) != null){
2110                 return shardActors.get(info.getShardName());
2111             }
2112
2113             if(shardActor != null) {
2114                 return shardActor;
2115             }
2116
2117             return super.newShardActor(schemaContext, info);
2118         }
2119     }
2120
2121     private static abstract class AbstractGenericBuilder<T extends AbstractGenericBuilder<T, ?>, C extends ShardManager>
2122                                                      extends ShardManager.AbstractBuilder<T> {
2123         private final Class<C> shardManagerClass;
2124
2125         AbstractGenericBuilder(Class<C> shardManagerClass) {
2126             this.shardManagerClass = shardManagerClass;
2127             cluster(new MockClusterWrapper()).configuration(new MockConfiguration()).
2128                     waitTillReadyCountdownLatch(ready).primaryShardInfoCache(new PrimaryShardInfoFutureCache());
2129         }
2130
2131         @Override
2132         public Props props() {
2133             verify();
2134             return Props.create(shardManagerClass, this);
2135         }
2136     }
2137
2138     private static class GenericBuilder<C extends ShardManager> extends AbstractGenericBuilder<GenericBuilder<C>, C> {
2139         GenericBuilder(Class<C> shardManagerClass) {
2140             super(shardManagerClass);
2141         }
2142     }
2143
2144     private static class DelegatingShardManagerCreator implements Creator<ShardManager> {
2145         private static final long serialVersionUID = 1L;
2146         private final Creator<ShardManager> delegate;
2147
2148         public DelegatingShardManagerCreator(Creator<ShardManager> delegate) {
2149             this.delegate = delegate;
2150         }
2151
2152         @Override
2153         public ShardManager create() throws Exception {
2154             return delegate.create();
2155         }
2156     }
2157
2158     interface MessageInterceptor extends Function<Object, Object> {
2159         boolean canIntercept(Object message);
2160     }
2161
2162     private MessageInterceptor newFindPrimaryInterceptor(final ActorRef primaryActor) {
2163         return new MessageInterceptor(){
2164             @Override
2165             public Object apply(Object message) {
2166                 return new RemotePrimaryShardFound(Serialization.serializedActorPath(primaryActor), (short) 1);
2167             }
2168
2169             @Override
2170             public boolean canIntercept(Object message) {
2171                 return message instanceof FindPrimary;
2172             }
2173         };
2174     }
2175
2176     private static class MockRespondActor extends MessageCollectorActor {
2177         static final String CLEAR_RESPONSE = "clear-response";
2178         static final org.slf4j.Logger LOG = LoggerFactory.getLogger(MockRespondActor.class);
2179
2180         private volatile Object responseMsg;
2181
2182         @SuppressWarnings("unused")
2183         public MockRespondActor() {
2184         }
2185
2186         @SuppressWarnings("unused")
2187         public MockRespondActor(Object responseMsg) {
2188             this.responseMsg = responseMsg;
2189         }
2190
2191         public void updateResponse(Object response) {
2192             responseMsg = response;
2193         }
2194
2195         @Override
2196         public void onReceive(Object message) throws Exception {
2197             if(!"get-all-messages".equals(message)) {
2198                 LOG.debug("Received message : {}", message);
2199             }
2200             super.onReceive(message);
2201             if (message instanceof AddServer && responseMsg != null) {
2202                 getSender().tell(responseMsg, getSelf());
2203             } else if(message instanceof RemoveServer && responseMsg != null){
2204                 getSender().tell(responseMsg, getSelf());
2205             } else if(message.equals(CLEAR_RESPONSE)) {
2206                 responseMsg = null;
2207             }
2208         }
2209     }
2210 }