Bug 4149: Implement per-shard DatastoreContext settings
[controller.git] / opendaylight / md-sal / sal-distributed-datastore / src / test / java / org / opendaylight / controller / cluster / datastore / ShardManagerTest.java
index 33d5a2ed981efbf4fb3260440866e339fbf9c949..6f3946d42219ed963b0ed03da2448a675165cf12 100644 (file)
@@ -1,7 +1,15 @@
+/*
+ * Copyright (c) 2014, 2015 Cisco Systems, Inc. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+
 package org.opendaylight.controller.cluster.datastore;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
@@ -9,11 +17,12 @@ import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
 import akka.actor.ActorRef;
 import akka.actor.ActorSystem;
 import akka.actor.AddressFromURIString;
+import akka.actor.PoisonPill;
 import akka.actor.Props;
+import akka.actor.Status;
 import akka.cluster.Cluster;
 import akka.cluster.ClusterEvent;
 import akka.dispatch.Dispatchers;
@@ -30,36 +39,48 @@ import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.Uninterruptibles;
 import com.typesafe.config.ConfigFactory;
 import java.net.URI;
+import java.util.AbstractMap;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.HashSet;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
+import java.util.Map.Entry;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import org.mockito.Mock;
+import org.mockito.Mockito;
 import org.mockito.MockitoAnnotations;
-import org.opendaylight.controller.cluster.DataPersistenceProvider;
+import org.opendaylight.controller.cluster.datastore.config.Configuration;
+import org.opendaylight.controller.cluster.datastore.config.ConfigurationImpl;
+import org.opendaylight.controller.cluster.datastore.config.EmptyModuleShardConfigProvider;
+import org.opendaylight.controller.cluster.datastore.config.ModuleShardConfiguration;
 import org.opendaylight.controller.cluster.datastore.exceptions.NoShardLeaderException;
 import org.opendaylight.controller.cluster.datastore.exceptions.NotInitializedException;
 import org.opendaylight.controller.cluster.datastore.exceptions.PrimaryNotFoundException;
 import org.opendaylight.controller.cluster.datastore.identifiers.ShardIdentifier;
 import org.opendaylight.controller.cluster.datastore.identifiers.ShardManagerIdentifier;
 import org.opendaylight.controller.cluster.datastore.messages.ActorInitialized;
+import org.opendaylight.controller.cluster.datastore.messages.AddShardReplica;
+import org.opendaylight.controller.cluster.datastore.messages.CreateShard;
+import org.opendaylight.controller.cluster.datastore.messages.CreateShardReply;
 import org.opendaylight.controller.cluster.datastore.messages.FindLocalShard;
 import org.opendaylight.controller.cluster.datastore.messages.FindPrimary;
 import org.opendaylight.controller.cluster.datastore.messages.LocalPrimaryShardFound;
 import org.opendaylight.controller.cluster.datastore.messages.LocalShardFound;
 import org.opendaylight.controller.cluster.datastore.messages.LocalShardNotFound;
+import org.opendaylight.controller.cluster.datastore.messages.PeerDown;
+import org.opendaylight.controller.cluster.datastore.messages.PeerUp;
 import org.opendaylight.controller.cluster.datastore.messages.PrimaryShardInfo;
 import org.opendaylight.controller.cluster.datastore.messages.RemotePrimaryShardFound;
+import org.opendaylight.controller.cluster.datastore.messages.RemoveShardReplica;
 import org.opendaylight.controller.cluster.datastore.messages.ShardLeaderStateChanged;
+import org.opendaylight.controller.cluster.datastore.messages.SwitchShardBehavior;
 import org.opendaylight.controller.cluster.datastore.messages.UpdateSchemaContext;
-import org.opendaylight.controller.cluster.datastore.utils.MessageCollectorActor;
 import org.opendaylight.controller.cluster.datastore.utils.MockClusterWrapper;
 import org.opendaylight.controller.cluster.datastore.utils.MockConfiguration;
 import org.opendaylight.controller.cluster.datastore.utils.PrimaryShardInfoFutureCache;
@@ -68,10 +89,14 @@ import org.opendaylight.controller.cluster.notifications.RegisterRoleChangeListe
 import org.opendaylight.controller.cluster.notifications.RoleChangeNotification;
 import org.opendaylight.controller.cluster.raft.RaftState;
 import org.opendaylight.controller.cluster.raft.base.messages.FollowerInitialSyncUpStatus;
+import org.opendaylight.controller.cluster.raft.base.messages.SwitchBehavior;
+import org.opendaylight.controller.cluster.raft.messages.AddServer;
+import org.opendaylight.controller.cluster.raft.messages.AddServerReply;
+import org.opendaylight.controller.cluster.raft.messages.ServerChangeStatus;
 import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal;
+import org.opendaylight.controller.cluster.raft.utils.MessageCollectorActor;
 import org.opendaylight.controller.md.cluster.datastore.model.TestModel;
 import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTree;
-import org.opendaylight.yangtools.yang.model.api.ModuleIdentifier;
 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
 import scala.concurrent.Await;
 import scala.concurrent.Future;
@@ -88,8 +113,11 @@ public class ShardManagerTest extends AbstractActorTest {
 
     private static TestActorRef<MessageCollectorActor> mockShardActor;
 
+    private static String mockShardName;
+
     private final DatastoreContext.Builder datastoreContextBuilder = DatastoreContext.newBuilder().
-            dataStoreType(shardMrgIDSuffix).shardInitializationTimeout(600, TimeUnit.MILLISECONDS);
+            dataStoreType(shardMrgIDSuffix).shardInitializationTimeout(600, TimeUnit.MILLISECONDS)
+                   .shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(6);
 
     private static ActorRef newMockShardActor(ActorSystem system, String shardName, String memberName) {
         String name = new ShardIdentifier(shardName, memberName,"config").toString();
@@ -105,8 +133,8 @@ public class ShardManagerTest extends AbstractActorTest {
         InMemoryJournal.clear();
 
         if(mockShardActor == null) {
-            String name = new ShardIdentifier(Shard.DEFAULT_NAME, "member-1", "config").toString();
-            mockShardActor = TestActorRef.create(getSystem(), Props.create(MessageCollectorActor.class), name);
+            mockShardName = new ShardIdentifier(Shard.DEFAULT_NAME, "member-1", "config").toString();
+            mockShardActor = TestActorRef.create(getSystem(), Props.create(MessageCollectorActor.class), mockShardName);
         }
 
         mockShardActor.underlyingActor().clear();
@@ -117,9 +145,20 @@ public class ShardManagerTest extends AbstractActorTest {
         InMemoryJournal.clear();
     }
 
-    private Props newShardMgrProps(boolean persistent) {
-        return ShardManager.props(new MockClusterWrapper(), new MockConfiguration(),
-                datastoreContextBuilder.persistent(persistent).build(), ready, primaryShardInfoCache);
+    private Props newShardMgrProps() {
+        return newShardMgrProps(new MockConfiguration());
+    }
+
+    private static DatastoreContextFactory newDatastoreContextFactory(DatastoreContext datastoreContext) {
+        DatastoreContextFactory mockFactory = mock(DatastoreContextFactory.class);
+        Mockito.doReturn(datastoreContext).when(mockFactory).getBaseDatastoreContext();
+        Mockito.doReturn(datastoreContext).when(mockFactory).getShardDatastoreContext(Mockito.anyString());
+        return mockFactory;
+    }
+
+    private Props newShardMgrProps(Configuration config) {
+        return ShardManager.props(new MockClusterWrapper(), config,
+                newDatastoreContextFactory(datastoreContextBuilder.build()), ready, primaryShardInfoCache);
     }
 
     private Props newPropsShardMgrWithMockShardActor() {
@@ -133,14 +172,102 @@ public class ShardManagerTest extends AbstractActorTest {
             private static final long serialVersionUID = 1L;
             @Override
             public ShardManager create() throws Exception {
-                return new ForwardingShardManager(clusterWrapper, config, datastoreContextBuilder.build(),
-                        ready, name, shardActor, primaryShardInfoCache);
+                return new ForwardingShardManager(clusterWrapper, config, newDatastoreContextFactory(
+                        datastoreContextBuilder.build()), ready, name, shardActor, primaryShardInfoCache);
             }
         };
 
         return Props.create(new DelegatingShardManagerCreator(creator)).withDispatcher(Dispatchers.DefaultDispatcherId());
     }
 
+    @Test
+    public void testPerShardDatastoreContext() throws Exception {
+        final DatastoreContextFactory mockFactory = newDatastoreContextFactory(
+                datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
+
+        Mockito.doReturn(DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).
+                shardElectionTimeoutFactor(6).build()).when(mockFactory).getShardDatastoreContext("default");
+
+        Mockito.doReturn(DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).
+                shardElectionTimeoutFactor(7).build()).when(mockFactory).getShardDatastoreContext("topology");
+
+        final MockConfiguration mockConfig = new MockConfiguration() {
+            @Override
+            public Collection<String> getMemberShardNames(String memberName) {
+                return Arrays.asList("default", "topology");
+            }
+
+            @Override
+            public Collection<String> getMembersFromShardName(String shardName) {
+                return Arrays.asList("member-1");
+            }
+        };
+
+        final TestActorRef<MessageCollectorActor> defaultShardActor = TestActorRef.create(getSystem(),
+                Props.create(MessageCollectorActor.class), "default");
+        final TestActorRef<MessageCollectorActor> topologyShardActor = TestActorRef.create(getSystem(),
+                Props.create(MessageCollectorActor.class), "topology");
+
+        final Map<String, Entry<ActorRef, DatastoreContext>> shardInfoMap = Collections.synchronizedMap(
+                new HashMap<String, Entry<ActorRef, DatastoreContext>>());
+        shardInfoMap.put("default", new AbstractMap.SimpleEntry<ActorRef, DatastoreContext>(defaultShardActor, null));
+        shardInfoMap.put("topology", new AbstractMap.SimpleEntry<ActorRef, DatastoreContext>(topologyShardActor, null));
+
+        final CountDownLatch newShardActorLatch = new CountDownLatch(2);
+        final Creator<ShardManager> creator = new Creator<ShardManager>() {
+            private static final long serialVersionUID = 1L;
+            @Override
+            public ShardManager create() throws Exception {
+                return new ShardManager(new MockClusterWrapper(), mockConfig, mockFactory, ready, primaryShardInfoCache) {
+                    @Override
+                    protected ActorRef newShardActor(SchemaContext schemaContext, ShardInformation info) {
+                        Entry<ActorRef, DatastoreContext> entry = shardInfoMap.get(info.getShardName());
+                        ActorRef ref = null;
+                        if(entry != null) {
+                            ref = entry.getKey();
+                            entry.setValue(info.getDatastoreContext());
+                        }
+
+                        newShardActorLatch.countDown();
+                        return ref;
+                    }
+                };
+            }
+        };
+
+        JavaTestKit kit = new JavaTestKit(getSystem());
+
+        final ActorRef shardManager = getSystem().actorOf(Props.create(new DelegatingShardManagerCreator(creator)).
+                    withDispatcher(Dispatchers.DefaultDispatcherId()));
+
+        shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), kit.getRef());
+
+        assertEquals("Shard actors created", true, newShardActorLatch.await(5, TimeUnit.SECONDS));
+        assertEquals("getShardElectionTimeoutFactor", 6, shardInfoMap.get("default").getValue().
+                getShardElectionTimeoutFactor());
+        assertEquals("getShardElectionTimeoutFactor", 7, shardInfoMap.get("topology").getValue().
+                getShardElectionTimeoutFactor());
+
+        DatastoreContextFactory newMockFactory = newDatastoreContextFactory(
+                datastoreContextBuilder.shardElectionTimeoutFactor(5).build());
+        Mockito.doReturn(DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).
+                shardElectionTimeoutFactor(66).build()).when(newMockFactory).getShardDatastoreContext("default");
+
+        Mockito.doReturn(DatastoreContext.newBuilderFrom(datastoreContextBuilder.build()).
+                shardElectionTimeoutFactor(77).build()).when(newMockFactory).getShardDatastoreContext("topology");
+
+        shardManager.tell(newMockFactory, kit.getRef());
+
+        DatastoreContext newContext = MessageCollectorActor.expectFirstMatching(defaultShardActor, DatastoreContext.class);
+        assertEquals("getShardElectionTimeoutFactor", 66, newContext.getShardElectionTimeoutFactor());
+
+        newContext = MessageCollectorActor.expectFirstMatching(topologyShardActor, DatastoreContext.class);
+        assertEquals("getShardElectionTimeoutFactor", 77, newContext.getShardElectionTimeoutFactor());
+
+        defaultShardActor.tell(PoisonPill.getInstance(), ActorRef.noSender());
+        topologyShardActor.tell(PoisonPill.getInstance(), ActorRef.noSender());
+    }
+
     @Test
     public void testOnReceiveFindPrimaryForNonExistentShard() throws Exception {
         new JavaTestKit(getSystem()) {{
@@ -165,7 +292,8 @@ public class ShardManagerTest extends AbstractActorTest {
             shardManager.tell(new ActorInitialized(), mockShardActor);
 
             DataTree mockDataTree = mock(DataTree.class);
-            shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, Optional.of(mockDataTree)), getRef());
+            shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, Optional.of(mockDataTree),
+                    DataStoreVersions.CURRENT_VERSION), getRef());
 
             MessageCollectorActor.expectFirstMatching(mockShardActor, RegisterRoleChangeListener.class);
             shardManager.tell((new RoleChangeNotification(memberId, RaftState.Candidate.name(),
@@ -192,7 +320,7 @@ public class ShardManagerTest extends AbstractActorTest {
             String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
             shardManager.tell(new RoleChangeNotification(memberId1,
                     RaftState.Candidate.name(), RaftState.Follower.name()), mockShardActor);
-            shardManager.tell(new LeaderStateChanged(memberId1, memberId2), mockShardActor);
+            shardManager.tell(new LeaderStateChanged(memberId1, memberId2, DataStoreVersions.CURRENT_VERSION), mockShardActor);
 
             shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 
@@ -214,13 +342,16 @@ public class ShardManagerTest extends AbstractActorTest {
             String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
             shardManager.tell(new RoleChangeNotification(memberId1,
                     RaftState.Candidate.name(), RaftState.Follower.name()), mockShardActor);
-            shardManager.tell(new ShardLeaderStateChanged(memberId1, memberId2, Optional.<DataTree>absent()), mockShardActor);
+            short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
+            shardManager.tell(new ShardLeaderStateChanged(memberId1, memberId2, Optional.<DataTree>absent(),
+                    leaderVersion), mockShardActor);
 
             shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 
             RemotePrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
             assertTrue("Unexpected primary path " +  primaryFound.getPrimaryPath(),
                     primaryFound.getPrimaryPath().contains("member-2-shard-default"));
+            assertEquals("getPrimaryVersion", leaderVersion, primaryFound.getPrimaryVersion());
         }};
     }
 
@@ -266,7 +397,8 @@ public class ShardManagerTest extends AbstractActorTest {
             expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
 
             DataTree mockDataTree = mock(DataTree.class);
-            shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, Optional.of(mockDataTree)), mockShardActor);
+            shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, Optional.of(mockDataTree),
+                    DataStoreVersions.CURRENT_VERSION), mockShardActor);
 
             shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, false), getRef());
 
@@ -301,7 +433,8 @@ public class ShardManagerTest extends AbstractActorTest {
             expectNoMsg(FiniteDuration.create(150, TimeUnit.MILLISECONDS));
 
             DataTree mockDataTree = mock(DataTree.class);
-            shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, Optional.of(mockDataTree)), mockShardActor);
+            shardManager.tell(new ShardLeaderStateChanged(memberId, memberId, Optional.of(mockDataTree),
+                    DataStoreVersions.CURRENT_VERSION), mockShardActor);
 
             LocalPrimaryShardFound primaryFound = expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
             assertTrue("Unexpected primary path " +  primaryFound.getPrimaryPath(),
@@ -345,6 +478,22 @@ public class ShardManagerTest extends AbstractActorTest {
         }};
     }
 
+    @Test
+    public void testOnReceiveFindPrimaryWaitForReadyWithIsolatedLeaderShard() throws Exception {
+        new JavaTestKit(getSystem()) {{
+            final ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor());
+
+            shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
+            shardManager.tell(new ActorInitialized(), mockShardActor);
+            shardManager.tell(new RoleChangeNotification("member-1-shard-default-" + shardMrgIDSuffix,
+                    null, RaftState.IsolatedLeader.name()), mockShardActor);
+
+            shardManager.tell(new FindPrimary(Shard.DEFAULT_NAME, true), getRef());
+
+            expectMsgClass(duration("2 seconds"), NoShardLeaderException.class);
+        }};
+    }
+
     @Test
     public void testOnReceiveFindPrimaryWaitForReadyWithNoRoleShard() throws Exception {
         new JavaTestKit(getSystem()) {{
@@ -398,8 +547,9 @@ public class ShardManagerTest extends AbstractActorTest {
             shardManager2.tell(new ActorInitialized(), mockShardActor2);
 
             String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
+            short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
             shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2,
-                    Optional.of(mock(DataTree.class))), mockShardActor2);
+                    Optional.of(mock(DataTree.class)), leaderVersion), mockShardActor2);
             shardManager2.tell(new RoleChangeNotification(memberId2,
                     RaftState.Candidate.name(), RaftState.Leader.name()), mockShardActor2);
 
@@ -410,6 +560,7 @@ public class ShardManagerTest extends AbstractActorTest {
             RemotePrimaryShardFound found = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
             String path = found.getPrimaryPath();
             assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-astronauts-config"));
+            assertEquals("getPrimaryVersion", leaderVersion, found.getPrimaryVersion());
 
             shardManager2.underlyingActor().verifyFindPrimary();
 
@@ -466,10 +617,11 @@ public class ShardManagerTest extends AbstractActorTest {
             String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
             String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
             shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2,
-                Optional.of(mock(DataTree.class))), mockShardActor1);
+                Optional.of(mock(DataTree.class)), DataStoreVersions.CURRENT_VERSION), mockShardActor1);
             shardManager1.tell(new RoleChangeNotification(memberId1,
                 RaftState.Candidate.name(), RaftState.Follower.name()), mockShardActor1);
-            shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, Optional.of(mock(DataTree.class))),
+            shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, Optional.of(mock(DataTree.class)),
+                    DataStoreVersions.CURRENT_VERSION),
                 mockShardActor2);
             shardManager2.tell(new RoleChangeNotification(memberId2,
                 RaftState.Candidate.name(), RaftState.Leader.name()), mockShardActor2);
@@ -486,6 +638,15 @@ public class ShardManagerTest extends AbstractActorTest {
 
             shardManager1.underlyingActor().waitForUnreachableMember();
 
+            PeerDown peerDown = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
+            assertEquals("getMemberName", "member-2", peerDown.getMemberName());
+            MessageCollectorActor.clearMessages(mockShardActor1);
+
+            shardManager1.underlyingActor().onReceiveCommand(MockClusterWrapper.
+                    createMemberRemoved("member-2", "akka.tcp://cluster-test@127.0.0.1:2558"));
+
+            MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerDown.class);
+
             shardManager1.tell(new FindPrimary("default", true), getRef());
 
             expectMsgClass(duration("5 seconds"), NoShardLeaderException.class);
@@ -495,12 +656,21 @@ public class ShardManagerTest extends AbstractActorTest {
 
             shardManager1.underlyingActor().waitForReachableMember();
 
+            PeerUp peerUp = MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
+            assertEquals("getMemberName", "member-2", peerUp.getMemberName());
+            MessageCollectorActor.clearMessages(mockShardActor1);
+
             shardManager1.tell(new FindPrimary("default", true), getRef());
 
             RemotePrimaryShardFound found1 = expectMsgClass(duration("5 seconds"), RemotePrimaryShardFound.class);
             String path1 = found1.getPrimaryPath();
             assertTrue("Unexpected primary path " + path1, path1.contains("member-2-shard-default-config"));
 
+            shardManager1.underlyingActor().onReceiveCommand(MockClusterWrapper.
+                    createMemberUp("member-2", "akka.tcp://cluster-test@127.0.0.1:2558"));
+
+            MessageCollectorActor.expectFirstMatching(mockShardActor1, PeerUp.class);
+
         }};
 
         JavaTestKit.shutdownActorSystem(system1);
@@ -547,10 +717,11 @@ public class ShardManagerTest extends AbstractActorTest {
             String memberId2 = "member-2-shard-default-" + shardMrgIDSuffix;
             String memberId1 = "member-1-shard-default-" + shardMrgIDSuffix;
             shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId2,
-                Optional.of(mock(DataTree.class))), mockShardActor1);
+                Optional.of(mock(DataTree.class)), DataStoreVersions.CURRENT_VERSION), mockShardActor1);
             shardManager1.tell(new RoleChangeNotification(memberId1,
                 RaftState.Candidate.name(), RaftState.Follower.name()), mockShardActor1);
-            shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, Optional.of(mock(DataTree.class))),
+            shardManager2.tell(new ShardLeaderStateChanged(memberId2, memberId2, Optional.of(mock(DataTree.class)),
+                    DataStoreVersions.CURRENT_VERSION),
                 mockShardActor2);
             shardManager2.tell(new RoleChangeNotification(memberId2,
                 RaftState.Candidate.name(), RaftState.Leader.name()), mockShardActor2);
@@ -563,7 +734,7 @@ public class ShardManagerTest extends AbstractActorTest {
             assertTrue("Unexpected primary path " + path, path.contains("member-2-shard-default-config"));
 
             primaryShardInfoCache.putSuccessful("default", new PrimaryShardInfo(system1.actorSelection(
-                    mockShardActor1.path()), Optional.<DataTree>absent()));
+                    mockShardActor1.path()), DataStoreVersions.CURRENT_VERSION, Optional.<DataTree>absent()));
 
             shardManager1.underlyingActor().onReceiveCommand(MockClusterWrapper.
                 createUnreachableMember("member-2", "akka.tcp://cluster-test@127.0.0.1:2558"));
@@ -576,8 +747,8 @@ public class ShardManagerTest extends AbstractActorTest {
 
             assertNull("Expected primaryShardInfoCache entry removed", primaryShardInfoCache.getIfPresent("default"));
 
-            shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId1, Optional.of(mock(DataTree.class))),
-                mockShardActor1);
+            shardManager1.tell(new ShardLeaderStateChanged(memberId1, memberId1, Optional.of(mock(DataTree.class)),
+                    DataStoreVersions.CURRENT_VERSION), mockShardActor1);
             shardManager1.tell(new RoleChangeNotification(memberId1,
                 RaftState.Follower.name(), RaftState.Leader.name()), mockShardActor1);
 
@@ -674,175 +845,16 @@ public class ShardManagerTest extends AbstractActorTest {
             // Journal entries up to the last one should've been deleted
             Map<Long, Object> journal = InMemoryJournal.get(shardMgrID);
             synchronized (journal) {
-                assertEquals("Journal size", 1, journal.size());
-                assertEquals("Journal entry seq #", Long.valueOf(2), journal.keySet().iterator().next());
+                assertEquals("Journal size", 0, journal.size());
             }
         }};
     }
 
-    @Test
-    public void testOnRecoveryPreviouslyKnownModulesAreDiscovered() throws Exception {
-        final ImmutableSet<String> persistedModules = ImmutableSet.of("foo", "bar");
-        InMemoryJournal.addEntry(shardMgrID, 1L, new ShardManager.SchemaContextModules(
-                persistedModules));
-        new JavaTestKit(getSystem()) {{
-            TestActorRef<TestShardManager> shardManager = TestActorRef.create(getSystem(),
-                    Props.create(new TestShardManagerCreator(shardMrgIDSuffix)));
-
-            shardManager.underlyingActor().waitForRecoveryComplete();
-
-            Collection<String> knownModules = shardManager.underlyingActor().getKnownModules();
-
-            assertEquals("getKnownModules", persistedModules, Sets.newHashSet(knownModules));
-        }};
-    }
-
-    @Test
-    public void testOnUpdateSchemaContextUpdateKnownModulesIfTheyContainASuperSetOfTheKnownModules()
-            throws Exception {
-        new JavaTestKit(getSystem()) {{
-            final TestActorRef<ShardManager> shardManager =
-                    TestActorRef.create(getSystem(), newShardMgrProps(true));
-
-            assertEquals("getKnownModules size", 0, shardManager.underlyingActor().getKnownModules().size());
-
-            ModuleIdentifier foo = mock(ModuleIdentifier.class);
-            when(foo.getNamespace()).thenReturn(new URI("foo"));
-
-            Set<ModuleIdentifier> moduleIdentifierSet = new HashSet<>();
-            moduleIdentifierSet.add(foo);
-
-            SchemaContext schemaContext = mock(SchemaContext.class);
-            when(schemaContext.getAllModuleIdentifiers()).thenReturn(moduleIdentifierSet);
-
-            shardManager.underlyingActor().onReceiveCommand(new UpdateSchemaContext(schemaContext));
-
-            assertEquals("getKnownModules", Sets.newHashSet("foo"),
-                    Sets.newHashSet(shardManager.underlyingActor().getKnownModules()));
-
-            ModuleIdentifier bar = mock(ModuleIdentifier.class);
-            when(bar.getNamespace()).thenReturn(new URI("bar"));
-
-            moduleIdentifierSet.add(bar);
-
-            shardManager.underlyingActor().onReceiveCommand(new UpdateSchemaContext(schemaContext));
-
-            assertEquals("getKnownModules", Sets.newHashSet("foo", "bar"),
-                    Sets.newHashSet(shardManager.underlyingActor().getKnownModules()));
-        }};
-    }
-
-    @Test
-    public void testOnUpdateSchemaContextDoNotUpdateKnownModulesIfTheyDoNotContainASuperSetOfKnownModules()
-            throws Exception {
-        new JavaTestKit(getSystem()) {{
-            final TestActorRef<ShardManager> shardManager =
-                    TestActorRef.create(getSystem(), newShardMgrProps(true));
-
-            SchemaContext schemaContext = mock(SchemaContext.class);
-            Set<ModuleIdentifier> moduleIdentifierSet = new HashSet<>();
-
-            ModuleIdentifier foo = mock(ModuleIdentifier.class);
-            when(foo.getNamespace()).thenReturn(new URI("foo"));
-
-            moduleIdentifierSet.add(foo);
-
-            when(schemaContext.getAllModuleIdentifiers()).thenReturn(moduleIdentifierSet);
-
-            shardManager.underlyingActor().onReceiveCommand(new UpdateSchemaContext(schemaContext));
-
-            assertEquals("getKnownModules", Sets.newHashSet("foo"),
-                    Sets.newHashSet(shardManager.underlyingActor().getKnownModules()));
-
-            //Create a completely different SchemaContext with only the bar module in it
-            //schemaContext = mock(SchemaContext.class);
-            moduleIdentifierSet.clear();
-            ModuleIdentifier bar = mock(ModuleIdentifier.class);
-            when(bar.getNamespace()).thenReturn(new URI("bar"));
-
-            moduleIdentifierSet.add(bar);
-
-            shardManager.underlyingActor().onReceiveCommand(new UpdateSchemaContext(schemaContext));
-
-            assertEquals("getKnownModules", Sets.newHashSet("foo"),
-                    Sets.newHashSet(shardManager.underlyingActor().getKnownModules()));
-
-        }};
-    }
-
-    @Test
-    public void testRecoveryApplicable(){
-        new JavaTestKit(getSystem()) {
-            {
-                final Props persistentProps = newShardMgrProps(true);
-                final TestActorRef<ShardManager> persistentShardManager =
-                        TestActorRef.create(getSystem(), persistentProps);
-
-                DataPersistenceProvider dataPersistenceProvider1 = persistentShardManager.underlyingActor().getDataPersistenceProvider();
-
-                assertTrue("Recovery Applicable", dataPersistenceProvider1.isRecoveryApplicable());
-
-                final Props nonPersistentProps = newShardMgrProps(false);
-                final TestActorRef<ShardManager> nonPersistentShardManager =
-                        TestActorRef.create(getSystem(), nonPersistentProps);
-
-                DataPersistenceProvider dataPersistenceProvider2 = nonPersistentShardManager.underlyingActor().getDataPersistenceProvider();
-
-                assertFalse("Recovery Not Applicable", dataPersistenceProvider2.isRecoveryApplicable());
-
-
-            }};
-
-    }
-
-    @Test
-    public void testOnUpdateSchemaContextUpdateKnownModulesCallsDataPersistenceProvider()
-            throws Exception {
-        final CountDownLatch persistLatch = new CountDownLatch(1);
-        final Creator<ShardManager> creator = new Creator<ShardManager>() {
-            private static final long serialVersionUID = 1L;
-            @Override
-            public ShardManager create() throws Exception {
-                return new ShardManager(new MockClusterWrapper(), new MockConfiguration(), DatastoreContext.newBuilder().build(),
-                        ready, new PrimaryShardInfoFutureCache()) {
-                    @Override
-                    protected DataPersistenceProvider createDataPersistenceProvider(boolean persistent) {
-                        DataPersistenceProviderMonitor dataPersistenceProviderMonitor
-                                = new DataPersistenceProviderMonitor();
-                        dataPersistenceProviderMonitor.setPersistLatch(persistLatch);
-                        return dataPersistenceProviderMonitor;
-                    }
-                };
-            }
-        };
-
-        new JavaTestKit(getSystem()) {{
-
-            final TestActorRef<ShardManager> shardManager =
-                    TestActorRef.create(getSystem(), Props.create(new DelegatingShardManagerCreator(creator)));
-
-            ModuleIdentifier foo = mock(ModuleIdentifier.class);
-            when(foo.getNamespace()).thenReturn(new URI("foo"));
-
-            Set<ModuleIdentifier> moduleIdentifierSet = new HashSet<>();
-            moduleIdentifierSet.add(foo);
-
-            SchemaContext schemaContext = mock(SchemaContext.class);
-            when(schemaContext.getAllModuleIdentifiers()).thenReturn(moduleIdentifierSet);
-
-            shardManager.underlyingActor().onReceiveCommand(new UpdateSchemaContext(schemaContext));
-
-            assertEquals("Persisted", true,
-                    Uninterruptibles.awaitUninterruptibly(persistLatch, 5, TimeUnit.SECONDS));
-
-        }};
-    }
-
     @Test
     public void testRoleChangeNotificationAndShardLeaderStateChangedReleaseReady() throws Exception {
         new JavaTestKit(getSystem()) {
             {
-                TestActorRef<ShardManager> shardManager = TestActorRef.create(getSystem(), newShardMgrProps(true));
+                TestActorRef<ShardManager> shardManager = TestActorRef.create(getSystem(), newShardMgrProps());
 
                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
                 shardManager.underlyingActor().onReceiveCommand(new RoleChangeNotification(
@@ -851,7 +863,7 @@ public class ShardManagerTest extends AbstractActorTest {
                 verify(ready, never()).countDown();
 
                 shardManager.underlyingActor().onReceiveCommand(new ShardLeaderStateChanged(memberId, memberId,
-                        Optional.of(mock(DataTree.class))));
+                        Optional.of(mock(DataTree.class)), DataStoreVersions.CURRENT_VERSION));
 
                 verify(ready, times(1)).countDown();
 
@@ -862,7 +874,7 @@ public class ShardManagerTest extends AbstractActorTest {
     public void testRoleChangeNotificationToFollowerWithShardLeaderStateChangedReleaseReady() throws Exception {
         new JavaTestKit(getSystem()) {
             {
-                TestActorRef<ShardManager> shardManager = TestActorRef.create(getSystem(), newShardMgrProps(true));
+                TestActorRef<ShardManager> shardManager = TestActorRef.create(getSystem(), newShardMgrProps());
 
                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
                 shardManager.underlyingActor().onReceiveCommand(new RoleChangeNotification(
@@ -873,7 +885,8 @@ public class ShardManagerTest extends AbstractActorTest {
                 shardManager.underlyingActor().onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
 
                 shardManager.underlyingActor().onReceiveCommand(new ShardLeaderStateChanged(memberId,
-                        "member-2-shard-default-" + shardMrgIDSuffix, Optional.of(mock(DataTree.class))));
+                        "member-2-shard-default-" + shardMrgIDSuffix, Optional.of(mock(DataTree.class)),
+                        DataStoreVersions.CURRENT_VERSION));
 
                 verify(ready, times(1)).countDown();
 
@@ -884,7 +897,7 @@ public class ShardManagerTest extends AbstractActorTest {
     public void testReadyCountDownForMemberUpAfterLeaderStateChanged() throws Exception {
         new JavaTestKit(getSystem()) {
             {
-                TestActorRef<ShardManager> shardManager = TestActorRef.create(getSystem(), newShardMgrProps(true));
+                TestActorRef<ShardManager> shardManager = TestActorRef.create(getSystem(), newShardMgrProps());
 
                 String memberId = "member-1-shard-default-" + shardMrgIDSuffix;
                 shardManager.underlyingActor().onReceiveCommand(new RoleChangeNotification(
@@ -893,7 +906,8 @@ public class ShardManagerTest extends AbstractActorTest {
                 verify(ready, never()).countDown();
 
                 shardManager.underlyingActor().onReceiveCommand(new ShardLeaderStateChanged(memberId,
-                        "member-2-shard-default-" + shardMrgIDSuffix, Optional.of(mock(DataTree.class))));
+                        "member-2-shard-default-" + shardMrgIDSuffix, Optional.of(mock(DataTree.class)),
+                        DataStoreVersions.CURRENT_VERSION));
 
                 shardManager.underlyingActor().onReceiveCommand(MockClusterWrapper.createMemberUp("member-2", getRef().path().toString()));
 
@@ -906,7 +920,7 @@ public class ShardManagerTest extends AbstractActorTest {
     public void testRoleChangeNotificationDoNothingForUnknownShard() throws Exception {
         new JavaTestKit(getSystem()) {
             {
-                TestActorRef<ShardManager> shardManager = TestActorRef.create(getSystem(), newShardMgrProps(true));
+                TestActorRef<ShardManager> shardManager = TestActorRef.create(getSystem(), newShardMgrProps());
 
                 shardManager.underlyingActor().onReceiveCommand(new RoleChangeNotification(
                         "unknown", RaftState.Candidate.name(), RaftState.Leader.name()));
@@ -919,7 +933,7 @@ public class ShardManagerTest extends AbstractActorTest {
 
     @Test
     public void testByDefaultSyncStatusIsFalse() throws Exception{
-        final Props persistentProps = newShardMgrProps(true);
+        final Props persistentProps = newShardMgrProps();
         final TestActorRef<ShardManager> shardManager =
                 TestActorRef.create(getSystem(), persistentProps);
 
@@ -930,10 +944,9 @@ public class ShardManagerTest extends AbstractActorTest {
 
     @Test
     public void testWhenShardIsLeaderSyncStatusIsTrue() throws Exception{
-        final Props persistentProps = ShardManager.props(
-                new MockClusterWrapper(),
-                new MockConfiguration(),
-                DatastoreContext.newBuilder().persistent(true).build(), ready, primaryShardInfoCache);
+        final Props persistentProps = ShardManager.props(new MockClusterWrapper(), new MockConfiguration(),
+                newDatastoreContextFactory(DatastoreContext.newBuilder().persistent(true).build()), ready,
+                primaryShardInfoCache);
         final TestActorRef<ShardManager> shardManager =
                 TestActorRef.create(getSystem(), persistentProps);
 
@@ -946,7 +959,7 @@ public class ShardManagerTest extends AbstractActorTest {
 
     @Test
     public void testWhenShardIsCandidateSyncStatusIsFalse() throws Exception{
-        final Props persistentProps = newShardMgrProps(true);
+        final Props persistentProps = newShardMgrProps();
         final TestActorRef<ShardManager> shardManager =
                 TestActorRef.create(getSystem(), persistentProps);
 
@@ -964,10 +977,9 @@ public class ShardManagerTest extends AbstractActorTest {
 
     @Test
     public void testWhenShardIsFollowerSyncStatusDependsOnFollowerInitialSyncStatus() throws Exception{
-        final Props persistentProps = ShardManager.props(
-                new MockClusterWrapper(),
-                new MockConfiguration(),
-                DatastoreContext.newBuilder().persistent(true).build(), ready, primaryShardInfoCache);
+        final Props persistentProps = ShardManager.props(new MockClusterWrapper(), new MockConfiguration(),
+                newDatastoreContextFactory(DatastoreContext.newBuilder().persistent(true).build()), ready,
+                primaryShardInfoCache);
         final TestActorRef<ShardManager> shardManager =
                 TestActorRef.create(getSystem(), persistentProps);
 
@@ -992,15 +1004,15 @@ public class ShardManagerTest extends AbstractActorTest {
 
     @Test
     public void testWhenMultipleShardsPresentSyncStatusMustBeTrueForAllShards() throws Exception{
-        final Props persistentProps = ShardManager.props(
-                new MockClusterWrapper(),
+        final Props persistentProps = ShardManager.props(new MockClusterWrapper(),
                 new MockConfiguration() {
                     @Override
                     public List<String> getMemberShardNames(String memberName) {
                         return Arrays.asList("default", "astronauts");
                     }
                 },
-                DatastoreContext.newBuilder().persistent(true).build(), ready, primaryShardInfoCache);
+                newDatastoreContextFactory(DatastoreContext.newBuilder().persistent(true).build()), ready,
+                primaryShardInfoCache);
         final TestActorRef<ShardManager> shardManager =
                 TestActorRef.create(getSystem(), persistentProps);
 
@@ -1038,13 +1050,250 @@ public class ShardManagerTest extends AbstractActorTest {
 
     }
 
+    @Test
+    public void testOnReceiveSwitchShardBehavior() throws Exception {
+        new JavaTestKit(getSystem()) {{
+            final ActorRef shardManager = getSystem().actorOf(newPropsShardMgrWithMockShardActor());
+
+            shardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
+            shardManager.tell(new ActorInitialized(), mockShardActor);
+
+            shardManager.tell(new SwitchShardBehavior(mockShardName, "Leader", 1000), getRef());
+
+            SwitchBehavior switchBehavior = MessageCollectorActor.expectFirstMatching(mockShardActor, SwitchBehavior.class);
+
+            assertEquals(RaftState.Leader, switchBehavior.getNewState());
+            assertEquals(1000, switchBehavior.getNewTerm());
+        }};
+    }
+
+    @Test
+    public void testOnReceiveCreateShard() {
+        new JavaTestKit(getSystem()) {{
+            datastoreContextBuilder.shardInitializationTimeout(1, TimeUnit.MINUTES).persistent(true);
+
+            ActorRef shardManager = getSystem().actorOf(newShardMgrProps(
+                    new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
+
+            SchemaContext schemaContext = TestModel.createTestContext();
+            shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
+
+            DatastoreContext datastoreContext = DatastoreContext.newBuilder().shardElectionTimeoutFactor(100).
+                    persistent(false).build();
+            TestShardPropsCreator shardPropsCreator = new TestShardPropsCreator();
+
+            ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
+                    "foo", null, Arrays.asList("member-1", "member-5", "member-6"));
+            shardManager.tell(new CreateShard(config, shardPropsCreator, datastoreContext), getRef());
+
+            expectMsgClass(duration("5 seconds"), CreateShardReply.class);
+
+            shardManager.tell(new FindLocalShard("foo", true), getRef());
+
+            expectMsgClass(duration("5 seconds"), LocalShardFound.class);
+
+            assertEquals("isRecoveryApplicable", false, shardPropsCreator.datastoreContext.isPersistent());
+            assertTrue("Epxected ShardPeerAddressResolver", shardPropsCreator.datastoreContext.getShardRaftConfig().
+                    getPeerAddressResolver() instanceof ShardPeerAddressResolver);
+            assertEquals("peerMembers", Sets.newHashSet(new ShardIdentifier("foo", "member-5", shardMrgIDSuffix).toString(),
+                    new ShardIdentifier("foo", "member-6", shardMrgIDSuffix).toString()),
+                    shardPropsCreator.peerAddresses.keySet());
+            assertEquals("ShardIdentifier", new ShardIdentifier("foo", "member-1", shardMrgIDSuffix),
+                    shardPropsCreator.shardId);
+            assertSame("schemaContext", schemaContext, shardPropsCreator.schemaContext);
+
+            // Send CreateShard with same name - should fail.
+
+            shardManager.tell(new CreateShard(config, shardPropsCreator, null), getRef());
+
+            expectMsgClass(duration("5 seconds"), akka.actor.Status.Failure.class);
+        }};
+    }
+
+    @Test
+    public void testOnReceiveCreateShardWithNoInitialSchemaContext() {
+        new JavaTestKit(getSystem()) {{
+            ActorRef shardManager = getSystem().actorOf(newShardMgrProps(
+                    new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
+
+            TestShardPropsCreator shardPropsCreator = new TestShardPropsCreator();
+
+            ModuleShardConfiguration config = new ModuleShardConfiguration(URI.create("foo-ns"), "foo-module",
+                    "foo", null, Arrays.asList("member-1"));
+            shardManager.tell(new CreateShard(config, shardPropsCreator, null), getRef());
+
+            expectMsgClass(duration("5 seconds"), CreateShardReply.class);
+
+            SchemaContext schemaContext = TestModel.createTestContext();
+            shardManager.tell(new UpdateSchemaContext(schemaContext), ActorRef.noSender());
+
+            shardManager.tell(new FindLocalShard("foo", true), getRef());
+
+            expectMsgClass(duration("5 seconds"), LocalShardFound.class);
+
+            assertSame("schemaContext", schemaContext, shardPropsCreator.schemaContext);
+            assertNotNull("schemaContext is null", shardPropsCreator.datastoreContext);
+        }};
+    }
+
+    @Test
+    public void testAddShardReplicaForNonExistentShard() throws Exception {
+        new JavaTestKit(getSystem()) {{
+            ActorRef shardManager = getSystem().actorOf(newShardMgrProps(
+                    new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
+
+            shardManager.tell(new AddShardReplica("model-inventory"), getRef());
+            Status.Failure resp = expectMsgClass(duration("2 seconds"), Status.Failure.class);
+
+            assertEquals("Failure obtained", true,
+                          (resp.cause() instanceof IllegalArgumentException));
+        }};
+    }
+
+    @Test
+    public void testAddShardReplicaForAlreadyCreatedShard() throws Exception {
+        new JavaTestKit(getSystem()) {{
+            ActorRef shardManager = getSystem().actorOf(newShardMgrProps());
+            shardManager.tell(new AddShardReplica("default"), getRef());
+            Status.Failure resp = expectMsgClass(duration("2 seconds"), Status.Failure.class);
+            assertEquals("Failure obtained", true,
+                          (resp.cause() instanceof IllegalArgumentException));
+        }};
+    }
+
+    @Test
+    public void testAddShardReplica() throws Exception {
+        MockConfiguration mockConfig =
+                new MockConfiguration(ImmutableMap.<String, List<String>>builder().
+                   put("default", Arrays.asList("member-1", "member-2")).
+                   put("astronauts", Arrays.asList("member-2")).build());
+
+        String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
+
+        // Create an ActorSystem ShardManager actor for member-1.
+        final ActorSystem system1 = ActorSystem.create("cluster-test", ConfigFactory.load().getConfig("Member1"));
+        Cluster.get(system1).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
+        ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
+        final TestActorRef<ForwardingShardManager> newReplicaShardManager = TestActorRef.create(system1,
+                newPropsShardMgrWithMockShardActor("shardManager1", mockDefaultShardActor,
+                   new ClusterWrapperImpl(system1), mockConfig), shardManagerID);
+
+        // Create an ActorSystem ShardManager actor for member-2.
+        final ActorSystem system2 = ActorSystem.create("cluster-test",
+            ConfigFactory.load().getConfig("Member2"));
+        Cluster.get(system2).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
+
+        String name = new ShardIdentifier("astronauts", "member-2", "config").toString();
+        final TestActorRef<MockRespondActor> mockShardLeaderActor =
+            TestActorRef.create(system2, Props.create(MockRespondActor.class), name);
+        final TestActorRef<ForwardingShardManager> leaderShardManager = TestActorRef.create(system2,
+                newPropsShardMgrWithMockShardActor("shardManager2", mockShardLeaderActor,
+                        new ClusterWrapperImpl(system2), mockConfig), shardManagerID);
+
+        new JavaTestKit(system1) {{
+
+            newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
+            leaderShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
+
+            leaderShardManager.tell(new ActorInitialized(), mockShardLeaderActor);
+
+            String memberId2 = "member-2-shard-astronauts-" + shardMrgIDSuffix;
+            short leaderVersion = DataStoreVersions.CURRENT_VERSION - 1;
+            leaderShardManager.tell(new ShardLeaderStateChanged(memberId2, memberId2,
+                    Optional.of(mock(DataTree.class)), leaderVersion), mockShardLeaderActor);
+            leaderShardManager.tell(new RoleChangeNotification(memberId2,
+                    RaftState.Candidate.name(), RaftState.Leader.name()), mockShardLeaderActor);
+
+            newReplicaShardManager.underlyingActor().waitForMemberUp();
+            leaderShardManager.underlyingActor().waitForMemberUp();
+
+            //construct a mock response message
+            AddServerReply response = new AddServerReply(ServerChangeStatus.OK, memberId2);
+            mockShardLeaderActor.underlyingActor().updateResponse(response);
+            newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
+            AddServer addServerMsg = MessageCollectorActor.expectFirstMatching(mockShardLeaderActor,
+                AddServer.class);
+            String addServerId = "member-1-shard-astronauts-" + shardMrgIDSuffix;
+            assertEquals("AddServer serverId", addServerId, addServerMsg.getNewServerId());
+
+            expectMsgClass(duration("5 seconds"), Status.Success.class);
+        }};
+
+        JavaTestKit.shutdownActorSystem(system1);
+        JavaTestKit.shutdownActorSystem(system2);
+    }
+
+    @Test
+    public void testAddShardReplicaWithFindPrimaryTimeout() throws Exception {
+        MockConfiguration mockConfig =
+                new MockConfiguration(ImmutableMap.<String, List<String>>builder().
+                   put("default", Arrays.asList("member-1", "member-2")).
+                   put("astronauts", Arrays.asList("member-2")).build());
+
+        String shardManagerID = ShardManagerIdentifier.builder().type(shardMrgIDSuffix).build().toString();
+
+        // Create an ActorSystem ShardManager actor for member-1.
+        final ActorSystem system1 = ActorSystem.create("cluster-test", ConfigFactory.load().getConfig("Member1"));
+        Cluster.get(system1).join(AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558"));
+        ActorRef mockDefaultShardActor = newMockShardActor(system1, Shard.DEFAULT_NAME, "member-1");
+        final TestActorRef<ForwardingShardManager> newReplicaShardManager = TestActorRef.create(system1,
+                newPropsShardMgrWithMockShardActor("shardManager1", mockDefaultShardActor,
+                   new ClusterWrapperImpl(system1), mockConfig), shardManagerID);
+
+        new JavaTestKit(system1) {{
+
+            newReplicaShardManager.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef());
+            MockClusterWrapper.sendMemberUp(newReplicaShardManager, "member-2", getRef().path().toString());
+            newReplicaShardManager.underlyingActor().waitForMemberUp();
+
+            newReplicaShardManager.tell(new AddShardReplica("astronauts"), getRef());
+            Status.Failure resp = expectMsgClass(duration("5 seconds"), Status.Failure.class);
+            assertEquals("Failure obtained", true,
+                          (resp.cause() instanceof RuntimeException));
+        }};
+
+        JavaTestKit.shutdownActorSystem(system1);
+    }
+
+    @Test
+    public void testRemoveShardReplicaForNonExistentShard() throws Exception {
+        new JavaTestKit(getSystem()) {{
+            ActorRef shardManager = getSystem().actorOf(newShardMgrProps(
+                    new ConfigurationImpl(new EmptyModuleShardConfigProvider())));
+
+            shardManager.tell(new RemoveShardReplica("model-inventory"), getRef());
+            Status.Failure resp = expectMsgClass(duration("2 seconds"), Status.Failure.class);
+            assertEquals("Failure obtained", true,
+                         (resp.cause() instanceof IllegalArgumentException));
+        }};
+
+    }
+
+    private static class TestShardPropsCreator implements ShardPropsCreator {
+        ShardIdentifier shardId;
+        Map<String, String> peerAddresses;
+        SchemaContext schemaContext;
+        DatastoreContext datastoreContext;
+
+        @Override
+        public Props newProps(ShardIdentifier shardId, Map<String, String> peerAddresses,
+                DatastoreContext datastoreContext, SchemaContext schemaContext) {
+            this.shardId = shardId;
+            this.peerAddresses = peerAddresses;
+            this.schemaContext = schemaContext;
+            this.datastoreContext = datastoreContext;
+            return Shard.props(shardId, peerAddresses, datastoreContext, schemaContext);
+        }
+
+    }
+
     private static class TestShardManager extends ShardManager {
         private final CountDownLatch recoveryComplete = new CountDownLatch(1);
 
         TestShardManager(String shardMrgIDSuffix) {
             super(new MockClusterWrapper(), new MockConfiguration(),
-                    DatastoreContext.newBuilder().dataStoreType(shardMrgIDSuffix).build(), ready,
-                    new PrimaryShardInfoFutureCache());
+                    newDatastoreContextFactory(DatastoreContext.newBuilder().dataStoreType(shardMrgIDSuffix).build()),
+                    ready, new PrimaryShardInfoFutureCache());
         }
 
         @Override
@@ -1103,9 +1352,9 @@ public class ShardManagerTest extends AbstractActorTest {
         private final String name;
 
         protected ForwardingShardManager(ClusterWrapper cluster, Configuration configuration,
-                DatastoreContext datastoreContext, CountDownLatch waitTillReadyCountdownLatch, String name,
+                DatastoreContextFactory factory, CountDownLatch waitTillReadyCountdownLatch, String name,
                 ActorRef shardActor, PrimaryShardInfoFutureCache primaryShardInfoCache) {
-            super(cluster, configuration, datastoreContext, waitTillReadyCountdownLatch, primaryShardInfoCache);
+            super(cluster, configuration, factory, waitTillReadyCountdownLatch, primaryShardInfoCache);
             this.shardActor = shardActor;
             this.name = name;
         }
@@ -1182,4 +1431,24 @@ public class ShardManagerTest extends AbstractActorTest {
             findPrimaryMessageReceived = new CountDownLatch(1);
         }
     }
+
+    private static class MockRespondActor extends MessageCollectorActor {
+
+        private volatile Object responseMsg;
+
+        public void updateResponse(Object response) {
+            responseMsg = response;
+        }
+
+        @Override
+        public void onReceive(Object message) throws Exception {
+            super.onReceive(message);
+            if (message instanceof AddServer) {
+                if (responseMsg != null) {
+                    getSender().tell(responseMsg, getSelf());
+                    responseMsg = null;
+                }
+            }
+        }
+    }
 }