+ resp = followerTestKit.expectMsgClass(Object.class);
+ if (resp instanceof akka.actor.Status.Failure) {
+ throw new AssertionError("Unexpected failure response", ((akka.actor.Status.Failure)resp).cause());
+ }
+
+ assertEquals("Response type", ReadyTransactionReply.class, resp.getClass());
+
+ final ActorSelection txActor = leaderDistributedDataStore.getActorUtils().actorSelection(
+ ((ReadyTransactionReply)resp).getCohortPath());
+
+ final Supplier<Short> versionSupplier = Mockito.mock(Supplier.class);
+ Mockito.doReturn(DataStoreVersions.CURRENT_VERSION).when(versionSupplier).get();
+ ThreePhaseCommitCohortProxy cohort = new ThreePhaseCommitCohortProxy(
+ leaderDistributedDataStore.getActorUtils(), Arrays.asList(
+ new ThreePhaseCommitCohortProxy.CohortInfo(Futures.successful(txActor), versionSupplier)), tx2);
+ cohort.canCommit().get(5, TimeUnit.SECONDS);
+ cohort.preCommit().get(5, TimeUnit.SECONDS);
+ cohort.commit().get(5, TimeUnit.SECONDS);
+
+ verifyCars(leaderDistributedDataStore.newReadOnlyTransaction(), car1, car2);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testForwardedReadyTransactionForwardedToLeader() throws Exception {
+ initDatastoresWithCars("testForwardedReadyTransactionForwardedToLeader");
+ followerTestKit.waitUntilLeader(followerDistributedDataStore.getActorUtils(), "cars");
+
+ final Optional<ActorRef> carsFollowerShard =
+ followerDistributedDataStore.getActorUtils().findLocalShard("cars");
+ assertTrue("Cars follower shard found", carsFollowerShard.isPresent());
+
+ carsFollowerShard.get().tell(GetShardDataTree.INSTANCE, followerTestKit.getRef());
+ final DataTree dataTree = followerTestKit.expectMsgClass(DataTree.class);
+
+ // Send a tx with immediate commit.
+
+ DataTreeModification modification = dataTree.takeSnapshot().newModification();
+ new WriteModification(CarsModel.BASE_PATH, CarsModel.emptyContainer()).apply(modification);
+ new MergeModification(CarsModel.CAR_LIST_PATH, CarsModel.newCarMapNode()).apply(modification);
+
+ final MapEntryNode car1 = CarsModel.newCarEntry("optima", Uint64.valueOf(20000));
+ new WriteModification(CarsModel.newCarPath("optima"), car1).apply(modification);
+
+ ForwardedReadyTransaction forwardedReady = new ForwardedReadyTransaction(tx1,
+ DataStoreVersions.CURRENT_VERSION, new ReadWriteShardDataTreeTransaction(
+ Mockito.mock(ShardDataTreeTransactionParent.class), tx1, modification), true,
+ Optional.empty());
+
+ carsFollowerShard.get().tell(forwardedReady, followerTestKit.getRef());
+ Object resp = followerTestKit.expectMsgClass(Object.class);
+ if (resp instanceof akka.actor.Status.Failure) {
+ throw new AssertionError("Unexpected failure response", ((akka.actor.Status.Failure)resp).cause());
+ }
+
+ assertEquals("Response type", CommitTransactionReply.class, resp.getClass());
+
+ verifyCars(leaderDistributedDataStore.newReadOnlyTransaction(), car1);
+
+ // Send another tx without immediate commit.
+
+ modification = dataTree.takeSnapshot().newModification();
+ MapEntryNode car2 = CarsModel.newCarEntry("sportage", Uint64.valueOf(30000));
+ new WriteModification(CarsModel.newCarPath("sportage"), car2).apply(modification);
+
+ forwardedReady = new ForwardedReadyTransaction(tx2,
+ DataStoreVersions.CURRENT_VERSION, new ReadWriteShardDataTreeTransaction(
+ Mockito.mock(ShardDataTreeTransactionParent.class), tx2, modification), false,
+ Optional.empty());
+
+ carsFollowerShard.get().tell(forwardedReady, followerTestKit.getRef());
+ resp = followerTestKit.expectMsgClass(Object.class);
+ if (resp instanceof akka.actor.Status.Failure) {
+ throw new AssertionError("Unexpected failure response", ((akka.actor.Status.Failure)resp).cause());
+ }
+
+ assertEquals("Response type", ReadyTransactionReply.class, resp.getClass());
+
+ ActorSelection txActor = leaderDistributedDataStore.getActorUtils().actorSelection(
+ ((ReadyTransactionReply)resp).getCohortPath());
+
+ final Supplier<Short> versionSupplier = Mockito.mock(Supplier.class);
+ Mockito.doReturn(DataStoreVersions.CURRENT_VERSION).when(versionSupplier).get();
+ final ThreePhaseCommitCohortProxy cohort = new ThreePhaseCommitCohortProxy(
+ leaderDistributedDataStore.getActorUtils(), Arrays.asList(
+ new ThreePhaseCommitCohortProxy.CohortInfo(Futures.successful(txActor), versionSupplier)), tx2);
+ cohort.canCommit().get(5, TimeUnit.SECONDS);
+ cohort.preCommit().get(5, TimeUnit.SECONDS);
+ cohort.commit().get(5, TimeUnit.SECONDS);
+
+ verifyCars(leaderDistributedDataStore.newReadOnlyTransaction(), car1, car2);
+ }
+
+ @Test
+ public void testTransactionForwardedToLeaderAfterRetry() throws Exception {
+ // FIXME: remove when test passes also for ClientBackedDataStore
+ Assume.assumeTrue(DistributedDataStore.class.isAssignableFrom(testParameter));
+ followerDatastoreContextBuilder.shardBatchedModificationCount(2);
+ leaderDatastoreContextBuilder.shardBatchedModificationCount(2);
+ initDatastoresWithCarsAndPeople("testTransactionForwardedToLeaderAfterRetry");
+
+ // Do an initial write to get the primary shard info cached.
+
+ final DOMStoreWriteTransaction initialWriteTx = followerDistributedDataStore.newWriteOnlyTransaction();
+ initialWriteTx.write(CarsModel.BASE_PATH, CarsModel.emptyContainer());
+ initialWriteTx.write(PeopleModel.BASE_PATH, PeopleModel.emptyContainer());
+ followerTestKit.doCommit(initialWriteTx.ready());
+
+ // Wait for the commit to be replicated to the follower.
+
+ MemberNode.verifyRaftState(followerDistributedDataStore, "cars",
+ raftState -> assertEquals("getLastApplied", 1, raftState.getLastApplied()));
+
+ MemberNode.verifyRaftState(followerDistributedDataStore, "people",
+ raftState -> assertEquals("getLastApplied", 1, raftState.getLastApplied()));
+
+ // Prepare, ready and canCommit a WO tx that writes to 2 shards. This will become the current tx in
+ // the leader shard.
+
+ final DOMStoreWriteTransaction writeTx1 = followerDistributedDataStore.newWriteOnlyTransaction();
+ writeTx1.write(CarsModel.CAR_LIST_PATH, CarsModel.newCarMapNode());
+ writeTx1.write(PeopleModel.BASE_PATH, PeopleModel.emptyContainer());
+ final DOMStoreThreePhaseCommitCohort writeTx1Cohort = writeTx1.ready();
+ final ListenableFuture<Boolean> writeTx1CanCommit = writeTx1Cohort.canCommit();
+ writeTx1CanCommit.get(5, TimeUnit.SECONDS);
+
+ // Prepare and ready another WO tx that writes to 2 shards but don't canCommit yet. This will be queued
+ // in the leader shard.
+
+ final DOMStoreWriteTransaction writeTx2 = followerDistributedDataStore.newWriteOnlyTransaction();
+ final LinkedList<MapEntryNode> cars = new LinkedList<>();
+ int carIndex = 1;
+ cars.add(CarsModel.newCarEntry("car" + carIndex, Uint64.valueOf(carIndex)));
+ writeTx2.write(CarsModel.newCarPath("car" + carIndex), cars.getLast());
+ carIndex++;
+ NormalizedNode<?, ?> people = ImmutableNodes.mapNodeBuilder(PeopleModel.PERSON_QNAME)
+ .withChild(PeopleModel.newPersonEntry("Dude")).build();
+ writeTx2.write(PeopleModel.PERSON_LIST_PATH, people);
+ final DOMStoreThreePhaseCommitCohort writeTx2Cohort = writeTx2.ready();
+
+ // Prepare another WO that writes to a single shard and thus will be directly committed on ready. This
+ // tx writes 5 cars so 2 BatchedModidifications messages will be sent initially and cached in the
+ // leader shard (with shardBatchedModificationCount set to 2). The 3rd BatchedModidifications will be
+ // sent on ready.
+
+ final DOMStoreWriteTransaction writeTx3 = followerDistributedDataStore.newWriteOnlyTransaction();
+ for (int i = 1; i <= 5; i++, carIndex++) {
+ cars.add(CarsModel.newCarEntry("car" + carIndex, Uint64.valueOf(carIndex)));
+ writeTx3.write(CarsModel.newCarPath("car" + carIndex), cars.getLast());
+ }
+
+ // Prepare another WO that writes to a single shard. This will send a single BatchedModidifications
+ // message on ready.
+
+ final DOMStoreWriteTransaction writeTx4 = followerDistributedDataStore.newWriteOnlyTransaction();
+ cars.add(CarsModel.newCarEntry("car" + carIndex, Uint64.valueOf(carIndex)));
+ writeTx4.write(CarsModel.newCarPath("car" + carIndex), cars.getLast());
+ carIndex++;
+
+ // Prepare a RW tx that will create a tx actor and send a ForwardedReadyTransaciton message to the
+ // leader shard on ready.
+
+ final DOMStoreReadWriteTransaction readWriteTx = followerDistributedDataStore.newReadWriteTransaction();
+ cars.add(CarsModel.newCarEntry("car" + carIndex, Uint64.valueOf(carIndex)));
+ readWriteTx.write(CarsModel.newCarPath("car" + carIndex), cars.getLast());
+
+ IntegrationTestKit.verifyShardStats(leaderDistributedDataStore, "cars",
+ stats -> assertEquals("getReadWriteTransactionCount", 5, stats.getReadWriteTransactionCount()));
+
+ // Disable elections on the leader so it switches to follower.
+
+ sendDatastoreContextUpdate(leaderDistributedDataStore, leaderDatastoreContextBuilder
+ .customRaftPolicyImplementation(DisableElectionsRaftPolicy.class.getName())
+ .shardElectionTimeoutFactor(10));
+
+ leaderTestKit.waitUntilNoLeader(leaderDistributedDataStore.getActorUtils(), "cars");
+
+ // Submit all tx's - the messages should get queued for retry.
+
+ final ListenableFuture<Boolean> writeTx2CanCommit = writeTx2Cohort.canCommit();
+ final DOMStoreThreePhaseCommitCohort writeTx3Cohort = writeTx3.ready();
+ final DOMStoreThreePhaseCommitCohort writeTx4Cohort = writeTx4.ready();
+ final DOMStoreThreePhaseCommitCohort rwTxCohort = readWriteTx.ready();
+
+ // Enable elections on the other follower so it becomes the leader, at which point the
+ // tx's should get forwarded from the previous leader to the new leader to complete the commits.
+
+ sendDatastoreContextUpdate(followerDistributedDataStore, followerDatastoreContextBuilder
+ .customRaftPolicyImplementation(null).shardElectionTimeoutFactor(1));
+ IntegrationTestKit.findLocalShard(followerDistributedDataStore.getActorUtils(), "cars")
+ .tell(TimeoutNow.INSTANCE, ActorRef.noSender());
+ IntegrationTestKit.findLocalShard(followerDistributedDataStore.getActorUtils(), "people")
+ .tell(TimeoutNow.INSTANCE, ActorRef.noSender());
+
+ followerTestKit.doCommit(writeTx1CanCommit, writeTx1Cohort);
+ followerTestKit.doCommit(writeTx2CanCommit, writeTx2Cohort);
+ followerTestKit.doCommit(writeTx3Cohort);
+ followerTestKit.doCommit(writeTx4Cohort);
+ followerTestKit.doCommit(rwTxCohort);
+
+ DOMStoreReadTransaction readTx = leaderDistributedDataStore.newReadOnlyTransaction();
+ verifyCars(readTx, cars.toArray(new MapEntryNode[cars.size()]));
+ verifyNode(readTx, PeopleModel.PERSON_LIST_PATH, people);
+ }
+
+ @Test
+ public void testLeadershipTransferOnShutdown() throws Exception {
+ // FIXME: remove when test passes also for ClientBackedDataStore
+ Assume.assumeTrue(DistributedDataStore.class.isAssignableFrom(testParameter));
+ leaderDatastoreContextBuilder.shardBatchedModificationCount(1);
+ followerDatastoreContextBuilder.shardElectionTimeoutFactor(10).customRaftPolicyImplementation(null);
+ final String testName = "testLeadershipTransferOnShutdown";
+ initDatastores(testName, MODULE_SHARDS_CARS_PEOPLE_1_2_3, CARS_AND_PEOPLE);
+
+ final IntegrationTestKit follower2TestKit = new IntegrationTestKit(follower2System,
+ DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build()).operationTimeoutInMillis(500),
+ commitTimeout);
+ try (AbstractDataStore follower2DistributedDataStore = follower2TestKit.setupAbstractDataStore(
+ testParameter, testName, MODULE_SHARDS_CARS_PEOPLE_1_2_3, false)) {
+
+ followerTestKit.waitForMembersUp("member-3");
+ follower2TestKit.waitForMembersUp("member-1", "member-2");
+
+ // Create and submit a couple tx's so they're pending.
+
+ DOMStoreWriteTransaction writeTx = followerDistributedDataStore.newWriteOnlyTransaction();
+ writeTx.write(CarsModel.BASE_PATH, CarsModel.emptyContainer());
+ writeTx.write(CarsModel.CAR_LIST_PATH, CarsModel.newCarMapNode());
+ writeTx.write(PeopleModel.BASE_PATH, PeopleModel.emptyContainer());
+ final DOMStoreThreePhaseCommitCohort cohort1 = writeTx.ready();
+
+ IntegrationTestKit.verifyShardStats(leaderDistributedDataStore, "cars",
+ stats -> assertEquals("getTxCohortCacheSize", 1, stats.getTxCohortCacheSize()));
+
+ writeTx = followerDistributedDataStore.newWriteOnlyTransaction();
+ final MapEntryNode car = CarsModel.newCarEntry("optima", Uint64.valueOf(20000));
+ writeTx.write(CarsModel.newCarPath("optima"), car);
+ final DOMStoreThreePhaseCommitCohort cohort2 = writeTx.ready();
+
+ IntegrationTestKit.verifyShardStats(leaderDistributedDataStore, "cars",
+ stats -> assertEquals("getTxCohortCacheSize", 2, stats.getTxCohortCacheSize()));
+
+ // Gracefully stop the leader via a Shutdown message.
+
+ sendDatastoreContextUpdate(leaderDistributedDataStore, leaderDatastoreContextBuilder
+ .shardElectionTimeoutFactor(100));
+
+ final FiniteDuration duration = FiniteDuration.create(5, TimeUnit.SECONDS);
+ final Future<ActorRef> future = leaderDistributedDataStore.getActorUtils().findLocalShardAsync("cars");
+ final ActorRef leaderActor = Await.result(future, duration);
+
+ final Future<Boolean> stopFuture = Patterns.gracefulStop(leaderActor, duration, Shutdown.INSTANCE);
+
+ // Commit the 2 transactions. They should finish and succeed.
+
+ followerTestKit.doCommit(cohort1);
+ followerTestKit.doCommit(cohort2);
+
+ // Wait for the leader actor stopped.
+
+ final Boolean stopped = Await.result(stopFuture, duration);
+ assertEquals("Stopped", Boolean.TRUE, stopped);
+
+ // Verify leadership was transferred by reading the committed data from the other nodes.
+
+ verifyCars(followerDistributedDataStore.newReadOnlyTransaction(), car);
+ verifyCars(follower2DistributedDataStore.newReadOnlyTransaction(), car);
+ }
+ }
+
+ @Test
+ public void testTransactionWithIsolatedLeader() throws Exception {
+ // FIXME: remove when test passes also for ClientBackedDataStore
+ Assume.assumeTrue(DistributedDataStore.class.isAssignableFrom(testParameter));
+ // Set the isolated leader check interval high so we can control the switch to IsolatedLeader.
+ leaderDatastoreContextBuilder.shardIsolatedLeaderCheckIntervalInMillis(10000000);
+ final String testName = "testTransactionWithIsolatedLeader";
+ initDatastoresWithCars(testName);
+
+ // Tx that is submitted after the follower is stopped but before the leader transitions to IsolatedLeader.
+ final DOMStoreWriteTransaction preIsolatedLeaderWriteTx = leaderDistributedDataStore.newWriteOnlyTransaction();
+ preIsolatedLeaderWriteTx.write(CarsModel.BASE_PATH, CarsModel.emptyContainer());
+
+ // Tx that is submitted after the leader transitions to IsolatedLeader.
+ final DOMStoreWriteTransaction noShardLeaderWriteTx = leaderDistributedDataStore.newWriteOnlyTransaction();
+ noShardLeaderWriteTx.write(CarsModel.BASE_PATH, CarsModel.emptyContainer());
+
+ // Tx that is submitted after the follower is reinstated.
+ final DOMStoreWriteTransaction successWriteTx = leaderDistributedDataStore.newWriteOnlyTransaction();
+ successWriteTx.merge(CarsModel.BASE_PATH, CarsModel.emptyContainer());
+
+ // Stop the follower
+ followerTestKit.watch(followerDistributedDataStore.getActorUtils().getShardManager());
+ followerDistributedDataStore.close();
+ followerTestKit.expectTerminated(followerDistributedDataStore.getActorUtils().getShardManager());
+
+ // Submit the preIsolatedLeaderWriteTx so it's pending
+ final DOMStoreThreePhaseCommitCohort preIsolatedLeaderTxCohort = preIsolatedLeaderWriteTx.ready();
+
+ // Change the isolated leader check interval low so it changes to IsolatedLeader.
+ sendDatastoreContextUpdate(leaderDistributedDataStore, leaderDatastoreContextBuilder
+ .shardIsolatedLeaderCheckIntervalInMillis(200));
+
+ MemberNode.verifyRaftState(leaderDistributedDataStore, "cars",
+ raftState -> assertEquals("getRaftState", "IsolatedLeader", raftState.getRaftState()));
+
+ try {
+ leaderTestKit.doCommit(noShardLeaderWriteTx.ready());
+ fail("Expected NoShardLeaderException");
+ } catch (final ExecutionException e) {
+ assertEquals("getCause", NoShardLeaderException.class, Throwables.getRootCause(e).getClass());
+ }
+
+ sendDatastoreContextUpdate(leaderDistributedDataStore, leaderDatastoreContextBuilder
+ .shardElectionTimeoutFactor(100));
+
+ final DOMStoreThreePhaseCommitCohort successTxCohort = successWriteTx.ready();
+
+ followerDistributedDataStore = followerTestKit.setupAbstractDataStore(
+ testParameter, testName, MODULE_SHARDS_CARS_ONLY_1_2, false, CARS);
+
+ leaderTestKit.doCommit(preIsolatedLeaderTxCohort);
+ leaderTestKit.doCommit(successTxCohort);
+ }
+
+ @Test
+ public void testTransactionWithShardLeaderNotResponding() throws Exception {
+ followerDatastoreContextBuilder.frontendRequestTimeoutInSeconds(2);
+ followerDatastoreContextBuilder.shardElectionTimeoutFactor(50);
+ initDatastoresWithCars("testTransactionWithShardLeaderNotResponding");
+
+ // Do an initial read to get the primary shard info cached.
+
+ final DOMStoreReadTransaction readTx = followerDistributedDataStore.newReadOnlyTransaction();
+ readTx.read(CarsModel.BASE_PATH).get(5, TimeUnit.SECONDS);
+
+ // Shutdown the leader and try to create a new tx.
+
+ TestKit.shutdownActorSystem(leaderSystem, true);
+
+ followerDatastoreContextBuilder.operationTimeoutInMillis(50).shardElectionTimeoutFactor(1);
+ sendDatastoreContextUpdate(followerDistributedDataStore, followerDatastoreContextBuilder);
+
+ final DOMStoreReadWriteTransaction rwTx = followerDistributedDataStore.newReadWriteTransaction();
+
+ rwTx.write(CarsModel.BASE_PATH, CarsModel.emptyContainer());
+
+ try {
+ followerTestKit.doCommit(rwTx.ready());
+ fail("Exception expected");
+ } catch (final ExecutionException e) {
+ final String msg = "Unexpected exception: " + Throwables.getStackTraceAsString(e.getCause());
+ if (DistributedDataStore.class.isAssignableFrom(testParameter)) {
+ assertTrue(msg, Throwables.getRootCause(e) instanceof NoShardLeaderException
+ || e.getCause() instanceof ShardLeaderNotRespondingException);
+ } else {
+ assertTrue(msg, Throwables.getRootCause(e) instanceof RequestTimeoutException);
+ }
+ }
+ }
+
+ @Test
+ public void testTransactionWithCreateTxFailureDueToNoLeader() throws Exception {
+ followerDatastoreContextBuilder.frontendRequestTimeoutInSeconds(2);
+ initDatastoresWithCars("testTransactionWithCreateTxFailureDueToNoLeader");
+
+ // Do an initial read to get the primary shard info cached.
+
+ final DOMStoreReadTransaction readTx = followerDistributedDataStore.newReadOnlyTransaction();
+ readTx.read(CarsModel.BASE_PATH).get(5, TimeUnit.SECONDS);
+
+ // Shutdown the leader and try to create a new tx.
+
+ TestKit.shutdownActorSystem(leaderSystem, true);
+
+ Cluster.get(followerSystem).leave(MEMBER_1_ADDRESS);
+
+ Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
+
+ sendDatastoreContextUpdate(followerDistributedDataStore, followerDatastoreContextBuilder
+ .operationTimeoutInMillis(10).shardElectionTimeoutFactor(1).customRaftPolicyImplementation(null));
+
+ final DOMStoreReadWriteTransaction rwTx = followerDistributedDataStore.newReadWriteTransaction();
+
+ rwTx.write(CarsModel.BASE_PATH, CarsModel.emptyContainer());
+
+ try {
+ followerTestKit.doCommit(rwTx.ready());
+ fail("Exception expected");
+ } catch (final ExecutionException e) {
+ final String msg = "Unexpected exception: " + Throwables.getStackTraceAsString(e.getCause());
+ if (DistributedDataStore.class.isAssignableFrom(testParameter)) {
+ assertTrue(msg, Throwables.getRootCause(e) instanceof NoShardLeaderException);
+ } else {
+ assertTrue(msg, Throwables.getRootCause(e) instanceof RequestTimeoutException);
+ }
+ }
+ }
+
+ @Test
+ public void testTransactionRetryWithInitialAskTimeoutExOnCreateTx() throws Exception {
+ followerDatastoreContextBuilder.backendAlivenessTimerIntervalInSeconds(2);
+ String testName = "testTransactionRetryWithInitialAskTimeoutExOnCreateTx";
+ initDatastores(testName, MODULE_SHARDS_CARS_1_2_3, CARS);
+
+ final DatastoreContext.Builder follower2DatastoreContextBuilder = DatastoreContext.newBuilder()
+ .shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(10);
+ final IntegrationTestKit follower2TestKit = new IntegrationTestKit(
+ follower2System, follower2DatastoreContextBuilder, commitTimeout);
+
+ try (AbstractDataStore ds =
+ follower2TestKit.setupAbstractDataStore(
+ testParameter, testName, MODULE_SHARDS_CARS_1_2_3, false, CARS)) {
+
+ followerTestKit.waitForMembersUp("member-1", "member-3");
+ follower2TestKit.waitForMembersUp("member-1", "member-2");
+
+ // Do an initial read to get the primary shard info cached.
+
+ final DOMStoreReadTransaction readTx = followerDistributedDataStore.newReadOnlyTransaction();
+ readTx.read(CarsModel.BASE_PATH).get(5, TimeUnit.SECONDS);
+
+ // Shutdown the leader and try to create a new tx.
+
+ TestKit.shutdownActorSystem(leaderSystem, true);
+
+ Cluster.get(followerSystem).leave(MEMBER_1_ADDRESS);
+
+ sendDatastoreContextUpdate(followerDistributedDataStore, followerDatastoreContextBuilder
+ .operationTimeoutInMillis(500).shardElectionTimeoutFactor(5).customRaftPolicyImplementation(null));
+
+ final DOMStoreReadWriteTransaction rwTx = followerDistributedDataStore.newReadWriteTransaction();
+
+ rwTx.write(CarsModel.BASE_PATH, CarsModel.emptyContainer());
+
+ followerTestKit.doCommit(rwTx.ready());
+ }
+ }
+
+ @Test
+ public void testSemiReachableCandidateNotDroppingLeader() throws Exception {
+ final String testName = "testSemiReachableCandidateNotDroppingLeader";
+ initDatastores(testName, MODULE_SHARDS_CARS_1_2_3, CARS);
+
+ final DatastoreContext.Builder follower2DatastoreContextBuilder = DatastoreContext.newBuilder()
+ .shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(10);
+ final IntegrationTestKit follower2TestKit = new IntegrationTestKit(
+ follower2System, follower2DatastoreContextBuilder, commitTimeout);
+
+ final AbstractDataStore ds2 =
+ follower2TestKit.setupAbstractDataStore(
+ testParameter, testName, MODULE_SHARDS_CARS_1_2_3, false, CARS);
+
+ followerTestKit.waitForMembersUp("member-1", "member-3");
+ follower2TestKit.waitForMembersUp("member-1", "member-2");
+
+ TestKit.shutdownActorSystem(follower2System);
+
+ ActorRef cars = leaderDistributedDataStore.getActorUtils().findLocalShard("cars").get();
+ OnDemandRaftState initialState = (OnDemandRaftState) leaderDistributedDataStore.getActorUtils()
+ .executeOperation(cars, GetOnDemandRaftState.INSTANCE);
+
+ Cluster leaderCluster = Cluster.get(leaderSystem);
+ Cluster followerCluster = Cluster.get(followerSystem);
+ Cluster follower2Cluster = Cluster.get(follower2System);
+
+ Member follower2Member = follower2Cluster.readView().self();
+
+ await().atMost(10, TimeUnit.SECONDS)
+ .until(() -> leaderCluster.readView().unreachableMembers().contains(follower2Member));
+ await().atMost(10, TimeUnit.SECONDS)
+ .until(() -> followerCluster.readView().unreachableMembers().contains(follower2Member));
+
+ ActorRef followerCars = followerDistributedDataStore.getActorUtils().findLocalShard("cars").get();
+
+ // to simulate a follower not being able to receive messages, but still being able to send messages and becoming
+ // candidate, we can just send a couple of RequestVotes to both leader and follower.
+ cars.tell(new RequestVote(initialState.getCurrentTerm() + 1, "member-3-shard-cars", -1, -1), null);
+ followerCars.tell(new RequestVote(initialState.getCurrentTerm() + 1, "member-3-shard-cars", -1, -1), null);
+ cars.tell(new RequestVote(initialState.getCurrentTerm() + 3, "member-3-shard-cars", -1, -1), null);
+ followerCars.tell(new RequestVote(initialState.getCurrentTerm() + 3, "member-3-shard-cars", -1, -1), null);
+
+ OnDemandRaftState stateAfter = (OnDemandRaftState) leaderDistributedDataStore.getActorUtils()
+ .executeOperation(cars, GetOnDemandRaftState.INSTANCE);
+ OnDemandRaftState followerState = (OnDemandRaftState) followerDistributedDataStore.getActorUtils()
+ .executeOperation(cars, GetOnDemandRaftState.INSTANCE);
+
+ assertEquals(initialState.getCurrentTerm(), stateAfter.getCurrentTerm());
+ assertEquals(initialState.getCurrentTerm(), followerState.getCurrentTerm());
+
+ ds2.close();
+ }
+
+ @Test
+ public void testInstallSnapshot() throws Exception {
+ final String testName = "testInstallSnapshot";
+ final String leaderCarShardName = "member-1-shard-cars-" + testName;
+ final String followerCarShardName = "member-2-shard-cars-" + testName;
+
+ // Setup a saved snapshot on the leader. The follower will startup with no data and the leader should
+ // install a snapshot to sync the follower.
+
+ DataTree tree = new InMemoryDataTreeFactory().create(DataTreeConfiguration.DEFAULT_CONFIGURATION,
+ SchemaContextHelper.full());
+
+ final ContainerNode carsNode = CarsModel.newCarsNode(
+ CarsModel.newCarsMapNode(CarsModel.newCarEntry("optima", Uint64.valueOf(20000))));
+ AbstractShardTest.writeToStore(tree, CarsModel.BASE_PATH, carsNode);
+
+ final NormalizedNode<?, ?> snapshotRoot = AbstractShardTest.readStore(tree, YangInstanceIdentifier.empty());
+ final Snapshot initialSnapshot = Snapshot.create(
+ new ShardSnapshotState(new MetadataShardDataTreeSnapshot(snapshotRoot)),
+ Collections.emptyList(), 5, 1, 5, 1, 1, null, null);
+ InMemorySnapshotStore.addSnapshot(leaderCarShardName, initialSnapshot);
+
+ InMemorySnapshotStore.addSnapshotSavedLatch(leaderCarShardName);
+ InMemorySnapshotStore.addSnapshotSavedLatch(followerCarShardName);
+
+ initDatastoresWithCars(testName);
+
+ final Optional<NormalizedNode<?, ?>> readOptional = leaderDistributedDataStore.newReadOnlyTransaction().read(
+ CarsModel.BASE_PATH).get(5, TimeUnit.SECONDS);
+ assertTrue("isPresent", readOptional.isPresent());
+ assertEquals("Node", carsNode, readOptional.get());
+
+ verifySnapshot(InMemorySnapshotStore.waitForSavedSnapshot(leaderCarShardName, Snapshot.class),
+ initialSnapshot, snapshotRoot);
+
+ verifySnapshot(InMemorySnapshotStore.waitForSavedSnapshot(followerCarShardName, Snapshot.class),
+ initialSnapshot, snapshotRoot);
+ }
+
+ @Test
+ public void testReadWriteMessageSlicing() throws Exception {
+ // The slicing is only implemented for tell-based protocol
+ Assume.assumeTrue(ClientBackedDataStore.class.isAssignableFrom(testParameter));
+
+ leaderDatastoreContextBuilder.maximumMessageSliceSize(100);
+ followerDatastoreContextBuilder.maximumMessageSliceSize(100);
+ initDatastoresWithCars("testLargeReadReplySlicing");
+
+ final DOMStoreReadWriteTransaction rwTx = followerDistributedDataStore.newReadWriteTransaction();
+
+ final NormalizedNode<?, ?> carsNode = CarsModel.create();
+ rwTx.write(CarsModel.BASE_PATH, carsNode);
+
+ verifyNode(rwTx, CarsModel.BASE_PATH, carsNode);
+ }
+
+ @SuppressWarnings("IllegalCatch")
+ @Test
+ public void testRaftCallbackDuringLeadershipDrop() throws Exception {
+ final String testName = "testRaftCallbackDuringLeadershipDrop";
+ initDatastores(testName, MODULE_SHARDS_CARS_1_2_3, CARS);
+
+ final ExecutorService executor = Executors.newSingleThreadExecutor();
+
+ final IntegrationTestKit follower2TestKit = new IntegrationTestKit(follower2System,
+ DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build()).operationTimeoutInMillis(500)
+ .shardLeaderElectionTimeoutInSeconds(3600),
+ commitTimeout);
+
+ final DOMStoreWriteTransaction initialWriteTx = leaderDistributedDataStore.newWriteOnlyTransaction();
+ initialWriteTx.write(CarsModel.BASE_PATH, CarsModel.emptyContainer());
+ leaderTestKit.doCommit(initialWriteTx.ready());
+
+ try (AbstractDataStore follower2DistributedDataStore = follower2TestKit.setupAbstractDataStore(
+ testParameter, testName, MODULE_SHARDS_CARS_1_2_3, false)) {
+
+ final ActorRef member3Cars = ((LocalShardStore) follower2DistributedDataStore).getLocalShards()
+ .getLocalShards().get("cars").getActor();
+ final ActorRef member2Cars = ((LocalShardStore)followerDistributedDataStore).getLocalShards()
+ .getLocalShards().get("cars").getActor();
+ member2Cars.tell(new StartDropMessages(AppendEntries.class), null);
+ member3Cars.tell(new StartDropMessages(AppendEntries.class), null);
+
+ final DOMStoreWriteTransaction newTx = leaderDistributedDataStore.newWriteOnlyTransaction();
+ newTx.write(CarsModel.CAR_LIST_PATH, CarsModel.newCarMapNode());
+ final AtomicBoolean submitDone = new AtomicBoolean(false);
+ executor.submit(() -> {
+ try {
+ leaderTestKit.doCommit(newTx.ready());
+ submitDone.set(true);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+ final ActorRef leaderCars = ((LocalShardStore) leaderDistributedDataStore).getLocalShards()
+ .getLocalShards().get("cars").getActor();
+ await().atMost(10, TimeUnit.SECONDS)
+ .until(() -> ((OnDemandRaftState) leaderDistributedDataStore.getActorUtils()
+ .executeOperation(leaderCars, GetOnDemandRaftState.INSTANCE)).getLastIndex() >= 1);
+
+ final OnDemandRaftState raftState = (OnDemandRaftState)leaderDistributedDataStore.getActorUtils()
+ .executeOperation(leaderCars, GetOnDemandRaftState.INSTANCE);
+
+ // Simulate a follower not receiving heartbeats but still being able to send messages ie RequestVote with
+ // new term(switching to candidate after election timeout)
+ leaderCars.tell(new RequestVote(raftState.getCurrentTerm() + 1,
+ "member-3-shard-cars-testRaftCallbackDuringLeadershipDrop", -1,
+ -1), member3Cars);
+
+ member2Cars.tell(new StopDropMessages(AppendEntries.class), null);
+ member3Cars.tell(new StopDropMessages(AppendEntries.class), null);
+
+ await("Is tx stuck in COMMIT_PENDING")
+ .atMost(10, TimeUnit.SECONDS).untilAtomic(submitDone, equalTo(true));
+
+ }
+
+ executor.shutdownNow();
+ }
+
+ private static void verifySnapshot(final Snapshot actual, final Snapshot expected,
+ final NormalizedNode<?, ?> expRoot) {
+ assertEquals("Snapshot getLastAppliedTerm", expected.getLastAppliedTerm(), actual.getLastAppliedTerm());
+ assertEquals("Snapshot getLastAppliedIndex", expected.getLastAppliedIndex(), actual.getLastAppliedIndex());
+ assertEquals("Snapshot getLastTerm", expected.getLastTerm(), actual.getLastTerm());
+ assertEquals("Snapshot getLastIndex", expected.getLastIndex(), actual.getLastIndex());
+ assertEquals("Snapshot state type", ShardSnapshotState.class, actual.getState().getClass());
+ MetadataShardDataTreeSnapshot shardSnapshot =
+ (MetadataShardDataTreeSnapshot) ((ShardSnapshotState)actual.getState()).getSnapshot();
+ assertEquals("Snapshot root node", expRoot, shardSnapshot.getRootNode().get());
+ }