X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?p=controller.git;a=blobdiff_plain;f=opendaylight%2Fmd-sal%2Fsal-distributed-datastore%2Fsrc%2Ftest%2Fjava%2Forg%2Fopendaylight%2Fcontroller%2Fcluster%2Fdatastore%2FShardTest.java;h=adc7f4706c1e871f6ddab28bf8364e06a975d0fc;hp=2792342ab2f3921f451999651477edca5664397b;hb=3104f91c7d1b3ee5914d8778f87315f4ac64036d;hpb=f3fc2b47ca3e5be0b95a273664b6003d45fb9dd8 diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/ShardTest.java b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/ShardTest.java index 2792342ab2..adc7f4706c 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/ShardTest.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/ShardTest.java @@ -5,24 +5,24 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; -import static org.opendaylight.controller.cluster.datastore.messages.CreateTransaction.CURRENT_VERSION; +import static org.opendaylight.controller.cluster.datastore.DataStoreVersions.CURRENT_VERSION; import akka.actor.ActorRef; +import akka.actor.ActorSelection; import akka.actor.PoisonPill; import akka.actor.Props; import akka.dispatch.Dispatchers; import akka.dispatch.OnComplete; import akka.japi.Creator; +import akka.japi.Procedure; import akka.pattern.Patterns; +import akka.persistence.SnapshotSelectionCriteria; import akka.testkit.TestActorRef; import akka.util.Timeout; import com.google.common.base.Function; import com.google.common.base.Optional; -import com.google.common.util.concurrent.CheckedFuture; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; @@ -30,22 +30,21 @@ import com.google.common.util.concurrent.Uninterruptibles; import java.io.IOException; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import org.junit.After; -import org.junit.Before; import org.junit.Test; import org.mockito.InOrder; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.opendaylight.controller.cluster.datastore.identifiers.ShardIdentifier; +import org.opendaylight.controller.cluster.DataPersistenceProvider; import org.opendaylight.controller.cluster.datastore.messages.AbortTransaction; import org.opendaylight.controller.cluster.datastore.messages.AbortTransactionReply; +import org.opendaylight.controller.cluster.datastore.messages.BatchedModifications; +import org.opendaylight.controller.cluster.datastore.messages.BatchedModificationsReply; import org.opendaylight.controller.cluster.datastore.messages.CanCommitTransaction; import org.opendaylight.controller.cluster.datastore.messages.CanCommitTransactionReply; import org.opendaylight.controller.cluster.datastore.messages.CommitTransaction; @@ -53,84 +52,58 @@ import org.opendaylight.controller.cluster.datastore.messages.CommitTransactionR import org.opendaylight.controller.cluster.datastore.messages.CreateTransaction; import org.opendaylight.controller.cluster.datastore.messages.ForwardedReadyTransaction; import org.opendaylight.controller.cluster.datastore.messages.PeerAddressResolved; +import org.opendaylight.controller.cluster.datastore.messages.ReadData; +import org.opendaylight.controller.cluster.datastore.messages.ReadDataReply; import org.opendaylight.controller.cluster.datastore.messages.ReadyTransactionReply; import org.opendaylight.controller.cluster.datastore.messages.RegisterChangeListener; import org.opendaylight.controller.cluster.datastore.messages.RegisterChangeListenerReply; import org.opendaylight.controller.cluster.datastore.messages.UpdateSchemaContext; +import org.opendaylight.controller.cluster.datastore.modification.DeleteModification; import org.opendaylight.controller.cluster.datastore.modification.MergeModification; import org.opendaylight.controller.cluster.datastore.modification.Modification; +import org.opendaylight.controller.cluster.datastore.modification.ModificationPayload; import org.opendaylight.controller.cluster.datastore.modification.MutableCompositeModification; import org.opendaylight.controller.cluster.datastore.modification.WriteModification; -import org.opendaylight.controller.cluster.datastore.node.NormalizedNodeToNodeCodec; -import org.opendaylight.controller.cluster.datastore.utils.InMemoryJournal; -import org.opendaylight.controller.cluster.datastore.utils.InMemorySnapshotStore; +import org.opendaylight.controller.cluster.datastore.utils.MessageCollectorActor; import org.opendaylight.controller.cluster.datastore.utils.MockDataChangeListener; +import org.opendaylight.controller.cluster.datastore.utils.SerializationUtils; +import org.opendaylight.controller.cluster.notifications.RegisterRoleChangeListener; +import org.opendaylight.controller.cluster.notifications.RegisterRoleChangeListenerReply; import org.opendaylight.controller.cluster.raft.ReplicatedLogEntry; import org.opendaylight.controller.cluster.raft.ReplicatedLogImplEntry; import org.opendaylight.controller.cluster.raft.Snapshot; -import org.opendaylight.controller.cluster.raft.base.messages.ApplyLogEntries; +import org.opendaylight.controller.cluster.raft.base.messages.ApplyJournalEntries; import org.opendaylight.controller.cluster.raft.base.messages.ApplySnapshot; import org.opendaylight.controller.cluster.raft.base.messages.ApplyState; import org.opendaylight.controller.cluster.raft.base.messages.CaptureSnapshot; import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout; +import org.opendaylight.controller.cluster.raft.base.messages.FollowerInitialSyncUpStatus; import org.opendaylight.controller.cluster.raft.client.messages.FindLeader; import org.opendaylight.controller.cluster.raft.client.messages.FindLeaderReply; -import org.opendaylight.controller.cluster.raft.protobuff.client.messages.CompositeModificationByteStringPayload; -import org.opendaylight.controller.cluster.raft.protobuff.client.messages.CompositeModificationPayload; -import org.opendaylight.controller.cluster.raft.protobuff.client.messages.Payload; +import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal; +import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore; import org.opendaylight.controller.md.cluster.datastore.model.SchemaContextHelper; import org.opendaylight.controller.md.cluster.datastore.model.TestModel; import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker; -import org.opendaylight.controller.md.sal.common.api.data.AsyncDataChangeEvent; -import org.opendaylight.controller.md.sal.common.api.data.AsyncDataChangeListener; import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException; import org.opendaylight.controller.md.sal.dom.store.impl.InMemoryDOMDataStore; import org.opendaylight.controller.md.sal.dom.store.impl.InMemoryDOMDataStoreFactory; -import org.opendaylight.controller.protobuff.messages.common.NormalizedNodeMessages; +import org.opendaylight.controller.protobuff.messages.cohort3pc.ThreePhaseCommitCohortMessages; import org.opendaylight.controller.protobuff.messages.transaction.ShardTransactionMessages.CreateTransactionReply; -import org.opendaylight.controller.sal.core.spi.data.DOMStoreReadTransaction; import org.opendaylight.controller.sal.core.spi.data.DOMStoreThreePhaseCommitCohort; import org.opendaylight.controller.sal.core.spi.data.DOMStoreWriteTransaction; import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier; import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.PathArgument; +import org.opendaylight.yangtools.yang.data.api.schema.ContainerNode; import org.opendaylight.yangtools.yang.data.api.schema.DataContainerChild; import org.opendaylight.yangtools.yang.data.api.schema.MapEntryNode; import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode; import org.opendaylight.yangtools.yang.data.impl.schema.ImmutableNodes; -import org.opendaylight.yangtools.yang.model.api.SchemaContext; import scala.concurrent.Await; import scala.concurrent.Future; import scala.concurrent.duration.FiniteDuration; -public class ShardTest extends AbstractActorTest { - - private static final SchemaContext SCHEMA_CONTEXT = TestModel.createTestContext(); - - private static final AtomicInteger NEXT_SHARD_NUM = new AtomicInteger(); - - private final ShardIdentifier shardID = ShardIdentifier.builder().memberName("member-1") - .shardName("inventory").type("config" + NEXT_SHARD_NUM.getAndIncrement()).build(); - - private DatastoreContext dataStoreContext = DatastoreContext.newBuilder(). - shardJournalRecoveryLogBatchSize(3).shardSnapshotBatchCount(5000). - shardHeartbeatIntervalInMillis(100).build(); - - @Before - public void setUp() { - InMemorySnapshotStore.clear(); - InMemoryJournal.clear(); - } - - @After - public void tearDown() { - InMemorySnapshotStore.clear(); - InMemoryJournal.clear(); - } - - private Props newShardProps() { - return Shard.props(shardID, Collections.emptyMap(), - dataStoreContext, SCHEMA_CONTEXT); - } +public class ShardTest extends AbstractShardTest { @Test public void testRegisterChangeListener() throws Exception { @@ -185,8 +158,8 @@ public class ShardTest extends AbstractActorTest { @Override public Shard create() throws Exception { - return new Shard(shardID, Collections.emptyMap(), - dataStoreContext, SCHEMA_CONTEXT) { + return new Shard(shardID, Collections.emptyMap(), + newDatastoreContext(), SCHEMA_CONTEXT) { @Override public void onReceiveCommand(final Object message) throws Exception { if(message instanceof ElectionTimeout && firstElectionTimeout) { @@ -313,8 +286,8 @@ public class ShardTest extends AbstractActorTest { final CountDownLatch recoveryComplete = new CountDownLatch(1); class TestShard extends Shard { TestShard() { - super(shardID, Collections.singletonMap(shardID, null), - dataStoreContext, SCHEMA_CONTEXT); + super(shardID, Collections.singletonMap(shardID.toString(), null), + newDatastoreContext(), SCHEMA_CONTEXT); } Map getPeerAddresses() { @@ -344,7 +317,7 @@ public class ShardTest extends AbstractActorTest { Uninterruptibles.awaitUninterruptibly(recoveryComplete, 5, TimeUnit.SECONDS)); String address = "akka://foobar"; - shard.underlyingActor().onReceiveCommand(new PeerAddressResolved(shardID, address)); + shard.underlyingActor().onReceiveCommand(new PeerAddressResolved(shardID.toString(), address)); assertEquals("getPeerAddresses", address, ((TestShard)shard.underlyingActor()).getPeerAddresses().get(shardID.toString())); @@ -358,25 +331,23 @@ public class ShardTest extends AbstractActorTest { TestActorRef shard = TestActorRef.create(getSystem(), newShardProps(), "testApplySnapshot"); - NormalizedNodeToNodeCodec codec = - new NormalizedNodeToNodeCodec(SCHEMA_CONTEXT); + InMemoryDOMDataStore store = new InMemoryDOMDataStore("OPER", MoreExecutors.sameThreadExecutor()); + store.onGlobalContextUpdated(SCHEMA_CONTEXT); - writeToStore(shard, TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); + writeToStore(store, TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); YangInstanceIdentifier root = YangInstanceIdentifier.builder().build(); - NormalizedNode expected = readStore(shard, root); - - NormalizedNodeMessages.Container encode = codec.encode(expected); + NormalizedNode expected = readStore(store, root); ApplySnapshot applySnapshot = new ApplySnapshot(Snapshot.create( - encode.getNormalizedNode().toByteString().toByteArray(), + SerializationUtils.serializeNormalizedNode(expected), Collections.emptyList(), 1, 2, 3, 4)); shard.underlyingActor().onReceiveCommand(applySnapshot); NormalizedNode actual = readStore(shard, root); - assertEquals(expected, actual); + assertEquals("Root node", expected, actual); shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); } @@ -388,11 +359,8 @@ public class ShardTest extends AbstractActorTest { NormalizedNode node = ImmutableNodes.containerNode(TestModel.TEST_QNAME); - MutableCompositeModification compMod = new MutableCompositeModification(); - compMod.addModification(new WriteModification(TestModel.TEST_PATH, node, SCHEMA_CONTEXT)); - Payload payload = new CompositeModificationPayload(compMod.toSerializable()); - ApplyState applyState = new ApplyState(null, "test", - new ReplicatedLogImplEntry(1, 2, payload)); + ApplyState applyState = new ApplyState(null, "test", new ReplicatedLogImplEntry(1, 2, + newModificationPayload(new WriteModification(TestModel.TEST_PATH, node)))); shard.underlyingActor().onReceiveCommand(applyState); @@ -402,7 +370,6 @@ public class ShardTest extends AbstractActorTest { shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); } - @SuppressWarnings("serial") @Test public void testRecovery() throws Exception { @@ -411,186 +378,47 @@ public class ShardTest extends AbstractActorTest { InMemoryDOMDataStore testStore = InMemoryDOMDataStoreFactory.create("Test", null, null); testStore.onGlobalContextUpdated(SCHEMA_CONTEXT); - DOMStoreWriteTransaction writeTx = testStore.newWriteOnlyTransaction(); - writeTx.write(TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); - DOMStoreThreePhaseCommitCohort commitCohort = writeTx.ready(); - commitCohort.preCommit().get(); - commitCohort.commit().get(); + writeToStore(testStore, TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); - DOMStoreReadTransaction readTx = testStore.newReadOnlyTransaction(); - NormalizedNode root = readTx.read(YangInstanceIdentifier.builder().build()).get().get(); + NormalizedNode root = readStore(testStore, YangInstanceIdentifier.builder().build()); InMemorySnapshotStore.addSnapshot(shardID.toString(), Snapshot.create( - new NormalizedNodeToNodeCodec(SCHEMA_CONTEXT).encode( - root). - getNormalizedNode().toByteString().toByteArray(), - Collections.emptyList(), 0, 1, -1, -1)); + SerializationUtils.serializeNormalizedNode(root), + Collections.emptyList(), 0, 1, -1, -1)); // Set up the InMemoryJournal. - InMemoryJournal.addEntry(shardID.toString(), 0, new ReplicatedLogImplEntry(0, 1, newPayload( + InMemoryJournal.addEntry(shardID.toString(), 0, new ReplicatedLogImplEntry(0, 1, newModificationPayload( new WriteModification(TestModel.OUTER_LIST_PATH, - ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build(), - SCHEMA_CONTEXT)))); + ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build())))); int nListEntries = 16; Set listEntryKeys = new HashSet<>(); - for(int i = 1; i <= nListEntries-5; i++) { - listEntryKeys.add(Integer.valueOf(i)); - YangInstanceIdentifier path = YangInstanceIdentifier.builder(TestModel.OUTER_LIST_PATH) - .nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, i).build(); - Modification mod = new MergeModification(path, - ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, i), - SCHEMA_CONTEXT); - InMemoryJournal.addEntry(shardID.toString(), i, new ReplicatedLogImplEntry(i, 1, - newPayload(mod))); - } - // Add some of the new CompositeModificationByteStringPayload - for(int i = 11; i <= nListEntries; i++) { + // Add some ModificationPayload entries + for(int i = 1; i <= nListEntries; i++) { listEntryKeys.add(Integer.valueOf(i)); YangInstanceIdentifier path = YangInstanceIdentifier.builder(TestModel.OUTER_LIST_PATH) .nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, i).build(); Modification mod = new MergeModification(path, - ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, i), - SCHEMA_CONTEXT); + ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, i)); InMemoryJournal.addEntry(shardID.toString(), i, new ReplicatedLogImplEntry(i, 1, - newByteStringPayload(mod))); + newModificationPayload(mod))); } - InMemoryJournal.addEntry(shardID.toString(), nListEntries + 1, - new ApplyLogEntries(nListEntries)); - - // Create the actor and wait for recovery complete. + new ApplyJournalEntries(nListEntries)); - final CountDownLatch recoveryComplete = new CountDownLatch(1); - - Creator creator = new Creator() { - @Override - public Shard create() throws Exception { - return new Shard(shardID, Collections.emptyMap(), - dataStoreContext, SCHEMA_CONTEXT) { - @Override - protected void onRecoveryComplete() { - try { - super.onRecoveryComplete(); - } finally { - recoveryComplete.countDown(); - } - } - }; - } - }; - - TestActorRef shard = TestActorRef.create(getSystem(), - Props.create(new DelegatingShardCreator(creator)), "testRecovery"); - - assertEquals("Recovery complete", true, recoveryComplete.await(5, TimeUnit.SECONDS)); - - // Verify data in the data store. - - NormalizedNode outerList = readStore(shard, TestModel.OUTER_LIST_PATH); - assertNotNull(TestModel.OUTER_LIST_QNAME.getLocalName() + " not found", outerList); - assertTrue(TestModel.OUTER_LIST_QNAME.getLocalName() + " value is not Iterable", - outerList.getValue() instanceof Iterable); - for(Object entry: (Iterable) outerList.getValue()) { - assertTrue(TestModel.OUTER_LIST_QNAME.getLocalName() + " entry is not MapEntryNode", - entry instanceof MapEntryNode); - MapEntryNode mapEntry = (MapEntryNode)entry; - Optional> idLeaf = - mapEntry.getChild(new YangInstanceIdentifier.NodeIdentifier(TestModel.ID_QNAME)); - assertTrue("Missing leaf " + TestModel.ID_QNAME.getLocalName(), idLeaf.isPresent()); - Object value = idLeaf.get().getValue(); - assertTrue("Unexpected value for leaf "+ TestModel.ID_QNAME.getLocalName() + ": " + value, - listEntryKeys.remove(value)); - } - - if(!listEntryKeys.isEmpty()) { - fail("Missing " + TestModel.OUTER_LIST_QNAME.getLocalName() + " entries with keys: " + - listEntryKeys); - } - - assertEquals("Last log index", nListEntries, - shard.underlyingActor().getShardMBean().getLastLogIndex()); - assertEquals("Commit index", nListEntries, - shard.underlyingActor().getShardMBean().getCommitIndex()); - assertEquals("Last applied", nListEntries, - shard.underlyingActor().getShardMBean().getLastApplied()); - - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); + testRecovery(listEntryKeys); } - private CompositeModificationPayload newPayload(final Modification... mods) { + private ModificationPayload newModificationPayload(final Modification... mods) throws IOException { MutableCompositeModification compMod = new MutableCompositeModification(); for(Modification mod: mods) { compMod.addModification(mod); } - return new CompositeModificationPayload(compMod.toSerializable()); - } - - private CompositeModificationByteStringPayload newByteStringPayload(final Modification... mods) { - MutableCompositeModification compMod = new MutableCompositeModification(); - for(Modification mod: mods) { - compMod.addModification(mod); - } - - return new CompositeModificationByteStringPayload(compMod.toSerializable()); - } - - - private DOMStoreThreePhaseCommitCohort setupMockWriteTransaction(final String cohortName, - final InMemoryDOMDataStore dataStore, final YangInstanceIdentifier path, final NormalizedNode data, - final MutableCompositeModification modification) { - return setupMockWriteTransaction(cohortName, dataStore, path, data, modification, null); - } - - private DOMStoreThreePhaseCommitCohort setupMockWriteTransaction(final String cohortName, - final InMemoryDOMDataStore dataStore, final YangInstanceIdentifier path, final NormalizedNode data, - final MutableCompositeModification modification, - final Function> preCommit) { - - DOMStoreWriteTransaction tx = dataStore.newWriteOnlyTransaction(); - tx.write(path, data); - final DOMStoreThreePhaseCommitCohort realCohort = tx.ready(); - DOMStoreThreePhaseCommitCohort cohort = mock(DOMStoreThreePhaseCommitCohort.class, cohortName); - - doAnswer(new Answer>() { - @Override - public ListenableFuture answer(final InvocationOnMock invocation) { - return realCohort.canCommit(); - } - }).when(cohort).canCommit(); - - doAnswer(new Answer>() { - @Override - public ListenableFuture answer(final InvocationOnMock invocation) throws Throwable { - if(preCommit != null) { - return preCommit.apply(realCohort); - } else { - return realCohort.preCommit(); - } - } - }).when(cohort).preCommit(); - - doAnswer(new Answer>() { - @Override - public ListenableFuture answer(final InvocationOnMock invocation) throws Throwable { - return realCohort.commit(); - } - }).when(cohort).commit(); - - doAnswer(new Answer>() { - @Override - public ListenableFuture answer(final InvocationOnMock invocation) throws Throwable { - return realCohort.abort(); - } - }).when(cohort).abort(); - - modification.addModification(new WriteModification(path, data, SCHEMA_CONTEXT)); - - return cohort; + return new ModificationPayload(compMod); } @SuppressWarnings({ "unchecked" }) @@ -603,42 +431,42 @@ public class ShardTest extends AbstractActorTest { waitUntilLeader(shard); - // Setup 3 simulated transactions with mock cohorts backed by real cohorts. - - InMemoryDOMDataStore dataStore = shard.underlyingActor().getDataStore(); - - String transactionID1 = "tx1"; - MutableCompositeModification modification1 = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort1 = setupMockWriteTransaction("cohort1", dataStore, - TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME), modification1); + final String transactionID1 = "tx1"; + final String transactionID2 = "tx2"; + final String transactionID3 = "tx3"; - String transactionID2 = "tx2"; - MutableCompositeModification modification2 = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort2 = setupMockWriteTransaction("cohort2", dataStore, - TestModel.OUTER_LIST_PATH, - ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build(), - modification2); + final AtomicReference mockCohort1 = new AtomicReference<>(); + final AtomicReference mockCohort2 = new AtomicReference<>(); + final AtomicReference mockCohort3 = new AtomicReference<>(); + ShardCommitCoordinator.CohortDecorator cohortDecorator = new ShardCommitCoordinator.CohortDecorator() { + @Override + public DOMStoreThreePhaseCommitCohort decorate(String transactionID, DOMStoreThreePhaseCommitCohort actual) { + if(transactionID.equals(transactionID1)) { + mockCohort1.set(createDelegatingMockCohort("cohort1", actual)); + return mockCohort1.get(); + } else if(transactionID.equals(transactionID2)) { + mockCohort2.set(createDelegatingMockCohort("cohort2", actual)); + return mockCohort2.get(); + } else { + mockCohort3.set(createDelegatingMockCohort("cohort3", actual)); + return mockCohort3.get(); + } + } + }; - String transactionID3 = "tx3"; - MutableCompositeModification modification3 = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort3 = setupMockWriteTransaction("cohort3", dataStore, - YangInstanceIdentifier.builder(TestModel.OUTER_LIST_PATH) - .nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1).build(), - ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1), - modification3); + shard.underlyingActor().getCommitCoordinator().setCohortDecorator(cohortDecorator); long timeoutSec = 5; final FiniteDuration duration = FiniteDuration.create(timeoutSec, TimeUnit.SECONDS); final Timeout timeout = new Timeout(duration); - // Simulate the ForwardedReadyTransaction message for the first Tx that would be sent - // by the ShardTransaction. + // Send a BatchedModifications message for the first transaction. - shard.tell(new ForwardedReadyTransaction(transactionID1, CURRENT_VERSION, - cohort1, modification1, true), getRef()); - ReadyTransactionReply readyReply = ReadyTransactionReply.fromSerializable( - expectMsgClass(duration, ReadyTransactionReply.SERIALIZABLE_CLASS)); - assertEquals("Cohort path", shard.path().toString(), readyReply.getCohortPath()); + shard.tell(newBatchedModifications(transactionID1, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true), getRef()); + BatchedModificationsReply batchedReply = expectMsgClass(duration, BatchedModificationsReply.class); + assertEquals("getCohortPath", shard.path().toString(), batchedReply.getCohortPath()); + assertEquals("getNumBatched", 1, batchedReply.getNumBatched()); // Send the CanCommitTransaction message for the first Tx. @@ -647,15 +475,16 @@ public class ShardTest extends AbstractActorTest { expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); assertEquals("Can commit", true, canCommitReply.getCanCommit()); - // Send the ForwardedReadyTransaction for the next 2 Tx's. + // Send BatchedModifications for the next 2 Tx's. - shard.tell(new ForwardedReadyTransaction(transactionID2, CURRENT_VERSION, - cohort2, modification2, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.SERIALIZABLE_CLASS); + shard.tell(newBatchedModifications(transactionID2, TestModel.OUTER_LIST_PATH, + ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build(), true), getRef()); + expectMsgClass(duration, BatchedModificationsReply.class); - shard.tell(new ForwardedReadyTransaction(transactionID3, CURRENT_VERSION, - cohort3, modification3, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.SERIALIZABLE_CLASS); + shard.tell(newBatchedModifications(transactionID3, YangInstanceIdentifier.builder( + TestModel.OUTER_LIST_PATH).nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1).build(), + ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1), true), getRef()); + expectMsgClass(duration, BatchedModificationsReply.class); // Send the CanCommitTransaction message for the next 2 Tx's. These should get queued and // processed after the first Tx completes. @@ -748,16 +577,16 @@ public class ShardTest extends AbstractActorTest { assertEquals("Commits complete", true, done); - InOrder inOrder = inOrder(cohort1, cohort2, cohort3); - inOrder.verify(cohort1).canCommit(); - inOrder.verify(cohort1).preCommit(); - inOrder.verify(cohort1).commit(); - inOrder.verify(cohort2).canCommit(); - inOrder.verify(cohort2).preCommit(); - inOrder.verify(cohort2).commit(); - inOrder.verify(cohort3).canCommit(); - inOrder.verify(cohort3).preCommit(); - inOrder.verify(cohort3).commit(); + InOrder inOrder = inOrder(mockCohort1.get(), mockCohort2.get(), mockCohort3.get()); + inOrder.verify(mockCohort1.get()).canCommit(); + inOrder.verify(mockCohort1.get()).preCommit(); + inOrder.verify(mockCohort1.get()).commit(); + inOrder.verify(mockCohort2.get()).canCommit(); + inOrder.verify(mockCohort2.get()).preCommit(); + inOrder.verify(mockCohort2.get()).commit(); + inOrder.verify(mockCohort3.get()).canCommit(); + inOrder.verify(mockCohort3.get()).preCommit(); + inOrder.verify(mockCohort3.get()).commit(); // Verify data in the data store. @@ -774,20 +603,347 @@ public class ShardTest extends AbstractActorTest { assertTrue("Missing leaf " + TestModel.ID_QNAME.getLocalName(), idLeaf.isPresent()); assertEquals(TestModel.ID_QNAME.getLocalName() + " value", 1, idLeaf.get().getValue()); - for(int i = 0; i < 20 * 5; i++) { - long lastLogIndex = shard.underlyingActor().getShardMBean().getLastLogIndex(); - if(lastLogIndex == 2) { - break; + verifyLastApplied(shard, 2); + + shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); + }}; + } + + private BatchedModifications newBatchedModifications(String transactionID, YangInstanceIdentifier path, + NormalizedNode data, boolean ready) { + return newBatchedModifications(transactionID, null, path, data, ready); + } + + private BatchedModifications newBatchedModifications(String transactionID, String transactionChainID, + YangInstanceIdentifier path, NormalizedNode data, boolean ready) { + BatchedModifications batched = new BatchedModifications(transactionID, CURRENT_VERSION, transactionChainID); + batched.addModification(new WriteModification(path, data)); + batched.setReady(ready); + return batched; + } + + @SuppressWarnings("unchecked") + @Test + public void testMultipleBatchedModifications() throws Throwable { + new ShardTestKit(getSystem()) {{ + final TestActorRef shard = TestActorRef.create(getSystem(), + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testMultipleBatchedModifications"); + + waitUntilLeader(shard); + + final String transactionID = "tx"; + FiniteDuration duration = duration("5 seconds"); + + final AtomicReference mockCohort = new AtomicReference<>(); + ShardCommitCoordinator.CohortDecorator cohortDecorator = new ShardCommitCoordinator.CohortDecorator() { + @Override + public DOMStoreThreePhaseCommitCohort decorate(String txID, DOMStoreThreePhaseCommitCohort actual) { + if(mockCohort.get() == null) { + mockCohort.set(createDelegatingMockCohort("cohort", actual)); + } + + return mockCohort.get(); } - Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS); - } + }; + + shard.underlyingActor().getCommitCoordinator().setCohortDecorator(cohortDecorator); + + // Send a BatchedModifications to start a transaction. + + shard.tell(newBatchedModifications(transactionID, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), false), getRef()); + expectMsgClass(duration, BatchedModificationsReply.class); - assertEquals("Last log index", 2, shard.underlyingActor().getShardMBean().getLastLogIndex()); + // Send a couple more BatchedModifications. + + shard.tell(newBatchedModifications(transactionID, TestModel.OUTER_LIST_PATH, + ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build(), false), getRef()); + expectMsgClass(duration, BatchedModificationsReply.class); + + shard.tell(newBatchedModifications(transactionID, YangInstanceIdentifier.builder( + TestModel.OUTER_LIST_PATH).nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1).build(), + ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1), true), getRef()); + expectMsgClass(duration, BatchedModificationsReply.class); + + // Send the CanCommitTransaction message. + + shard.tell(new CanCommitTransaction(transactionID).toSerializable(), getRef()); + CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( + expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); + assertEquals("Can commit", true, canCommitReply.getCanCommit()); + + // Send the CanCommitTransaction message. + + shard.tell(new CommitTransaction(transactionID).toSerializable(), getRef()); + expectMsgClass(duration, CommitTransactionReply.SERIALIZABLE_CLASS); + + InOrder inOrder = inOrder(mockCohort.get()); + inOrder.verify(mockCohort.get()).canCommit(); + inOrder.verify(mockCohort.get()).preCommit(); + inOrder.verify(mockCohort.get()).commit(); + + // Verify data in the data store. + + NormalizedNode outerList = readStore(shard, TestModel.OUTER_LIST_PATH); + assertNotNull(TestModel.OUTER_LIST_QNAME.getLocalName() + " not found", outerList); + assertTrue(TestModel.OUTER_LIST_QNAME.getLocalName() + " value is not Iterable", + outerList.getValue() instanceof Iterable); + Object entry = ((Iterable)outerList.getValue()).iterator().next(); + assertTrue(TestModel.OUTER_LIST_QNAME.getLocalName() + " entry is not MapEntryNode", + entry instanceof MapEntryNode); + MapEntryNode mapEntry = (MapEntryNode)entry; + Optional> idLeaf = + mapEntry.getChild(new YangInstanceIdentifier.NodeIdentifier(TestModel.ID_QNAME)); + assertTrue("Missing leaf " + TestModel.ID_QNAME.getLocalName(), idLeaf.isPresent()); + assertEquals(TestModel.ID_QNAME.getLocalName() + " value", 1, idLeaf.get().getValue()); shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); }}; } + @Test + public void testBatchedModificationsOnTransactionChain() throws Throwable { + new ShardTestKit(getSystem()) {{ + final TestActorRef shard = TestActorRef.create(getSystem(), + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testBatchedModificationsOnTransactionChain"); + + waitUntilLeader(shard); + + String transactionChainID = "txChain"; + String transactionID1 = "tx1"; + String transactionID2 = "tx2"; + + FiniteDuration duration = duration("5 seconds"); + + // Send a BatchedModifications to start a chained write transaction and ready it. + + ContainerNode containerNode = ImmutableNodes.containerNode(TestModel.TEST_QNAME); + YangInstanceIdentifier path = TestModel.TEST_PATH; + shard.tell(newBatchedModifications(transactionID1, transactionChainID, path, + containerNode, true), getRef()); + expectMsgClass(duration, BatchedModificationsReply.class); + + // Create a read Tx on the same chain. + + shard.tell(new CreateTransaction(transactionID2, TransactionProxy.TransactionType.READ_ONLY.ordinal() , + transactionChainID).toSerializable(), getRef()); + + CreateTransactionReply createReply = expectMsgClass(duration("3 seconds"), CreateTransactionReply.class); + + getSystem().actorSelection(createReply.getTransactionActorPath()).tell(new ReadData(path), getRef()); + ReadDataReply readReply = expectMsgClass(duration("3 seconds"), ReadDataReply.class); + assertEquals("Read node", containerNode, readReply.getNormalizedNode()); + + // Commit the write transaction. + + shard.tell(new CanCommitTransaction(transactionID1).toSerializable(), getRef()); + CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( + expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); + assertEquals("Can commit", true, canCommitReply.getCanCommit()); + + shard.tell(new CommitTransaction(transactionID1).toSerializable(), getRef()); + expectMsgClass(duration, CommitTransactionReply.SERIALIZABLE_CLASS); + + // Verify data in the data store. + + NormalizedNode actualNode = readStore(shard, path); + assertEquals("Stored node", containerNode, actualNode); + + shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); + }}; + } + + @Test + public void testOnBatchedModificationsWhenNotLeader() { + final AtomicBoolean overrideLeaderCalls = new AtomicBoolean(); + new ShardTestKit(getSystem()) {{ + Creator creator = new Creator() { + @Override + public Shard create() throws Exception { + return new Shard(shardID, Collections.emptyMap(), + newDatastoreContext(), SCHEMA_CONTEXT) { + @Override + protected boolean isLeader() { + return overrideLeaderCalls.get() ? false : super.isLeader(); + } + + @Override + protected ActorSelection getLeader() { + return overrideLeaderCalls.get() ? getSystem().actorSelection(getRef().path()) : + super.getLeader(); + } + }; + } + }; + + TestActorRef shard = TestActorRef.create(getSystem(), + Props.create(new DelegatingShardCreator(creator)), "testOnBatchedModificationsWhenNotLeader"); + + waitUntilLeader(shard); + + overrideLeaderCalls.set(true); + + BatchedModifications batched = new BatchedModifications("tx", DataStoreVersions.CURRENT_VERSION, ""); + + shard.tell(batched, ActorRef.noSender()); + + expectMsgEquals(batched); + + shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); + }}; + } + + @Test + public void testCommitWithPersistenceDisabled() throws Throwable { + dataStoreContextBuilder.persistent(false); + new ShardTestKit(getSystem()) {{ + final TestActorRef shard = TestActorRef.create(getSystem(), + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testCommitWithPersistenceDisabled"); + + waitUntilLeader(shard); + + String transactionID = "tx"; + FiniteDuration duration = duration("5 seconds"); + + // Send a BatchedModifications to start a transaction. + + NormalizedNode containerNode = ImmutableNodes.containerNode(TestModel.TEST_QNAME); + shard.tell(newBatchedModifications(transactionID, TestModel.TEST_PATH, containerNode, true), getRef()); + expectMsgClass(duration, BatchedModificationsReply.class); + + // Send the CanCommitTransaction message. + + shard.tell(new CanCommitTransaction(transactionID).toSerializable(), getRef()); + CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( + expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); + assertEquals("Can commit", true, canCommitReply.getCanCommit()); + + // Send the CanCommitTransaction message. + + shard.tell(new CommitTransaction(transactionID).toSerializable(), getRef()); + expectMsgClass(duration, CommitTransactionReply.SERIALIZABLE_CLASS); + + NormalizedNode actualNode = readStore(shard, TestModel.TEST_PATH); + assertEquals(TestModel.TEST_QNAME.getLocalName(), containerNode, actualNode); + + shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); + }}; + } + + @Test + public void testCommitWhenTransactionHasNoModifications(){ + // Note that persistence is enabled which would normally result in the entry getting written to the journal + // but here that need not happen + new ShardTestKit(getSystem()) { + { + final TestActorRef shard = TestActorRef.create(getSystem(), + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testCommitWhenTransactionHasNoModifications"); + + waitUntilLeader(shard); + + String transactionID = "tx1"; + MutableCompositeModification modification = new MutableCompositeModification(); + DOMStoreThreePhaseCommitCohort cohort = mock(DOMStoreThreePhaseCommitCohort.class, "cohort1"); + doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort).canCommit(); + doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort).preCommit(); + doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort).commit(); + + FiniteDuration duration = duration("5 seconds"); + + // Simulate the ForwardedReadyTransaction messages that would be sent + // by the ShardTransaction. + + shard.tell(new ForwardedReadyTransaction(transactionID, CURRENT_VERSION, + cohort, modification, true), getRef()); + expectMsgClass(duration, ReadyTransactionReply.SERIALIZABLE_CLASS); + + // Send the CanCommitTransaction message. + + shard.tell(new CanCommitTransaction(transactionID).toSerializable(), getRef()); + CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( + expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); + assertEquals("Can commit", true, canCommitReply.getCanCommit()); + + shard.tell(new CommitTransaction(transactionID).toSerializable(), getRef()); + expectMsgClass(duration, ThreePhaseCommitCohortMessages.CommitTransactionReply.class); + + InOrder inOrder = inOrder(cohort); + inOrder.verify(cohort).canCommit(); + inOrder.verify(cohort).preCommit(); + inOrder.verify(cohort).commit(); + + // Use MBean for verification + // Committed transaction count should increase as usual + assertEquals(1,shard.underlyingActor().getShardMBean().getCommittedTransactionsCount()); + + // Commit index should not advance because this does not go into the journal + assertEquals(-1, shard.underlyingActor().getShardMBean().getCommitIndex()); + + shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); + + } + }; + } + + @Test + public void testCommitWhenTransactionHasModifications(){ + new ShardTestKit(getSystem()) { + { + final TestActorRef shard = TestActorRef.create(getSystem(), + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testCommitWhenTransactionHasModifications"); + + waitUntilLeader(shard); + + String transactionID = "tx1"; + MutableCompositeModification modification = new MutableCompositeModification(); + modification.addModification(new DeleteModification(YangInstanceIdentifier.builder().build())); + DOMStoreThreePhaseCommitCohort cohort = mock(DOMStoreThreePhaseCommitCohort.class, "cohort1"); + doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort).canCommit(); + doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort).preCommit(); + doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort).commit(); + + FiniteDuration duration = duration("5 seconds"); + + // Simulate the ForwardedReadyTransaction messages that would be sent + // by the ShardTransaction. + + shard.tell(new ForwardedReadyTransaction(transactionID, CURRENT_VERSION, + cohort, modification, true), getRef()); + expectMsgClass(duration, ReadyTransactionReply.SERIALIZABLE_CLASS); + + // Send the CanCommitTransaction message. + + shard.tell(new CanCommitTransaction(transactionID).toSerializable(), getRef()); + CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( + expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); + assertEquals("Can commit", true, canCommitReply.getCanCommit()); + + shard.tell(new CommitTransaction(transactionID).toSerializable(), getRef()); + expectMsgClass(duration, ThreePhaseCommitCohortMessages.CommitTransactionReply.class); + + InOrder inOrder = inOrder(cohort); + inOrder.verify(cohort).canCommit(); + inOrder.verify(cohort).preCommit(); + inOrder.verify(cohort).commit(); + + // Use MBean for verification + // Committed transaction count should increase as usual + assertEquals(1,shard.underlyingActor().getShardMBean().getCommittedTransactionsCount()); + + // Commit index should advance as we do not have an empty modification + assertEquals(0, shard.underlyingActor().getShardMBean().getCommitIndex()); + + shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); + + } + }; + } + @Test public void testCommitPhaseFailure() throws Throwable { new ShardTestKit(getSystem()) {{ @@ -797,34 +953,40 @@ public class ShardTest extends AbstractActorTest { waitUntilLeader(shard); - // Setup 2 simulated transactions with mock cohorts. The first one fails in the - // commit phase. + // Setup 2 mock cohorts. The first one fails in the commit phase. - String transactionID1 = "tx1"; - MutableCompositeModification modification1 = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort1 = mock(DOMStoreThreePhaseCommitCohort.class, "cohort1"); + final String transactionID1 = "tx1"; + final DOMStoreThreePhaseCommitCohort cohort1 = mock(DOMStoreThreePhaseCommitCohort.class, "cohort1"); doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort1).canCommit(); doReturn(Futures.immediateFuture(null)).when(cohort1).preCommit(); doReturn(Futures.immediateFailedFuture(new IllegalStateException("mock"))).when(cohort1).commit(); - String transactionID2 = "tx2"; - MutableCompositeModification modification2 = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort2 = mock(DOMStoreThreePhaseCommitCohort.class, "cohort2"); + final String transactionID2 = "tx2"; + final DOMStoreThreePhaseCommitCohort cohort2 = mock(DOMStoreThreePhaseCommitCohort.class, "cohort2"); doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort2).canCommit(); + ShardCommitCoordinator.CohortDecorator cohortDecorator = new ShardCommitCoordinator.CohortDecorator() { + @Override + public DOMStoreThreePhaseCommitCohort decorate(String transactionID, + DOMStoreThreePhaseCommitCohort actual) { + return transactionID1.equals(transactionID) ? cohort1 : cohort2; + } + }; + + shard.underlyingActor().getCommitCoordinator().setCohortDecorator(cohortDecorator); + FiniteDuration duration = duration("5 seconds"); final Timeout timeout = new Timeout(duration); - // Simulate the ForwardedReadyTransaction messages that would be sent - // by the ShardTransaction. + // Send BatchedModifications to start and ready each transaction. - shard.tell(new ForwardedReadyTransaction(transactionID1, CURRENT_VERSION, - cohort1, modification1, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.SERIALIZABLE_CLASS); + shard.tell(newBatchedModifications(transactionID1, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true), getRef()); + expectMsgClass(duration, BatchedModificationsReply.class); - shard.tell(new ForwardedReadyTransaction(transactionID2, CURRENT_VERSION, - cohort2, modification2, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.SERIALIZABLE_CLASS); + shard.tell(newBatchedModifications(transactionID2, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true), getRef()); + expectMsgClass(duration, BatchedModificationsReply.class); // Send the CanCommitTransaction message for the first Tx. @@ -877,19 +1039,27 @@ public class ShardTest extends AbstractActorTest { waitUntilLeader(shard); String transactionID = "tx1"; - MutableCompositeModification modification = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort = mock(DOMStoreThreePhaseCommitCohort.class, "cohort1"); + final DOMStoreThreePhaseCommitCohort cohort = mock(DOMStoreThreePhaseCommitCohort.class, "cohort1"); doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort).canCommit(); doReturn(Futures.immediateFailedFuture(new IllegalStateException("mock"))).when(cohort).preCommit(); + ShardCommitCoordinator.CohortDecorator cohortDecorator = new ShardCommitCoordinator.CohortDecorator() { + @Override + public DOMStoreThreePhaseCommitCohort decorate(String transactionID, + DOMStoreThreePhaseCommitCohort actual) { + return cohort; + } + }; + + shard.underlyingActor().getCommitCoordinator().setCohortDecorator(cohortDecorator); + FiniteDuration duration = duration("5 seconds"); - // Simulate the ForwardedReadyTransaction messages that would be sent - // by the ShardTransaction. + // Send BatchedModifications to start and ready a transaction. - shard.tell(new ForwardedReadyTransaction(transactionID, CURRENT_VERSION, - cohort, modification, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.SERIALIZABLE_CLASS); + shard.tell(newBatchedModifications(transactionID, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true), getRef()); + expectMsgClass(duration, BatchedModificationsReply.class); // Send the CanCommitTransaction message. @@ -924,16 +1094,24 @@ public class ShardTest extends AbstractActorTest { final FiniteDuration duration = duration("5 seconds"); String transactionID = "tx1"; - MutableCompositeModification modification = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort = mock(DOMStoreThreePhaseCommitCohort.class, "cohort1"); + final DOMStoreThreePhaseCommitCohort cohort = mock(DOMStoreThreePhaseCommitCohort.class, "cohort1"); doReturn(Futures.immediateFailedFuture(new IllegalStateException("mock"))).when(cohort).canCommit(); - // Simulate the ForwardedReadyTransaction messages that would be sent - // by the ShardTransaction. + ShardCommitCoordinator.CohortDecorator cohortDecorator = new ShardCommitCoordinator.CohortDecorator() { + @Override + public DOMStoreThreePhaseCommitCohort decorate(String transactionID, + DOMStoreThreePhaseCommitCohort actual) { + return cohort; + } + }; + + shard.underlyingActor().getCommitCoordinator().setCohortDecorator(cohortDecorator); - shard.tell(new ForwardedReadyTransaction(transactionID, CURRENT_VERSION, - cohort, modification, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.SERIALIZABLE_CLASS); + // Send BatchedModifications to start and ready a transaction. + + shard.tell(newBatchedModifications(transactionID, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true), getRef()); + expectMsgClass(duration, BatchedModificationsReply.class); // Send the CanCommitTransaction message. @@ -954,53 +1132,46 @@ public class ShardTest extends AbstractActorTest { waitUntilLeader(shard); final FiniteDuration duration = duration("5 seconds"); - final Timeout timeout = new Timeout(duration); - InMemoryDOMDataStore dataStore = shard.underlyingActor().getDataStore(); final String transactionID = "tx1"; - final CountDownLatch abortComplete = new CountDownLatch(1); Function> preCommit = new Function>() { @Override public ListenableFuture apply(final DOMStoreThreePhaseCommitCohort cohort) { ListenableFuture preCommitFuture = cohort.preCommit(); - Future abortFuture = Patterns.ask(shard, - new AbortTransaction(transactionID).toSerializable(), timeout); - abortFuture.onComplete(new OnComplete() { - @Override - public void onComplete(final Throwable e, final Object resp) { - abortComplete.countDown(); - } - }, getSystem().dispatcher()); + // Simulate an AbortTransaction message occurring during replication, after + // persisting and before finishing the commit to the in-memory store. + // We have no followers so due to optimizations in the RaftActor, it does not + // attempt replication and thus we can't send an AbortTransaction message b/c + // it would be processed too late after CommitTransaction completes. So we'll + // simulate an AbortTransaction message occurring during replication by calling + // the shard directly. + // + shard.underlyingActor().doAbortTransaction(transactionID, null); return preCommitFuture; } }; - MutableCompositeModification modification = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort = setupMockWriteTransaction("cohort1", dataStore, - TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME), - modification, preCommit); - - shard.tell(new ForwardedReadyTransaction(transactionID, CURRENT_VERSION, - cohort, modification, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.SERIALIZABLE_CLASS); + shard.tell(newBatchedModifications(transactionID, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true), getRef()); + expectMsgClass(duration, BatchedModificationsReply.class); shard.tell(new CanCommitTransaction(transactionID).toSerializable(), getRef()); CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); assertEquals("Can commit", true, canCommitReply.getCanCommit()); - Future commitFuture = Patterns.ask(shard, - new CommitTransaction(transactionID).toSerializable(), timeout); - - assertEquals("Abort complete", true, abortComplete.await(5, TimeUnit.SECONDS)); - - Await.result(commitFuture, duration); + shard.tell(new CommitTransaction(transactionID).toSerializable(), getRef()); + expectMsgClass(duration, CommitTransactionReply.SERIALIZABLE_CLASS); NormalizedNode node = readStore(shard, TestModel.TEST_PATH); + + // Since we're simulating an abort occurring during replication and before finish commit, + // the data should still get written to the in-memory store since we've gotten past + // canCommit and preCommit and persisted the data. assertNotNull(TestModel.TEST_QNAME.getLocalName() + " not found", node); shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); @@ -1009,7 +1180,7 @@ public class ShardTest extends AbstractActorTest { @Test public void testTransactionCommitTimeout() throws Throwable { - dataStoreContext = DatastoreContext.newBuilder().shardTransactionCommitTimeoutInSeconds(1).build(); + dataStoreContextBuilder.shardTransactionCommitTimeoutInSeconds(1); new ShardTestKit(getSystem()) {{ final TestActorRef shard = TestActorRef.create(getSystem(), @@ -1020,42 +1191,26 @@ public class ShardTest extends AbstractActorTest { final FiniteDuration duration = duration("5 seconds"); - InMemoryDOMDataStore dataStore = shard.underlyingActor().getDataStore(); - writeToStore(shard, TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); writeToStore(shard, TestModel.OUTER_LIST_PATH, ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build()); - // Create 1st Tx - will timeout + // Create and ready the 1st Tx - will timeout String transactionID1 = "tx1"; - MutableCompositeModification modification1 = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort1 = setupMockWriteTransaction("cohort1", dataStore, - YangInstanceIdentifier.builder(TestModel.OUTER_LIST_PATH) - .nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1).build(), - ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1), - modification1); + shard.tell(newBatchedModifications(transactionID1, YangInstanceIdentifier.builder( + TestModel.OUTER_LIST_PATH).nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1).build(), + ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1), true), getRef()); + expectMsgClass(duration, BatchedModificationsReply.class); - // Create 2nd Tx + // Create and ready the 2nd Tx - String transactionID2 = "tx3"; - MutableCompositeModification modification2 = new MutableCompositeModification(); + String transactionID2 = "tx2"; YangInstanceIdentifier listNodePath = YangInstanceIdentifier.builder(TestModel.OUTER_LIST_PATH) - .nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 2).build(); - DOMStoreThreePhaseCommitCohort cohort2 = setupMockWriteTransaction("cohort3", dataStore, - listNodePath, - ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 2), - modification2); - - // Ready the Tx's - - shard.tell(new ForwardedReadyTransaction(transactionID1, CURRENT_VERSION, - cohort1, modification1, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.SERIALIZABLE_CLASS); - - shard.tell(new ForwardedReadyTransaction(transactionID2, CURRENT_VERSION, - cohort2, modification2, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.SERIALIZABLE_CLASS); + .nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 2).build(); + shard.tell(newBatchedModifications(transactionID2, listNodePath, + ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 2), true), getRef()); + expectMsgClass(duration, BatchedModificationsReply.class); // canCommit 1st Tx. We don't send the commit so it should timeout. @@ -1081,7 +1236,7 @@ public class ShardTest extends AbstractActorTest { @Test public void testTransactionCommitQueueCapacityExceeded() throws Throwable { - dataStoreContext = DatastoreContext.newBuilder().shardTransactionCommitQueueCapacity(1).build(); + dataStoreContextBuilder.shardTransactionCommitQueueCapacity(1); new ShardTestKit(getSystem()) {{ final TestActorRef shard = TestActorRef.create(getSystem(), @@ -1092,38 +1247,23 @@ public class ShardTest extends AbstractActorTest { final FiniteDuration duration = duration("5 seconds"); - InMemoryDOMDataStore dataStore = shard.underlyingActor().getDataStore(); - String transactionID1 = "tx1"; - MutableCompositeModification modification1 = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort1 = setupMockWriteTransaction("cohort1", dataStore, - TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME), modification1); - String transactionID2 = "tx2"; - MutableCompositeModification modification2 = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort2 = setupMockWriteTransaction("cohort2", dataStore, - TestModel.OUTER_LIST_PATH, - ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build(), - modification2); - String transactionID3 = "tx3"; - MutableCompositeModification modification3 = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort3 = setupMockWriteTransaction("cohort3", dataStore, - TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME), modification3); - // Ready the Tx's + // Send a BatchedModifications to start transactions and ready them. - shard.tell(new ForwardedReadyTransaction(transactionID1, CURRENT_VERSION, - cohort1, modification1, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.SERIALIZABLE_CLASS); + shard.tell(newBatchedModifications(transactionID1, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true), getRef()); + expectMsgClass(duration, BatchedModificationsReply.class); - shard.tell(new ForwardedReadyTransaction(transactionID2, CURRENT_VERSION, - cohort2, modification2, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.SERIALIZABLE_CLASS); + shard.tell(newBatchedModifications(transactionID2,TestModel.OUTER_LIST_PATH, + ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build(), true), getRef()); + expectMsgClass(duration, BatchedModificationsReply.class); - shard.tell(new ForwardedReadyTransaction(transactionID3, CURRENT_VERSION, - cohort3, modification3, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.SERIALIZABLE_CLASS); + shard.tell(newBatchedModifications(transactionID3, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true), getRef()); + expectMsgClass(duration, BatchedModificationsReply.class); // canCommit 1st Tx. @@ -1168,30 +1308,37 @@ public class ShardTest extends AbstractActorTest { // Setup 2 simulated transactions with mock cohorts. The first one will be aborted. - String transactionID1 = "tx1"; - MutableCompositeModification modification1 = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort1 = mock(DOMStoreThreePhaseCommitCohort.class, "cohort1"); + final String transactionID1 = "tx1"; + final DOMStoreThreePhaseCommitCohort cohort1 = mock(DOMStoreThreePhaseCommitCohort.class, "cohort1"); doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort1).canCommit(); doReturn(Futures.immediateFuture(null)).when(cohort1).abort(); - String transactionID2 = "tx2"; - MutableCompositeModification modification2 = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort2 = mock(DOMStoreThreePhaseCommitCohort.class, "cohort2"); + final String transactionID2 = "tx2"; + final DOMStoreThreePhaseCommitCohort cohort2 = mock(DOMStoreThreePhaseCommitCohort.class, "cohort2"); doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort2).canCommit(); FiniteDuration duration = duration("5 seconds"); final Timeout timeout = new Timeout(duration); - // Simulate the ForwardedReadyTransaction messages that would be sent - // by the ShardTransaction. + ShardCommitCoordinator.CohortDecorator cohortDecorator = new ShardCommitCoordinator.CohortDecorator() { + @Override + public DOMStoreThreePhaseCommitCohort decorate(String transactionID, + DOMStoreThreePhaseCommitCohort actual) { + return transactionID1.equals(transactionID) ? cohort1 : cohort2; + } + }; + + shard.underlyingActor().getCommitCoordinator().setCohortDecorator(cohortDecorator); + + // Send BatchedModifications to start and ready each transaction. - shard.tell(new ForwardedReadyTransaction(transactionID1, CURRENT_VERSION, - cohort1, modification1, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.SERIALIZABLE_CLASS); + shard.tell(newBatchedModifications(transactionID1, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true), getRef()); + expectMsgClass(duration, BatchedModificationsReply.class); - shard.tell(new ForwardedReadyTransaction(transactionID2, CURRENT_VERSION, - cohort2, modification2, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.SERIALIZABLE_CLASS); + shard.tell(newBatchedModifications(transactionID2, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true), getRef()); + expectMsgClass(duration, BatchedModificationsReply.class); // Send the CanCommitTransaction message for the first Tx. @@ -1214,15 +1361,7 @@ public class ShardTest extends AbstractActorTest { // Wait for the 2nd Tx to complete the canCommit phase. - final CountDownLatch latch = new CountDownLatch(1); - canCommitFuture.onComplete(new OnComplete() { - @Override - public void onComplete(final Throwable t, final Object resp) { - latch.countDown(); - } - }, getSystem().dispatcher()); - - assertEquals("2nd CanCommit complete", true, latch.await(5, TimeUnit.SECONDS)); + Await.ready(canCommitFuture, duration); InOrder inOrder = inOrder(cohort1, cohort2); inOrder.verify(cohort1).canCommit(); @@ -1233,27 +1372,74 @@ public class ShardTest extends AbstractActorTest { } @Test - public void testCreateSnapshot() throws IOException, InterruptedException { - testCreateSnapshot(true, "testCreateSnapshot"); + public void testCreateSnapshot() throws Exception { + testCreateSnapshot(true, "testCreateSnapshot"); } @Test - public void testCreateSnapshotWithNonPersistentData() throws IOException, InterruptedException { + public void testCreateSnapshotWithNonPersistentData() throws Exception { testCreateSnapshot(false, "testCreateSnapshotWithNonPersistentData"); } @SuppressWarnings("serial") - public void testCreateSnapshot(final boolean persistent, final String shardActorName) throws IOException, InterruptedException { - final DatastoreContext dataStoreContext = DatastoreContext.newBuilder(). - shardJournalRecoveryLogBatchSize(3).shardSnapshotBatchCount(5000).persistent(persistent).build(); + public void testCreateSnapshot(final boolean persistent, final String shardActorName) throws Exception{ + + final AtomicReference savedSnapshot = new AtomicReference<>(); + class DelegatingPersistentDataProvider implements DataPersistenceProvider { + DataPersistenceProvider delegate; + + DelegatingPersistentDataProvider(DataPersistenceProvider delegate) { + this.delegate = delegate; + } + + @Override + public boolean isRecoveryApplicable() { + return delegate.isRecoveryApplicable(); + } + + @Override + public void persist(T o, Procedure procedure) { + delegate.persist(o, procedure); + } + + @Override + public void saveSnapshot(Object o) { + savedSnapshot.set(o); + delegate.saveSnapshot(o); + } + + @Override + public void deleteSnapshots(SnapshotSelectionCriteria criteria) { + delegate.deleteSnapshots(criteria); + } + + @Override + public void deleteMessages(long sequenceNumber) { + delegate.deleteMessages(sequenceNumber); + } + } + + dataStoreContextBuilder.persistent(persistent); new ShardTestKit(getSystem()) {{ final AtomicReference latch = new AtomicReference<>(new CountDownLatch(1)); Creator creator = new Creator() { @Override public Shard create() throws Exception { - return new Shard(shardID, Collections.emptyMap(), - dataStoreContext, SCHEMA_CONTEXT) { + return new Shard(shardID, Collections.emptyMap(), + newDatastoreContext(), SCHEMA_CONTEXT) { + + DelegatingPersistentDataProvider delegating; + + @Override + protected DataPersistenceProvider persistence() { + if(delegating == null) { + delegating = new DelegatingPersistentDataProvider(super.persistence()); + } + + return delegating; + } + @Override protected void commitSnapshot(final long sequenceNumber) { super.commitSnapshot(sequenceNumber); @@ -1268,16 +1454,40 @@ public class ShardTest extends AbstractActorTest { waitUntilLeader(shard); - shard.tell(new CaptureSnapshot(-1,-1,-1,-1), getRef()); + writeToStore(shard, TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); + + NormalizedNode expectedRoot = readStore(shard, YangInstanceIdentifier.builder().build()); + + CaptureSnapshot capture = new CaptureSnapshot(-1, -1, -1, -1, -1, -1); + shard.tell(capture, getRef()); assertEquals("Snapshot saved", true, latch.get().await(5, TimeUnit.SECONDS)); + assertTrue("Invalid saved snapshot " + savedSnapshot.get(), + savedSnapshot.get() instanceof Snapshot); + + verifySnapshot((Snapshot)savedSnapshot.get(), expectedRoot); + latch.set(new CountDownLatch(1)); - shard.tell(new CaptureSnapshot(-1,-1,-1,-1), getRef()); + savedSnapshot.set(null); + + shard.tell(capture, getRef()); assertEquals("Snapshot saved", true, latch.get().await(5, TimeUnit.SECONDS)); + assertTrue("Invalid saved snapshot " + savedSnapshot.get(), + savedSnapshot.get() instanceof Snapshot); + + verifySnapshot((Snapshot)savedSnapshot.get(), expectedRoot); + shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); + } + + private void verifySnapshot(Snapshot snapshot, NormalizedNode expectedRoot) { + + NormalizedNode actual = SerializationUtils.deserializeNormalizedNode(snapshot.getState()); + assertEquals("Root node", expectedRoot, actual); + }}; } @@ -1317,13 +1527,13 @@ public class ShardTest extends AbstractActorTest { final DatastoreContext persistentContext = DatastoreContext.newBuilder(). shardJournalRecoveryLogBatchSize(3).shardSnapshotBatchCount(5000).persistent(true).build(); - final Props persistentProps = Shard.props(shardID, Collections.emptyMap(), + final Props persistentProps = Shard.props(shardID, Collections.emptyMap(), persistentContext, SCHEMA_CONTEXT); final DatastoreContext nonPersistentContext = DatastoreContext.newBuilder(). shardJournalRecoveryLogBatchSize(3).shardSnapshotBatchCount(5000).persistent(false).build(); - final Props nonPersistentProps = Shard.props(shardID, Collections.emptyMap(), + final Props nonPersistentProps = Shard.props(shardID, Collections.emptyMap(), nonPersistentContext, SCHEMA_CONTEXT); new ShardTestKit(getSystem()) {{ @@ -1345,80 +1555,85 @@ public class ShardTest extends AbstractActorTest { } + @Test + public void testOnDatastoreContext() { + new ShardTestKit(getSystem()) {{ + dataStoreContextBuilder.persistent(true); - private NormalizedNode readStore(final InMemoryDOMDataStore store) throws ReadFailedException { - DOMStoreReadTransaction transaction = store.newReadOnlyTransaction(); - CheckedFuture>, ReadFailedException> read = - transaction.read(YangInstanceIdentifier.builder().build()); + TestActorRef shard = TestActorRef.create(getSystem(), newShardProps(), "testOnDatastoreContext"); - Optional> optional = read.checkedGet(); + assertEquals("isRecoveryApplicable", true, + shard.underlyingActor().getDataPersistenceProvider().isRecoveryApplicable()); - NormalizedNode normalizedNode = optional.get(); + waitUntilLeader(shard); - transaction.close(); + shard.tell(dataStoreContextBuilder.persistent(false).build(), ActorRef.noSender()); - return normalizedNode; - } + assertEquals("isRecoveryApplicable", false, + shard.underlyingActor().getDataPersistenceProvider().isRecoveryApplicable()); - private void commitTransaction(final DOMStoreWriteTransaction transaction) { - DOMStoreThreePhaseCommitCohort commitCohort = transaction.ready(); - ListenableFuture future = - commitCohort.preCommit(); - try { - future.get(); - future = commitCohort.commit(); - future.get(); - } catch (InterruptedException | ExecutionException e) { - } - } + shard.tell(dataStoreContextBuilder.persistent(true).build(), ActorRef.noSender()); - private AsyncDataChangeListener> noOpDataChangeListener() { - return new AsyncDataChangeListener>() { - @Override - public void onDataChanged( - final AsyncDataChangeEvent> change) { + assertEquals("isRecoveryApplicable", true, + shard.underlyingActor().getDataPersistenceProvider().isRecoveryApplicable()); - } - }; + shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); + }}; } - static NormalizedNode readStore(final TestActorRef shard, final YangInstanceIdentifier id) - throws ExecutionException, InterruptedException { - DOMStoreReadTransaction transaction = shard.underlyingActor().getDataStore().newReadOnlyTransaction(); + @Test + public void testRegisterRoleChangeListener() throws Exception { + new ShardTestKit(getSystem()) { + { + final TestActorRef shard = TestActorRef.create(getSystem(), + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testRegisterRoleChangeListener"); - CheckedFuture>, ReadFailedException> future = - transaction.read(id); + waitUntilLeader(shard); - Optional> optional = future.get(); - NormalizedNode node = optional.isPresent()? optional.get() : null; + TestActorRef listener = + TestActorRef.create(getSystem(), Props.create(MessageCollectorActor.class)); - transaction.close(); + shard.tell(new RegisterRoleChangeListener(), listener); - return node; + // TODO: MessageCollectorActor exists as a test util in both the akka-raft and distributed-datastore + // projects. Need to move it to commons as a regular utility and then we can get rid of this arbitrary + // sleep. + Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS); + + List allMatching = MessageCollectorActor.getAllMatching(listener, RegisterRoleChangeListenerReply.class); + + assertEquals(1, allMatching.size()); + } + }; } - private void writeToStore(final TestActorRef shard, final YangInstanceIdentifier id, final NormalizedNode node) - throws ExecutionException, InterruptedException { - DOMStoreWriteTransaction transaction = shard.underlyingActor().getDataStore().newWriteOnlyTransaction(); + @Test + public void testFollowerInitialSyncStatus() throws Exception { + final TestActorRef shard = TestActorRef.create(getSystem(), + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testFollowerInitialSyncStatus"); - transaction.write(id, node); + shard.underlyingActor().onReceiveCommand(new FollowerInitialSyncUpStatus(false, "member-1-shard-inventory-operational")); - DOMStoreThreePhaseCommitCohort commitCohort = transaction.ready(); - commitCohort.preCommit().get(); - commitCohort.commit().get(); - } + assertEquals(false, shard.underlyingActor().getShardMBean().getFollowerInitialSyncStatus()); - @SuppressWarnings("serial") - private static final class DelegatingShardCreator implements Creator { - private final Creator delegate; + shard.underlyingActor().onReceiveCommand(new FollowerInitialSyncUpStatus(true, "member-1-shard-inventory-operational")); - DelegatingShardCreator(final Creator delegate) { - this.delegate = delegate; - } + assertEquals(true, shard.underlyingActor().getShardMBean().getFollowerInitialSyncStatus()); - @Override - public Shard create() throws Exception { - return delegate.create(); + shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); + } + + private void commitTransaction(final DOMStoreWriteTransaction transaction) { + DOMStoreThreePhaseCommitCohort commitCohort = transaction.ready(); + ListenableFuture future = + commitCohort.preCommit(); + try { + future.get(); + future = commitCohort.commit(); + future.get(); + } catch (InterruptedException | ExecutionException e) { } } }