X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?p=controller.git;a=blobdiff_plain;f=opendaylight%2Fmd-sal%2Fsal-distributed-datastore%2Fsrc%2Ftest%2Fjava%2Forg%2Fopendaylight%2Fcontroller%2Fcluster%2Fdatastore%2FShardTest.java;h=8310e067c5fa002ed52b53e932b58e076aec4a76;hp=e3b82df1743e75c433cec193d54a2cbfbd696319;hb=5464f50be733df1bbbe31cf05665d542d3b7c5e7;hpb=c31a6fcf9fb070d4419ca4c32d8b531fdcb5030d diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/ShardTest.java b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/ShardTest.java index e3b82df174..8310e067c5 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/ShardTest.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/ShardTest.java @@ -1,18 +1,29 @@ +/* + * Copyright (c) 2014, 2015 Cisco Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ + package org.opendaylight.controller.cluster.datastore; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.doReturn; +import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; import static org.opendaylight.controller.cluster.datastore.DataStoreVersions.CURRENT_VERSION; + import akka.actor.ActorRef; import akka.actor.ActorSelection; -import akka.actor.PoisonPill; import akka.actor.Props; +import akka.actor.Status.Failure; import akka.dispatch.Dispatchers; import akka.dispatch.OnComplete; import akka.japi.Creator; @@ -20,20 +31,16 @@ import akka.pattern.Patterns; import akka.persistence.SaveSnapshotSuccess; import akka.testkit.TestActorRef; import akka.util.Timeout; -import com.google.common.base.Function; -import com.google.common.base.Optional; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.base.Stopwatch; +import com.google.common.base.Throwables; +import com.google.common.primitives.UnsignedLong; import com.google.common.util.concurrent.Uninterruptibles; import java.io.IOException; import java.util.Collections; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -41,7 +48,12 @@ import org.junit.Test; import org.mockito.InOrder; import org.opendaylight.controller.cluster.DataPersistenceProvider; import org.opendaylight.controller.cluster.DelegatingPersistentDataProvider; +import org.opendaylight.controller.cluster.access.concepts.LocalHistoryIdentifier; +import org.opendaylight.controller.cluster.access.concepts.MemberName; +import org.opendaylight.controller.cluster.access.concepts.TransactionIdentifier; +import org.opendaylight.controller.cluster.datastore.exceptions.NoShardLeaderException; import org.opendaylight.controller.cluster.datastore.identifiers.ShardIdentifier; +import org.opendaylight.controller.cluster.datastore.jmx.mbeans.shard.ShardStats; import org.opendaylight.controller.cluster.datastore.messages.AbortTransaction; import org.opendaylight.controller.cluster.datastore.messages.AbortTransactionReply; import org.opendaylight.controller.cluster.datastore.messages.BatchedModifications; @@ -51,93 +63,104 @@ import org.opendaylight.controller.cluster.datastore.messages.CanCommitTransacti import org.opendaylight.controller.cluster.datastore.messages.CommitTransaction; import org.opendaylight.controller.cluster.datastore.messages.CommitTransactionReply; import org.opendaylight.controller.cluster.datastore.messages.CreateTransaction; +import org.opendaylight.controller.cluster.datastore.messages.CreateTransactionReply; import org.opendaylight.controller.cluster.datastore.messages.ForwardedReadyTransaction; import org.opendaylight.controller.cluster.datastore.messages.PeerAddressResolved; import org.opendaylight.controller.cluster.datastore.messages.ReadData; import org.opendaylight.controller.cluster.datastore.messages.ReadDataReply; +import org.opendaylight.controller.cluster.datastore.messages.ReadyLocalTransaction; import org.opendaylight.controller.cluster.datastore.messages.ReadyTransactionReply; import org.opendaylight.controller.cluster.datastore.messages.RegisterChangeListener; import org.opendaylight.controller.cluster.datastore.messages.RegisterChangeListenerReply; +import org.opendaylight.controller.cluster.datastore.messages.RegisterDataTreeChangeListener; +import org.opendaylight.controller.cluster.datastore.messages.RegisterDataTreeChangeListenerReply; +import org.opendaylight.controller.cluster.datastore.messages.ShardLeaderStateChanged; import org.opendaylight.controller.cluster.datastore.messages.UpdateSchemaContext; -import org.opendaylight.controller.cluster.datastore.modification.DeleteModification; import org.opendaylight.controller.cluster.datastore.modification.MergeModification; -import org.opendaylight.controller.cluster.datastore.modification.Modification; -import org.opendaylight.controller.cluster.datastore.modification.ModificationPayload; import org.opendaylight.controller.cluster.datastore.modification.MutableCompositeModification; import org.opendaylight.controller.cluster.datastore.modification.WriteModification; -import org.opendaylight.controller.cluster.datastore.utils.MessageCollectorActor; +import org.opendaylight.controller.cluster.datastore.persisted.MetadataShardDataTreeSnapshot; +import org.opendaylight.controller.cluster.datastore.persisted.ShardDataTreeSnapshot; import org.opendaylight.controller.cluster.datastore.utils.MockDataChangeListener; -import org.opendaylight.controller.cluster.datastore.utils.SerializationUtils; +import org.opendaylight.controller.cluster.datastore.utils.MockDataTreeChangeListener; import org.opendaylight.controller.cluster.notifications.RegisterRoleChangeListener; import org.opendaylight.controller.cluster.notifications.RegisterRoleChangeListenerReply; import org.opendaylight.controller.cluster.raft.RaftActorContext; import org.opendaylight.controller.cluster.raft.ReplicatedLogEntry; import org.opendaylight.controller.cluster.raft.ReplicatedLogImplEntry; import org.opendaylight.controller.cluster.raft.Snapshot; -import org.opendaylight.controller.cluster.raft.base.messages.ApplyJournalEntries; import org.opendaylight.controller.cluster.raft.base.messages.ApplySnapshot; import org.opendaylight.controller.cluster.raft.base.messages.ApplyState; import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout; import org.opendaylight.controller.cluster.raft.base.messages.FollowerInitialSyncUpStatus; +import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow; import org.opendaylight.controller.cluster.raft.client.messages.FindLeader; import org.opendaylight.controller.cluster.raft.client.messages.FindLeaderReply; +import org.opendaylight.controller.cluster.raft.client.messages.GetOnDemandRaftState; +import org.opendaylight.controller.cluster.raft.client.messages.OnDemandRaftState; +import org.opendaylight.controller.cluster.raft.messages.RequestVote; +import org.opendaylight.controller.cluster.raft.messages.ServerRemoved; +import org.opendaylight.controller.cluster.raft.persisted.ApplyJournalEntries; +import org.opendaylight.controller.cluster.raft.policy.DisableElectionsRaftPolicy; +import org.opendaylight.controller.cluster.raft.protobuff.client.messages.Payload; import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal; -import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore; +import org.opendaylight.controller.cluster.raft.utils.MessageCollectorActor; import org.opendaylight.controller.md.cluster.datastore.model.SchemaContextHelper; import org.opendaylight.controller.md.cluster.datastore.model.TestModel; import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker; import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException; -import org.opendaylight.controller.md.sal.dom.store.impl.InMemoryDOMDataStore; -import org.opendaylight.controller.md.sal.dom.store.impl.InMemoryDOMDataStoreFactory; -import org.opendaylight.controller.protobuff.messages.cohort3pc.ThreePhaseCommitCohortMessages; -import org.opendaylight.controller.protobuff.messages.transaction.ShardTransactionMessages.CreateTransactionReply; -import org.opendaylight.controller.sal.core.spi.data.DOMStoreThreePhaseCommitCohort; -import org.opendaylight.controller.sal.core.spi.data.DOMStoreWriteTransaction; import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier; -import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.PathArgument; import org.opendaylight.yangtools.yang.data.api.schema.ContainerNode; -import org.opendaylight.yangtools.yang.data.api.schema.DataContainerChild; -import org.opendaylight.yangtools.yang.data.api.schema.MapEntryNode; +import org.opendaylight.yangtools.yang.data.api.schema.MapNode; import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode; +import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTree; +import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidate; +import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeModification; +import org.opendaylight.yangtools.yang.data.api.schema.tree.DataValidationFailedException; +import org.opendaylight.yangtools.yang.data.api.schema.tree.TipProducingDataTree; +import org.opendaylight.yangtools.yang.data.api.schema.tree.TreeType; import org.opendaylight.yangtools.yang.data.impl.schema.ImmutableNodes; -import org.opendaylight.yangtools.yang.model.api.SchemaContext; +import org.opendaylight.yangtools.yang.data.impl.schema.builder.impl.ImmutableContainerNodeBuilder; +import org.opendaylight.yangtools.yang.data.impl.schema.tree.InMemoryDataTreeFactory; import scala.concurrent.Await; import scala.concurrent.Future; import scala.concurrent.duration.FiniteDuration; public class ShardTest extends AbstractShardTest { + private static final String DUMMY_DATA = "Dummy data as snapshot sequence number is set to 0 in " + + "InMemorySnapshotStore and journal recovery seq number will start from 1"; @Test public void testRegisterChangeListener() throws Exception { - new ShardTestKit(getSystem()) {{ - TestActorRef shard = TestActorRef.create(getSystem(), - newShardProps(), "testRegisterChangeListener"); - - waitUntilLeader(shard); + new ShardTestKit(getSystem()) { + { + final TestActorRef shard = actorFactory.createTestActor( + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testRegisterChangeListener"); - shard.tell(new UpdateSchemaContext(SchemaContextHelper.full()), ActorRef.noSender()); + waitUntilLeader(shard); - MockDataChangeListener listener = new MockDataChangeListener(1); - ActorRef dclActor = getSystem().actorOf(DataChangeListener.props(listener), - "testRegisterChangeListener-DataChangeListener"); + shard.tell(new UpdateSchemaContext(SchemaContextHelper.full()), ActorRef.noSender()); - shard.tell(new RegisterChangeListener(TestModel.TEST_PATH, - dclActor, AsyncDataBroker.DataChangeScope.BASE), getRef()); + final MockDataChangeListener listener = new MockDataChangeListener(1); + final ActorRef dclActor = actorFactory.createActor(DataChangeListener.props(listener), + "testRegisterChangeListener-DataChangeListener"); - RegisterChangeListenerReply reply = expectMsgClass(duration("3 seconds"), - RegisterChangeListenerReply.class); - String replyPath = reply.getListenerRegistrationPath().toString(); - assertTrue("Incorrect reply path: " + replyPath, replyPath.matches( - "akka:\\/\\/test\\/user\\/testRegisterChangeListener\\/\\$.*")); + shard.tell(new RegisterChangeListener(TestModel.TEST_PATH, dclActor, + AsyncDataBroker.DataChangeScope.BASE, true), getRef()); - YangInstanceIdentifier path = TestModel.TEST_PATH; - writeToStore(shard, path, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); + final RegisterChangeListenerReply reply = expectMsgClass(duration("3 seconds"), + RegisterChangeListenerReply.class); + final String replyPath = reply.getListenerRegistrationPath().toString(); + assertTrue("Incorrect reply path: " + replyPath, + replyPath.matches("akka:\\/\\/test\\/user\\/testRegisterChangeListener\\/\\$.*")); - listener.waitForChangeEvents(path); + final YangInstanceIdentifier path = TestModel.TEST_PATH; + writeToStore(shard, path, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); - dclActor.tell(PoisonPill.getInstance(), ActorRef.noSender()); - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); - }}; + listener.waitForChangeEvents(path); + } + }; } @SuppressWarnings("serial") @@ -146,1261 +169,1793 @@ public class ShardTest extends AbstractShardTest { // This test tests the timing window in which a change listener is registered before the // shard becomes the leader. We verify that the listener is registered and notified of the // existing data when the shard becomes the leader. - new ShardTestKit(getSystem()) {{ - // For this test, we want to send the RegisterChangeListener message after the shard - // has recovered from persistence and before it becomes the leader. So we subclass - // Shard to override onReceiveCommand and, when the first ElectionTimeout is received, - // we know that the shard has been initialized to a follower and has started the - // election process. The following 2 CountDownLatches are used to coordinate the - // ElectionTimeout with the sending of the RegisterChangeListener message. - final CountDownLatch onFirstElectionTimeout = new CountDownLatch(1); - final CountDownLatch onChangeListenerRegistered = new CountDownLatch(1); - Creator creator = new Creator() { - boolean firstElectionTimeout = true; - - @Override - public Shard create() throws Exception { - // Use a non persistent provider because this test actually invokes persist on the journal - // this will cause all other messages to not be queued properly after that. - // The basic issue is that you cannot use TestActorRef with a persistent actor (at least when - // it does do a persist) - return new Shard(shardID, Collections.emptyMap(), - dataStoreContextBuilder.persistent(false).build(), SCHEMA_CONTEXT) { - @Override - public void onReceiveCommand(final Object message) throws Exception { - if(message instanceof ElectionTimeout && firstElectionTimeout) { - // Got the first ElectionTimeout. We don't forward it to the - // base Shard yet until we've sent the RegisterChangeListener - // message. So we signal the onFirstElectionTimeout latch to tell - // the main thread to send the RegisterChangeListener message and - // start a thread to wait on the onChangeListenerRegistered latch, - // which the main thread signals after it has sent the message. - // After the onChangeListenerRegistered is triggered, we send the - // original ElectionTimeout message to proceed with the election. - firstElectionTimeout = false; - final ActorRef self = getSelf(); - new Thread() { - @Override - public void run() { - Uninterruptibles.awaitUninterruptibly( - onChangeListenerRegistered, 5, TimeUnit.SECONDS); - self.tell(message, self); - } - }.start(); - - onFirstElectionTimeout.countDown(); - } else { - super.onReceiveCommand(message); - } - } - }; - } - }; - - MockDataChangeListener listener = new MockDataChangeListener(1); - ActorRef dclActor = getSystem().actorOf(DataChangeListener.props(listener), - "testRegisterChangeListenerWhenNotLeaderInitially-DataChangeListener"); - - TestActorRef shard = TestActorRef.create(getSystem(), - Props.create(new DelegatingShardCreator(creator)), - "testRegisterChangeListenerWhenNotLeaderInitially"); - - // Write initial data into the in-memory store. - YangInstanceIdentifier path = TestModel.TEST_PATH; - writeToStore(shard, path, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); - - // Wait until the shard receives the first ElectionTimeout message. - assertEquals("Got first ElectionTimeout", true, - onFirstElectionTimeout.await(5, TimeUnit.SECONDS)); + // For this test, we want to send the RegisterChangeListener message after the shard + // has recovered from persistence and before it becomes the leader. So we subclass + // Shard to override onReceiveCommand and, when the first ElectionTimeout is received, + // we know that the shard has been initialized to a follower and has started the + // election process. The following 2 CountDownLatches are used to coordinate the + // ElectionTimeout with the sending of the RegisterChangeListener message. + final CountDownLatch onFirstElectionTimeout = new CountDownLatch(1); + final CountDownLatch onChangeListenerRegistered = new CountDownLatch(1); + final Creator creator = new Creator() { + boolean firstElectionTimeout = true; - // Now send the RegisterChangeListener and wait for the reply. - shard.tell(new RegisterChangeListener(path, dclActor, - AsyncDataBroker.DataChangeScope.SUBTREE), getRef()); - - RegisterChangeListenerReply reply = expectMsgClass(duration("5 seconds"), - RegisterChangeListenerReply.class); - assertNotNull("getListenerRegistrationPath", reply.getListenerRegistrationPath()); + @Override + public Shard create() throws Exception { + // Use a non persistent provider because this test actually invokes persist on the journal + // this will cause all other messages to not be queued properly after that. + // The basic issue is that you cannot use TestActorRef with a persistent actor (at least when + // it does do a persist) + return new Shard(newShardBuilder()) { + @Override + public void handleCommand(final Object message) { + if (message instanceof ElectionTimeout && firstElectionTimeout) { + // Got the first ElectionTimeout. We don't forward it to the + // base Shard yet until we've sent the RegisterChangeListener + // message. So we signal the onFirstElectionTimeout latch to tell + // the main thread to send the RegisterChangeListener message and + // start a thread to wait on the onChangeListenerRegistered latch, + // which the main thread signals after it has sent the message. + // After the onChangeListenerRegistered is triggered, we send the + // original ElectionTimeout message to proceed with the election. + firstElectionTimeout = false; + final ActorRef self = getSelf(); + new Thread() { + @Override + public void run() { + Uninterruptibles.awaitUninterruptibly( + onChangeListenerRegistered, 5, TimeUnit.SECONDS); + self.tell(message, self); + } + }.start(); + + onFirstElectionTimeout.countDown(); + } else { + super.handleCommand(message); + } + } + }; + } + }; - // Sanity check - verify the shard is not the leader yet. - shard.tell(new FindLeader(), getRef()); - FindLeaderReply findLeadeReply = - expectMsgClass(duration("5 seconds"), FindLeaderReply.class); - assertNull("Expected the shard not to be the leader", findLeadeReply.getLeaderActor()); + setupInMemorySnapshotStore(); - // Signal the onChangeListenerRegistered latch to tell the thread above to proceed - // with the election process. - onChangeListenerRegistered.countDown(); + final MockDataChangeListener listener = new MockDataChangeListener(1); + final ActorRef dclActor = actorFactory.createActor(DataChangeListener.props(listener), + "testRegisterChangeListenerWhenNotLeaderInitially-DataChangeListener"); - // Wait for the shard to become the leader and notify our listener with the existing - // data in the store. - listener.waitForChangeEvents(path); + final TestActorRef shard = actorFactory.createTestActor( + Props.create(new DelegatingShardCreator(creator)).withDispatcher(Dispatchers.DefaultDispatcherId()), + "testRegisterChangeListenerWhenNotLeaderInitially"); - dclActor.tell(PoisonPill.getInstance(), ActorRef.noSender()); - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); - }}; + new ShardTestKit(getSystem()) { + { + final YangInstanceIdentifier path = TestModel.TEST_PATH; + + // Wait until the shard receives the first ElectionTimeout + // message. + assertEquals("Got first ElectionTimeout", true, onFirstElectionTimeout.await(5, TimeUnit.SECONDS)); + + // Now send the RegisterChangeListener and wait for the reply. + shard.tell(new RegisterChangeListener(path, dclActor, AsyncDataBroker.DataChangeScope.SUBTREE, false), + getRef()); + + final RegisterChangeListenerReply reply = expectMsgClass(duration("5 seconds"), + RegisterChangeListenerReply.class); + assertNotNull("getListenerRegistrationPath", reply.getListenerRegistrationPath()); + + // Sanity check - verify the shard is not the leader yet. + shard.tell(FindLeader.INSTANCE, getRef()); + final FindLeaderReply findLeadeReply = expectMsgClass(duration("5 seconds"), FindLeaderReply.class); + assertFalse("Expected the shard not to be the leader", findLeadeReply.getLeaderActor().isPresent()); + + // Signal the onChangeListenerRegistered latch to tell the + // thread above to proceed + // with the election process. + onChangeListenerRegistered.countDown(); + + // Wait for the shard to become the leader and notify our + // listener with the existing + // data in the store. + listener.waitForChangeEvents(path); + } + }; } @Test - public void testCreateTransaction(){ - new ShardTestKit(getSystem()) {{ - ActorRef shard = getSystem().actorOf(newShardProps(), "testCreateTransaction"); - - waitUntilLeader(shard); - - shard.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef()); - - shard.tell(new CreateTransaction("txn-1", - TransactionProxy.TransactionType.READ_ONLY.ordinal() ).toSerializable(), getRef()); - - CreateTransactionReply reply = expectMsgClass(duration("3 seconds"), - CreateTransactionReply.class); - - String path = reply.getTransactionActorPath().toString(); - assertTrue("Unexpected transaction path " + path, - path.contains("akka://test/user/testCreateTransaction/shard-txn-1")); + public void testRegisterDataTreeChangeListener() throws Exception { + new ShardTestKit(getSystem()) { + { + final TestActorRef shard = actorFactory.createTestActor( + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testRegisterDataTreeChangeListener"); - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); - }}; - } + waitUntilLeader(shard); - @Test - public void testCreateTransactionOnChain(){ - new ShardTestKit(getSystem()) {{ - final ActorRef shard = getSystem().actorOf(newShardProps(), "testCreateTransactionOnChain"); + shard.tell(new UpdateSchemaContext(SchemaContextHelper.full()), ActorRef.noSender()); - waitUntilLeader(shard); + final MockDataTreeChangeListener listener = new MockDataTreeChangeListener(1); + final ActorRef dclActor = actorFactory.createActor(DataTreeChangeListenerActor.props(listener), + "testRegisterDataTreeChangeListener-DataTreeChangeListener"); - shard.tell(new CreateTransaction("txn-1", - TransactionProxy.TransactionType.READ_ONLY.ordinal() , "foobar").toSerializable(), - getRef()); + shard.tell(new RegisterDataTreeChangeListener(TestModel.TEST_PATH, dclActor, false), getRef()); - CreateTransactionReply reply = expectMsgClass(duration("3 seconds"), - CreateTransactionReply.class); + final RegisterDataTreeChangeListenerReply reply = expectMsgClass(duration("3 seconds"), + RegisterDataTreeChangeListenerReply.class); + final String replyPath = reply.getListenerRegistrationPath().toString(); + assertTrue("Incorrect reply path: " + replyPath, + replyPath.matches("akka:\\/\\/test\\/user\\/testRegisterDataTreeChangeListener\\/\\$.*")); - String path = reply.getTransactionActorPath().toString(); - assertTrue("Unexpected transaction path " + path, - path.contains("akka://test/user/testCreateTransactionOnChain/shard-txn-1")); + final YangInstanceIdentifier path = TestModel.TEST_PATH; + writeToStore(shard, path, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); - }}; + listener.waitForChangeEvents(); + } + }; } @SuppressWarnings("serial") @Test - public void testPeerAddressResolved() throws Exception { - new ShardTestKit(getSystem()) {{ - final CountDownLatch recoveryComplete = new CountDownLatch(1); - class TestShard extends Shard { - TestShard() { - super(shardID, Collections.singletonMap(shardID.toString(), null), - newDatastoreContext(), SCHEMA_CONTEXT); - } - - Map getPeerAddresses() { - return getRaftActorContext().getPeerAddresses(); - } + public void testDataTreeChangeListenerNotifiedWhenNotTheLeaderOnRegistration() throws Exception { + final CountDownLatch onFirstElectionTimeout = new CountDownLatch(1); + final CountDownLatch onChangeListenerRegistered = new CountDownLatch(1); + final Creator creator = new Creator() { + boolean firstElectionTimeout = true; - @Override - protected void onRecoveryComplete() { - try { - super.onRecoveryComplete(); - } finally { - recoveryComplete.countDown(); + @Override + public Shard create() throws Exception { + return new Shard(newShardBuilder()) { + @Override + public void handleCommand(final Object message) { + if (message instanceof ElectionTimeout && firstElectionTimeout) { + firstElectionTimeout = false; + final ActorRef self = getSelf(); + new Thread() { + @Override + public void run() { + Uninterruptibles.awaitUninterruptibly( + onChangeListenerRegistered, 5, TimeUnit.SECONDS); + self.tell(message, self); + } + }.start(); + + onFirstElectionTimeout.countDown(); + } else { + super.handleCommand(message); + } } - } + }; } + }; - final TestActorRef shard = TestActorRef.create(getSystem(), - Props.create(new DelegatingShardCreator(new Creator() { - @Override - public TestShard create() throws Exception { - return new TestShard(); - } - })), "testPeerAddressResolved"); + setupInMemorySnapshotStore(); - //waitUntilLeader(shard); - assertEquals("Recovery complete", true, - Uninterruptibles.awaitUninterruptibly(recoveryComplete, 5, TimeUnit.SECONDS)); + final MockDataTreeChangeListener listener = new MockDataTreeChangeListener(1); + final ActorRef dclActor = actorFactory.createActor(DataTreeChangeListenerActor.props(listener), + "testDataTreeChangeListenerNotifiedWhenNotTheLeaderOnRegistration-DataChangeListener"); - String address = "akka://foobar"; - shard.underlyingActor().onReceiveCommand(new PeerAddressResolved(shardID.toString(), address)); + final TestActorRef shard = actorFactory.createTestActor( + Props.create(new DelegatingShardCreator(creator)).withDispatcher(Dispatchers.DefaultDispatcherId()), + "testDataTreeChangeListenerNotifiedWhenNotTheLeaderOnRegistration"); - assertEquals("getPeerAddresses", address, - ((TestShard)shard.underlyingActor()).getPeerAddresses().get(shardID.toString())); + final YangInstanceIdentifier path = TestModel.TEST_PATH; - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); - }}; - } + new ShardTestKit(getSystem()) { + { + assertEquals("Got first ElectionTimeout", true, onFirstElectionTimeout.await(5, TimeUnit.SECONDS)); - @Test - public void testApplySnapshot() throws Exception { - TestActorRef shard = TestActorRef.create(getSystem(), newShardProps(), - "testApplySnapshot"); + shard.tell(new RegisterDataTreeChangeListener(path, dclActor, false), getRef()); + final RegisterDataTreeChangeListenerReply reply = expectMsgClass(duration("5 seconds"), + RegisterDataTreeChangeListenerReply.class); + assertNotNull("getListenerRegistratioznPath", reply.getListenerRegistrationPath()); - InMemoryDOMDataStore store = new InMemoryDOMDataStore("OPER", MoreExecutors.sameThreadExecutor()); - store.onGlobalContextUpdated(SCHEMA_CONTEXT); + shard.tell(FindLeader.INSTANCE, getRef()); + final FindLeaderReply findLeadeReply = expectMsgClass(duration("5 seconds"), FindLeaderReply.class); + assertFalse("Expected the shard not to be the leader", findLeadeReply.getLeaderActor().isPresent()); - writeToStore(store, TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); + onChangeListenerRegistered.countDown(); + + // TODO: investigate why we do not receive data chage events + listener.waitForChangeEvents(); + } + }; + } - YangInstanceIdentifier root = YangInstanceIdentifier.builder().build(); - NormalizedNode expected = readStore(store, root); + @Test + public void testCreateTransaction() { + new ShardTestKit(getSystem()) { + { + final ActorRef shard = actorFactory.createActor(newShardProps(), "testCreateTransaction"); - ApplySnapshot applySnapshot = new ApplySnapshot(Snapshot.create( - SerializationUtils.serializeNormalizedNode(expected), - Collections.emptyList(), 1, 2, 3, 4)); + waitUntilLeader(shard); - shard.underlyingActor().onReceiveCommand(applySnapshot); + shard.tell(new UpdateSchemaContext(TestModel.createTestContext()), getRef()); - NormalizedNode actual = readStore(shard, root); + shard.tell(new CreateTransaction(nextTransactionId(), TransactionType.READ_ONLY.ordinal(), + DataStoreVersions.CURRENT_VERSION).toSerializable(), getRef()); - assertEquals("Root node", expected, actual); + final CreateTransactionReply reply = expectMsgClass(duration("3 seconds"), + CreateTransactionReply.class); - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); + final String path = reply.getTransactionPath().toString(); + assertTrue("Unexpected transaction path " + path, path + .startsWith("akka://test/user/testCreateTransaction/shard-member-1:ShardTransactionTest@0:")); + } + }; } @Test - public void testApplyState() throws Exception { - - TestActorRef shard = TestActorRef.create(getSystem(), newShardProps(), "testApplyState"); - - NormalizedNode node = ImmutableNodes.containerNode(TestModel.TEST_QNAME); + public void testCreateTransactionOnChain() { + new ShardTestKit(getSystem()) { + { + final ActorRef shard = actorFactory.createActor(newShardProps(), "testCreateTransactionOnChain"); - ApplyState applyState = new ApplyState(null, "test", new ReplicatedLogImplEntry(1, 2, - newModificationPayload(new WriteModification(TestModel.TEST_PATH, node)))); + waitUntilLeader(shard); - shard.underlyingActor().onReceiveCommand(applyState); + shard.tell(new CreateTransaction(nextTransactionId(), TransactionType.READ_ONLY.ordinal(), + DataStoreVersions.CURRENT_VERSION).toSerializable(), getRef()); - NormalizedNode actual = readStore(shard, TestModel.TEST_PATH); - assertEquals("Applied state", node, actual); + final CreateTransactionReply reply = expectMsgClass(duration("3 seconds"), + CreateTransactionReply.class); - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); + final String path = reply.getTransactionPath().toString(); + assertTrue("Unexpected transaction path " + path, path.startsWith( + "akka://test/user/testCreateTransactionOnChain/shard-member-1:ShardTransactionTest@0:")); + } + }; } @Test - public void testRecovery() throws Exception { - - // Set up the InMemorySnapshotStore. + public void testPeerAddressResolved() throws Exception { + new ShardTestKit(getSystem()) { + { + final ShardIdentifier peerID = ShardIdentifier.create("inventory", MemberName.forName("member-2"), + "config"); + final TestActorRef shard = actorFactory.createTestActor(newShardBuilder() + .peerAddresses(Collections.singletonMap(peerID.toString(), null)) + .props().withDispatcher(Dispatchers.DefaultDispatcherId()), "testPeerAddressResolved"); + + final String address = "akka://foobar"; + shard.tell(new PeerAddressResolved(peerID.toString(), address), ActorRef.noSender()); + + shard.tell(GetOnDemandRaftState.INSTANCE, getRef()); + final OnDemandRaftState state = expectMsgClass(OnDemandRaftState.class); + assertEquals("getPeerAddress", address, state.getPeerAddresses().get(peerID.toString())); + } + }; + } - InMemoryDOMDataStore testStore = InMemoryDOMDataStoreFactory.create("Test", null, null); - testStore.onGlobalContextUpdated(SCHEMA_CONTEXT); + @Test + public void testApplySnapshot() throws Exception { - writeToStore(testStore, TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); + final TestActorRef shard = actorFactory.createTestActor(newShardProps() + .withDispatcher(Dispatchers.DefaultDispatcherId()), "testApplySnapshot"); - NormalizedNode root = readStore(testStore, YangInstanceIdentifier.builder().build()); + ShardTestKit.waitUntilLeader(shard); - InMemorySnapshotStore.addSnapshot(shardID.toString(), Snapshot.create( - SerializationUtils.serializeNormalizedNode(root), - Collections.emptyList(), 0, 1, -1, -1)); + final DataTree store = InMemoryDataTreeFactory.getInstance().create(TreeType.OPERATIONAL); + store.setSchemaContext(SCHEMA_CONTEXT); - // Set up the InMemoryJournal. + final ContainerNode container = ImmutableContainerNodeBuilder.create().withNodeIdentifier( + new YangInstanceIdentifier.NodeIdentifier(TestModel.TEST_QNAME)) + .withChild(ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).addChild( + ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1)).build()).build(); - InMemoryJournal.addEntry(shardID.toString(), 0, new ReplicatedLogImplEntry(0, 1, newModificationPayload( - new WriteModification(TestModel.OUTER_LIST_PATH, - ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build())))); + writeToStore(store, TestModel.TEST_PATH, container); - int nListEntries = 16; - Set listEntryKeys = new HashSet<>(); + final YangInstanceIdentifier root = YangInstanceIdentifier.EMPTY; + final NormalizedNode expected = readStore(store, root); - // Add some ModificationPayload entries - for(int i = 1; i <= nListEntries; i++) { - listEntryKeys.add(Integer.valueOf(i)); - YangInstanceIdentifier path = YangInstanceIdentifier.builder(TestModel.OUTER_LIST_PATH) - .nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, i).build(); - Modification mod = new MergeModification(path, - ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, i)); - InMemoryJournal.addEntry(shardID.toString(), i, new ReplicatedLogImplEntry(i, 1, - newModificationPayload(mod))); - } + final Snapshot snapshot = Snapshot.create(new MetadataShardDataTreeSnapshot(expected).serialize(), + Collections.emptyList(), 1, 2, 3, 4); - InMemoryJournal.addEntry(shardID.toString(), nListEntries + 1, - new ApplyJournalEntries(nListEntries)); + shard.tell(new ApplySnapshot(snapshot), ActorRef.noSender()); - testRecovery(listEntryKeys); - } + final Stopwatch sw = Stopwatch.createStarted(); + while (sw.elapsed(TimeUnit.SECONDS) <= 5) { + Uninterruptibles.sleepUninterruptibly(75, TimeUnit.MILLISECONDS); - private ModificationPayload newModificationPayload(final Modification... mods) throws IOException { - MutableCompositeModification compMod = new MutableCompositeModification(); - for(Modification mod: mods) { - compMod.addModification(mod); + try { + assertEquals("Root node", expected, readStore(shard, root)); + return; + } catch (final AssertionError e) { + // try again + } } - return new ModificationPayload(compMod); + fail("Snapshot was not applied"); } - @SuppressWarnings({ "unchecked" }) @Test - public void testConcurrentThreePhaseCommits() throws Throwable { - new ShardTestKit(getSystem()) {{ - final TestActorRef shard = TestActorRef.create(getSystem(), - newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testConcurrentThreePhaseCommits"); + public void testApplyState() throws Exception { + final TestActorRef shard = actorFactory.createTestActor( + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), "testApplyState"); - waitUntilLeader(shard); + ShardTestKit.waitUntilLeader(shard); - // Setup 3 simulated transactions with mock cohorts backed by real cohorts. + final DataTree store = InMemoryDataTreeFactory.getInstance().create(TreeType.OPERATIONAL); + store.setSchemaContext(SCHEMA_CONTEXT); + writeToStore(store, TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); + + final NormalizedNode root = readStore(store, YangInstanceIdentifier.EMPTY); + final Snapshot snapshot = Snapshot.create(new MetadataShardDataTreeSnapshot(root).serialize(), + Collections.emptyList(), 1, 2, 3, 4); - InMemoryDOMDataStore dataStore = shard.underlyingActor().getDataStore(); + shard.tell(new ApplySnapshot(snapshot), ActorRef.noSender()); - String transactionID1 = "tx1"; - MutableCompositeModification modification1 = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort1 = setupMockWriteTransaction("cohort1", dataStore, - TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME), modification1); + final DataTreeModification writeMod = store.takeSnapshot().newModification(); + final ContainerNode node = ImmutableNodes.containerNode(TestModel.TEST_QNAME); + writeMod.write(TestModel.TEST_PATH, node); + writeMod.ready(); - String transactionID2 = "tx2"; - MutableCompositeModification modification2 = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort2 = setupMockWriteTransaction("cohort2", dataStore, - TestModel.OUTER_LIST_PATH, - ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build(), - modification2); + final TransactionIdentifier tx = nextTransactionId(); + final ApplyState applyState = new ApplyState(null, tx, + new ReplicatedLogImplEntry(1, 2, payloadForModification(store, writeMod, tx))); - String transactionID3 = "tx3"; - MutableCompositeModification modification3 = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort3 = setupMockWriteTransaction("cohort3", dataStore, - YangInstanceIdentifier.builder(TestModel.OUTER_LIST_PATH) - .nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1).build(), - ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1), - modification3); + shard.tell(applyState, shard); - long timeoutSec = 5; - final FiniteDuration duration = FiniteDuration.create(timeoutSec, TimeUnit.SECONDS); - final Timeout timeout = new Timeout(duration); + final Stopwatch sw = Stopwatch.createStarted(); + while (sw.elapsed(TimeUnit.SECONDS) <= 5) { + Uninterruptibles.sleepUninterruptibly(75, TimeUnit.MILLISECONDS); + + final NormalizedNode actual = readStore(shard, TestModel.TEST_PATH); + if (actual != null) { + assertEquals("Applied state", node, actual); + return; + } + } - // Simulate the ForwardedReadyTransaction message for the first Tx that would be sent - // by the ShardTransaction. + fail("State was not applied"); + } - shard.tell(new ForwardedReadyTransaction(transactionID1, CURRENT_VERSION, - cohort1, modification1, true), getRef()); - ReadyTransactionReply readyReply = ReadyTransactionReply.fromSerializable( - expectMsgClass(duration, ReadyTransactionReply.class)); - assertEquals("Cohort path", shard.path().toString(), readyReply.getCohortPath()); + @Test + public void testDataTreeCandidateRecovery() throws Exception { + // Set up the InMemorySnapshotStore. + final DataTree source = setupInMemorySnapshotStore(); - // Send the CanCommitTransaction message for the first Tx. + final DataTreeModification writeMod = source.takeSnapshot().newModification(); + writeMod.write(TestModel.OUTER_LIST_PATH, ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build()); + writeMod.ready(); + InMemoryJournal.addEntry(shardID.toString(), 0, DUMMY_DATA); - shard.tell(new CanCommitTransaction(transactionID1).toSerializable(), getRef()); - CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( - expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); - assertEquals("Can commit", true, canCommitReply.getCanCommit()); + // Set up the InMemoryJournal. + InMemoryJournal.addEntry(shardID.toString(), 1, new ReplicatedLogImplEntry(0, 1, + payloadForModification(source, writeMod, nextTransactionId()))); - // Send the ForwardedReadyTransaction for the next 2 Tx's. + final int nListEntries = 16; + final Set listEntryKeys = new HashSet<>(); - shard.tell(new ForwardedReadyTransaction(transactionID2, CURRENT_VERSION, - cohort2, modification2, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.class); + // Add some ModificationPayload entries + for (int i = 1; i <= nListEntries; i++) { + listEntryKeys.add(Integer.valueOf(i)); - shard.tell(new ForwardedReadyTransaction(transactionID3, CURRENT_VERSION, - cohort3, modification3, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.class); + final YangInstanceIdentifier path = YangInstanceIdentifier.builder(TestModel.OUTER_LIST_PATH) + .nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, i).build(); - // Send the CanCommitTransaction message for the next 2 Tx's. These should get queued and - // processed after the first Tx completes. + final DataTreeModification mod = source.takeSnapshot().newModification(); + mod.merge(path, ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, i)); + mod.ready(); - Future canCommitFuture1 = Patterns.ask(shard, - new CanCommitTransaction(transactionID2).toSerializable(), timeout); + InMemoryJournal.addEntry(shardID.toString(), i + 1, new ReplicatedLogImplEntry(i, 1, + payloadForModification(source, mod, nextTransactionId()))); + } - Future canCommitFuture2 = Patterns.ask(shard, - new CanCommitTransaction(transactionID3).toSerializable(), timeout); + InMemoryJournal.addEntry(shardID.toString(), nListEntries + 2, + new ApplyJournalEntries(nListEntries)); - // Send the CommitTransaction message for the first Tx. After it completes, it should - // trigger the 2nd Tx to proceed which should in turn then trigger the 3rd. + testRecovery(listEntryKeys); + } - shard.tell(new CommitTransaction(transactionID1).toSerializable(), getRef()); - expectMsgClass(duration, CommitTransactionReply.SERIALIZABLE_CLASS); + @Test + @SuppressWarnings("checkstyle:IllegalCatch") + public void testConcurrentThreePhaseCommits() throws Exception { + final AtomicReference caughtEx = new AtomicReference<>(); + final CountDownLatch commitLatch = new CountDownLatch(2); - // Wait for the next 2 Tx's to complete. + final long timeoutSec = 5; + final FiniteDuration duration = FiniteDuration.create(timeoutSec, TimeUnit.SECONDS); + final Timeout timeout = new Timeout(duration); - final AtomicReference caughtEx = new AtomicReference<>(); - final CountDownLatch commitLatch = new CountDownLatch(2); + final TestActorRef shard = actorFactory.createTestActor( + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testConcurrentThreePhaseCommits"); - class OnFutureComplete extends OnComplete { - private final Class expRespType; + class OnFutureComplete extends OnComplete { + private final Class expRespType; - OnFutureComplete(final Class expRespType) { - this.expRespType = expRespType; - } + OnFutureComplete(final Class expRespType) { + this.expRespType = expRespType; + } - @Override - public void onComplete(final Throwable error, final Object resp) { - if(error != null) { - caughtEx.set(new AssertionError(getClass().getSimpleName() + " failure", error)); - } else { - try { - assertEquals("Commit response type", expRespType, resp.getClass()); - onSuccess(resp); - } catch (Exception e) { - caughtEx.set(e); - } + @Override + public void onComplete(final Throwable error, final Object resp) { + if (error != null) { + caughtEx.set(new AssertionError(getClass().getSimpleName() + " failure", error)); + } else { + try { + assertEquals("Commit response type", expRespType, resp.getClass()); + onSuccess(resp); + } catch (final Exception e) { + caughtEx.set(e); } } + } - void onSuccess(final Object resp) throws Exception { - } + void onSuccess(final Object resp) throws Exception { } + } - class OnCommitFutureComplete extends OnFutureComplete { - OnCommitFutureComplete() { - super(CommitTransactionReply.SERIALIZABLE_CLASS); - } + class OnCommitFutureComplete extends OnFutureComplete { + OnCommitFutureComplete() { + super(CommitTransactionReply.class); + } - @Override - public void onComplete(final Throwable error, final Object resp) { - super.onComplete(error, resp); - commitLatch.countDown(); - } + @Override + public void onComplete(final Throwable error, final Object resp) { + super.onComplete(error, resp); + commitLatch.countDown(); } + } - class OnCanCommitFutureComplete extends OnFutureComplete { - private final String transactionID; + class OnCanCommitFutureComplete extends OnFutureComplete { + private final TransactionIdentifier transactionID; - OnCanCommitFutureComplete(final String transactionID) { - super(CanCommitTransactionReply.SERIALIZABLE_CLASS); - this.transactionID = transactionID; - } + OnCanCommitFutureComplete(final TransactionIdentifier transactionID) { + super(CanCommitTransactionReply.class); + this.transactionID = transactionID; + } - @Override - void onSuccess(final Object resp) throws Exception { - CanCommitTransactionReply canCommitReply = - CanCommitTransactionReply.fromSerializable(resp); - assertEquals("Can commit", true, canCommitReply.getCanCommit()); + @Override + void onSuccess(final Object resp) throws Exception { + final CanCommitTransactionReply canCommitReply = + CanCommitTransactionReply.fromSerializable(resp); + assertEquals("Can commit", true, canCommitReply.getCanCommit()); - Future commitFuture = Patterns.ask(shard, - new CommitTransaction(transactionID).toSerializable(), timeout); - commitFuture.onComplete(new OnCommitFutureComplete(), getSystem().dispatcher()); - } + final Future commitFuture = Patterns.ask(shard, + new CommitTransaction(transactionID, CURRENT_VERSION).toSerializable(), timeout); + commitFuture.onComplete(new OnCommitFutureComplete(), getSystem().dispatcher()); } + } - canCommitFuture1.onComplete(new OnCanCommitFutureComplete(transactionID2), - getSystem().dispatcher()); + new ShardTestKit(getSystem()) { + { + waitUntilLeader(shard); - canCommitFuture2.onComplete(new OnCanCommitFutureComplete(transactionID3), - getSystem().dispatcher()); + final TransactionIdentifier transactionID1 = nextTransactionId(); + final TransactionIdentifier transactionID2 = nextTransactionId(); + final TransactionIdentifier transactionID3 = nextTransactionId(); - boolean done = commitLatch.await(timeoutSec, TimeUnit.SECONDS); + final Map cohortMap = setupCohortDecorator( + shard.underlyingActor(), transactionID1, transactionID2, transactionID3); + final CapturingShardDataTreeCohort cohort1 = cohortMap.get(transactionID1); + final CapturingShardDataTreeCohort cohort2 = cohortMap.get(transactionID2); + final CapturingShardDataTreeCohort cohort3 = cohortMap.get(transactionID3); - if(caughtEx.get() != null) { - throw caughtEx.get(); - } + shard.tell(prepareBatchedModifications(transactionID1, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), false), getRef()); + final ReadyTransactionReply readyReply = ReadyTransactionReply + .fromSerializable(expectMsgClass(duration, ReadyTransactionReply.class)); + assertEquals("Cohort path", shard.path().toString(), readyReply.getCohortPath()); - assertEquals("Commits complete", true, done); - - InOrder inOrder = inOrder(cohort1, cohort2, cohort3); - inOrder.verify(cohort1).canCommit(); - inOrder.verify(cohort1).preCommit(); - inOrder.verify(cohort1).commit(); - inOrder.verify(cohort2).canCommit(); - inOrder.verify(cohort2).preCommit(); - inOrder.verify(cohort2).commit(); - inOrder.verify(cohort3).canCommit(); - inOrder.verify(cohort3).preCommit(); - inOrder.verify(cohort3).commit(); - - // Verify data in the data store. - - NormalizedNode outerList = readStore(shard, TestModel.OUTER_LIST_PATH); - assertNotNull(TestModel.OUTER_LIST_QNAME.getLocalName() + " not found", outerList); - assertTrue(TestModel.OUTER_LIST_QNAME.getLocalName() + " value is not Iterable", - outerList.getValue() instanceof Iterable); - Object entry = ((Iterable)outerList.getValue()).iterator().next(); - assertTrue(TestModel.OUTER_LIST_QNAME.getLocalName() + " entry is not MapEntryNode", - entry instanceof MapEntryNode); - MapEntryNode mapEntry = (MapEntryNode)entry; - Optional> idLeaf = - mapEntry.getChild(new YangInstanceIdentifier.NodeIdentifier(TestModel.ID_QNAME)); - assertTrue("Missing leaf " + TestModel.ID_QNAME.getLocalName(), idLeaf.isPresent()); - assertEquals(TestModel.ID_QNAME.getLocalName() + " value", 1, idLeaf.get().getValue()); - - verifyLastApplied(shard, 2); - - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); - }}; - } + // Send the CanCommitTransaction message for the first Tx. - private BatchedModifications newBatchedModifications(String transactionID, YangInstanceIdentifier path, - NormalizedNode data, boolean ready) { - return newBatchedModifications(transactionID, null, path, data, ready); - } + shard.tell(new CanCommitTransaction(transactionID1, CURRENT_VERSION).toSerializable(), getRef()); + final CanCommitTransactionReply canCommitReply = CanCommitTransactionReply + .fromSerializable(expectMsgClass(duration, CanCommitTransactionReply.class)); + assertEquals("Can commit", true, canCommitReply.getCanCommit()); - private BatchedModifications newBatchedModifications(String transactionID, String transactionChainID, - YangInstanceIdentifier path, NormalizedNode data, boolean ready) { - BatchedModifications batched = new BatchedModifications(transactionID, CURRENT_VERSION, transactionChainID); - batched.addModification(new WriteModification(path, data)); - batched.setReady(ready); - return batched; - } + // Ready 2 more Tx's. - @SuppressWarnings("unchecked") - @Test - public void testMultipleBatchedModifications() throws Throwable { - new ShardTestKit(getSystem()) {{ - final TestActorRef shard = TestActorRef.create(getSystem(), - newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testMultipleBatchedModifications"); - - waitUntilLeader(shard); - - final String transactionID = "tx"; - FiniteDuration duration = duration("5 seconds"); - - final AtomicReference mockCohort = new AtomicReference<>(); - ShardCommitCoordinator.CohortDecorator cohortDecorator = new ShardCommitCoordinator.CohortDecorator() { - @Override - public DOMStoreThreePhaseCommitCohort decorate(String txID, DOMStoreThreePhaseCommitCohort actual) { - if(mockCohort.get() == null) { - mockCohort.set(createDelegatingMockCohort("cohort", actual)); - } + shard.tell(prepareBatchedModifications(transactionID2, TestModel.OUTER_LIST_PATH, + ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build(), false), getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); - return mockCohort.get(); - } - }; + shard.tell( + prepareBatchedModifications(transactionID3, + YangInstanceIdentifier.builder(TestModel.OUTER_LIST_PATH) + .nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1).build(), + ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1), false), + getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); + + // Send the CanCommitTransaction message for the next 2 Tx's. + // These should get queued and + // processed after the first Tx completes. - shard.underlyingActor().getCommitCoordinator().setCohortDecorator(cohortDecorator); + final Future canCommitFuture1 = Patterns.ask(shard, + new CanCommitTransaction(transactionID2, CURRENT_VERSION).toSerializable(), timeout); - // Send a BatchedModifications to start a transaction. + final Future canCommitFuture2 = Patterns.ask(shard, + new CanCommitTransaction(transactionID3, CURRENT_VERSION).toSerializable(), timeout); - shard.tell(newBatchedModifications(transactionID, TestModel.TEST_PATH, - ImmutableNodes.containerNode(TestModel.TEST_QNAME), false), getRef()); - expectMsgClass(duration, BatchedModificationsReply.class); + // Send the CommitTransaction message for the first Tx. After it + // completes, it should + // trigger the 2nd Tx to proceed which should in turn then + // trigger the 3rd. - // Send a couple more BatchedModifications. + shard.tell(new CommitTransaction(transactionID1, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, CommitTransactionReply.class); - shard.tell(newBatchedModifications(transactionID, TestModel.OUTER_LIST_PATH, - ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build(), false), getRef()); - expectMsgClass(duration, BatchedModificationsReply.class); + // Wait for the next 2 Tx's to complete. - shard.tell(newBatchedModifications(transactionID, YangInstanceIdentifier.builder( - TestModel.OUTER_LIST_PATH).nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1).build(), - ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1), true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.class); + canCommitFuture1.onComplete(new OnCanCommitFutureComplete(transactionID2), getSystem().dispatcher()); - // Send the CanCommitTransaction message. + canCommitFuture2.onComplete(new OnCanCommitFutureComplete(transactionID3), getSystem().dispatcher()); - shard.tell(new CanCommitTransaction(transactionID).toSerializable(), getRef()); - CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( - expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); - assertEquals("Can commit", true, canCommitReply.getCanCommit()); + final boolean done = commitLatch.await(timeoutSec, TimeUnit.SECONDS); - // Send the CanCommitTransaction message. + if (caughtEx.get() != null) { + Throwables.propagateIfInstanceOf(caughtEx.get(), Exception.class); + Throwables.propagate(caughtEx.get()); + } - shard.tell(new CommitTransaction(transactionID).toSerializable(), getRef()); - expectMsgClass(duration, CommitTransactionReply.SERIALIZABLE_CLASS); + assertEquals("Commits complete", true, done); - InOrder inOrder = inOrder(mockCohort.get()); - inOrder.verify(mockCohort.get()).canCommit(); - inOrder.verify(mockCohort.get()).preCommit(); - inOrder.verify(mockCohort.get()).commit(); + final InOrder inOrder = inOrder(cohort1.getCanCommit(), cohort1.getPreCommit(), cohort1.getCommit(), + cohort2.getCanCommit(), cohort2.getPreCommit(), cohort2.getCommit(), cohort3.getCanCommit(), + cohort3.getPreCommit(), cohort3.getCommit()); + inOrder.verify(cohort1.getCanCommit()).onSuccess(any(Void.class)); + inOrder.verify(cohort1.getPreCommit()).onSuccess(any(DataTreeCandidate.class)); + inOrder.verify(cohort1.getCommit()).onSuccess(any(UnsignedLong.class)); + inOrder.verify(cohort2.getCanCommit()).onSuccess(any(Void.class)); + inOrder.verify(cohort2.getPreCommit()).onSuccess(any(DataTreeCandidate.class)); + inOrder.verify(cohort2.getCommit()).onSuccess(any(UnsignedLong.class)); + inOrder.verify(cohort3.getCanCommit()).onSuccess(any(Void.class)); + inOrder.verify(cohort3.getPreCommit()).onSuccess(any(DataTreeCandidate.class)); + inOrder.verify(cohort3.getCommit()).onSuccess(any(UnsignedLong.class)); - // Verify data in the data store. + // Verify data in the data store. - NormalizedNode outerList = readStore(shard, TestModel.OUTER_LIST_PATH); - assertNotNull(TestModel.OUTER_LIST_QNAME.getLocalName() + " not found", outerList); - assertTrue(TestModel.OUTER_LIST_QNAME.getLocalName() + " value is not Iterable", - outerList.getValue() instanceof Iterable); - Object entry = ((Iterable)outerList.getValue()).iterator().next(); - assertTrue(TestModel.OUTER_LIST_QNAME.getLocalName() + " entry is not MapEntryNode", - entry instanceof MapEntryNode); - MapEntryNode mapEntry = (MapEntryNode)entry; - Optional> idLeaf = - mapEntry.getChild(new YangInstanceIdentifier.NodeIdentifier(TestModel.ID_QNAME)); - assertTrue("Missing leaf " + TestModel.ID_QNAME.getLocalName(), idLeaf.isPresent()); - assertEquals(TestModel.ID_QNAME.getLocalName() + " value", 1, idLeaf.get().getValue()); + verifyOuterListEntry(shard, 1); - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); - }}; + verifyLastApplied(shard, 2); + } + }; } @Test - public void testBatchedModificationsOnTransactionChain() throws Throwable { - new ShardTestKit(getSystem()) {{ - final TestActorRef shard = TestActorRef.create(getSystem(), - newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testBatchedModificationsOnTransactionChain"); - - waitUntilLeader(shard); - - String transactionChainID = "txChain"; - String transactionID1 = "tx1"; - String transactionID2 = "tx2"; + public void testBatchedModificationsWithNoCommitOnReady() throws Exception { + new ShardTestKit(getSystem()) { + { + final TestActorRef shard = actorFactory.createTestActor( + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testBatchedModificationsWithNoCommitOnReady"); - FiniteDuration duration = duration("5 seconds"); + waitUntilLeader(shard); - // Send a BatchedModifications to start a chained write transaction and ready it. + final TransactionIdentifier transactionID = nextTransactionId(); + final FiniteDuration duration = duration("5 seconds"); - ContainerNode containerNode = ImmutableNodes.containerNode(TestModel.TEST_QNAME); - YangInstanceIdentifier path = TestModel.TEST_PATH; - shard.tell(newBatchedModifications(transactionID1, transactionChainID, path, - containerNode, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.class); + // Send a BatchedModifications to start a transaction. - // Create a read Tx on the same chain. + shard.tell(newBatchedModifications(transactionID, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), false, false, 1), getRef()); + expectMsgClass(duration, BatchedModificationsReply.class); - shard.tell(new CreateTransaction(transactionID2, TransactionProxy.TransactionType.READ_ONLY.ordinal() , - transactionChainID).toSerializable(), getRef()); + // Send a couple more BatchedModifications. - CreateTransactionReply createReply = expectMsgClass(duration("3 seconds"), CreateTransactionReply.class); + shard.tell( + newBatchedModifications(transactionID, TestModel.OUTER_LIST_PATH, + ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build(), false, false, 2), + getRef()); + expectMsgClass(duration, BatchedModificationsReply.class); - getSystem().actorSelection(createReply.getTransactionActorPath()).tell(new ReadData(path), getRef()); - ReadDataReply readReply = expectMsgClass(duration("3 seconds"), ReadDataReply.class); - assertEquals("Read node", containerNode, readReply.getNormalizedNode()); + shard.tell(newBatchedModifications(transactionID, + YangInstanceIdentifier.builder(TestModel.OUTER_LIST_PATH) + .nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1).build(), + ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1), true, false, 3), + getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); - // Commit the write transaction. + // Send the CanCommitTransaction message. - shard.tell(new CanCommitTransaction(transactionID1).toSerializable(), getRef()); - CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( - expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); - assertEquals("Can commit", true, canCommitReply.getCanCommit()); + shard.tell(new CanCommitTransaction(transactionID, CURRENT_VERSION).toSerializable(), getRef()); + final CanCommitTransactionReply canCommitReply = CanCommitTransactionReply + .fromSerializable(expectMsgClass(duration, CanCommitTransactionReply.class)); + assertEquals("Can commit", true, canCommitReply.getCanCommit()); - shard.tell(new CommitTransaction(transactionID1).toSerializable(), getRef()); - expectMsgClass(duration, CommitTransactionReply.SERIALIZABLE_CLASS); + // Send the CommitTransaction message. - // Verify data in the data store. + shard.tell(new CommitTransaction(transactionID, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, CommitTransactionReply.class); - NormalizedNode actualNode = readStore(shard, path); - assertEquals("Stored node", containerNode, actualNode); + // Verify data in the data store. - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); - }}; + verifyOuterListEntry(shard, 1); + } + }; } @Test - public void testOnBatchedModificationsWhenNotLeader() { - final AtomicBoolean overrideLeaderCalls = new AtomicBoolean(); - new ShardTestKit(getSystem()) {{ - Creator creator = new Creator() { - @Override - public Shard create() throws Exception { - return new Shard(shardID, Collections.emptyMap(), - newDatastoreContext(), SCHEMA_CONTEXT) { - @Override - protected boolean isLeader() { - return overrideLeaderCalls.get() ? false : super.isLeader(); - } - - @Override - protected ActorSelection getLeader() { - return overrideLeaderCalls.get() ? getSystem().actorSelection(getRef().path()) : - super.getLeader(); - } - }; - } - }; - - TestActorRef shard = TestActorRef.create(getSystem(), - Props.create(new DelegatingShardCreator(creator)), "testOnBatchedModificationsWhenNotLeader"); - - waitUntilLeader(shard); - - overrideLeaderCalls.set(true); - - BatchedModifications batched = new BatchedModifications("tx", DataStoreVersions.CURRENT_VERSION, ""); - - shard.tell(batched, ActorRef.noSender()); - - expectMsgEquals(batched); + public void testBatchedModificationsWithCommitOnReady() throws Exception { + new ShardTestKit(getSystem()) { + { + final TestActorRef shard = actorFactory.createTestActor( + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testBatchedModificationsWithCommitOnReady"); - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); - }}; - } + waitUntilLeader(shard); - @Test - public void testCommitWithPersistenceDisabled() throws Throwable { - dataStoreContextBuilder.persistent(false); - new ShardTestKit(getSystem()) {{ - final TestActorRef shard = TestActorRef.create(getSystem(), - newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testCommitWithPersistenceDisabled"); + final TransactionIdentifier transactionID = nextTransactionId(); + final FiniteDuration duration = duration("5 seconds"); - waitUntilLeader(shard); + // Send a BatchedModifications to start a transaction. - InMemoryDOMDataStore dataStore = shard.underlyingActor().getDataStore(); + shard.tell(newBatchedModifications(transactionID, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), false, false, 1), getRef()); + expectMsgClass(duration, BatchedModificationsReply.class); - // Setup a simulated transactions with a mock cohort. + // Send a couple more BatchedModifications. - String transactionID = "tx"; - MutableCompositeModification modification = new MutableCompositeModification(); - NormalizedNode containerNode = ImmutableNodes.containerNode(TestModel.TEST_QNAME); - DOMStoreThreePhaseCommitCohort cohort = setupMockWriteTransaction("cohort", dataStore, - TestModel.TEST_PATH, containerNode, modification); + shard.tell(newBatchedModifications(transactionID, TestModel.OUTER_LIST_PATH, + ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build(), false, false, 2), + getRef()); + expectMsgClass(duration, BatchedModificationsReply.class); - FiniteDuration duration = duration("5 seconds"); + shard.tell(newBatchedModifications(transactionID, + YangInstanceIdentifier.builder(TestModel.OUTER_LIST_PATH) + .nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1).build(), + ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1), true, true, 3), + getRef()); - // Simulate the ForwardedReadyTransaction messages that would be sent - // by the ShardTransaction. + expectMsgClass(duration, CommitTransactionReply.class); - shard.tell(new ForwardedReadyTransaction(transactionID, CURRENT_VERSION, - cohort, modification, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.class); + // Verify data in the data store. - // Send the CanCommitTransaction message. + verifyOuterListEntry(shard, 1); + } + }; + } - shard.tell(new CanCommitTransaction(transactionID).toSerializable(), getRef()); - CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( - expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); - assertEquals("Can commit", true, canCommitReply.getCanCommit()); + @Test(expected = IllegalStateException.class) + public void testBatchedModificationsReadyWithIncorrectTotalMessageCount() throws Exception { + new ShardTestKit(getSystem()) { + { + final TestActorRef shard = actorFactory.createTestActor( + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testBatchedModificationsReadyWithIncorrectTotalMessageCount"); - // Send the CanCommitTransaction message. + waitUntilLeader(shard); - shard.tell(new CommitTransaction(transactionID).toSerializable(), getRef()); - expectMsgClass(duration, CommitTransactionReply.SERIALIZABLE_CLASS); + final TransactionIdentifier transactionID = nextTransactionId(); + final BatchedModifications batched = new BatchedModifications(transactionID, + DataStoreVersions.CURRENT_VERSION); + batched.setReady(true); + batched.setTotalMessagesSent(2); - InOrder inOrder = inOrder(cohort); - inOrder.verify(cohort).canCommit(); - inOrder.verify(cohort).preCommit(); - inOrder.verify(cohort).commit(); + shard.tell(batched, getRef()); - NormalizedNode actualNode = readStore(shard, TestModel.TEST_PATH); - assertEquals(TestModel.TEST_QNAME.getLocalName(), containerNode, actualNode); + final Failure failure = expectMsgClass(duration("5 seconds"), akka.actor.Status.Failure.class); - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); - }}; + if (failure != null) { + Throwables.propagateIfInstanceOf(failure.cause(), Exception.class); + Throwables.propagate(failure.cause()); + } + } + }; } @Test - public void testCommitWhenTransactionHasNoModifications(){ - // Note that persistence is enabled which would normally result in the entry getting written to the journal - // but here that need not happen + public void testBatchedModificationsWithOperationFailure() throws Exception { new ShardTestKit(getSystem()) { { - final TestActorRef shard = TestActorRef.create(getSystem(), + final TestActorRef shard = actorFactory.createTestActor( newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testCommitWhenTransactionHasNoModifications"); + "testBatchedModificationsWithOperationFailure"); waitUntilLeader(shard); - String transactionID = "tx1"; - MutableCompositeModification modification = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort = mock(DOMStoreThreePhaseCommitCohort.class, "cohort1"); - doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort).canCommit(); - doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort).preCommit(); - doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort).commit(); - - FiniteDuration duration = duration("5 seconds"); - - // Simulate the ForwardedReadyTransaction messages that would be sent - // by the ShardTransaction. - - shard.tell(new ForwardedReadyTransaction(transactionID, CURRENT_VERSION, - cohort, modification, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.class); - - // Send the CanCommitTransaction message. + // Test merge with invalid data. An exception should occur when + // the merge is applied. Note that + // write will not validate the children for performance reasons. - shard.tell(new CanCommitTransaction(transactionID).toSerializable(), getRef()); - CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( - expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); - assertEquals("Can commit", true, canCommitReply.getCanCommit()); + final TransactionIdentifier transactionID = nextTransactionId(); - shard.tell(new CommitTransaction(transactionID).toSerializable(), getRef()); - expectMsgClass(duration, ThreePhaseCommitCohortMessages.CommitTransactionReply.class); + final ContainerNode invalidData = ImmutableContainerNodeBuilder.create() + .withNodeIdentifier(new YangInstanceIdentifier.NodeIdentifier(TestModel.TEST_QNAME)) + .withChild(ImmutableNodes.leafNode(TestModel.JUNK_QNAME, "junk")).build(); - InOrder inOrder = inOrder(cohort); - inOrder.verify(cohort).canCommit(); - inOrder.verify(cohort).preCommit(); - inOrder.verify(cohort).commit(); + BatchedModifications batched = new BatchedModifications(transactionID, CURRENT_VERSION); + batched.addModification(new MergeModification(TestModel.TEST_PATH, invalidData)); + shard.tell(batched, getRef()); + Failure failure = expectMsgClass(duration("5 seconds"), akka.actor.Status.Failure.class); - // Use MBean for verification - // Committed transaction count should increase as usual - assertEquals(1,shard.underlyingActor().getShardMBean().getCommittedTransactionsCount()); + final Throwable cause = failure.cause(); - // Commit index should not advance because this does not go into the journal - assertEquals(-1, shard.underlyingActor().getShardMBean().getCommitIndex()); + batched = new BatchedModifications(transactionID, DataStoreVersions.CURRENT_VERSION); + batched.setReady(true); + batched.setTotalMessagesSent(2); - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); + shard.tell(batched, getRef()); + failure = expectMsgClass(duration("5 seconds"), akka.actor.Status.Failure.class); + assertEquals("Failure cause", cause, failure.cause()); } }; } @Test - public void testCommitWhenTransactionHasModifications(){ + public void testBatchedModificationsOnTransactionChain() throws Exception { new ShardTestKit(getSystem()) { { - final TestActorRef shard = TestActorRef.create(getSystem(), + final TestActorRef shard = actorFactory.createTestActor( newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testCommitWhenTransactionHasModifications"); + "testBatchedModificationsOnTransactionChain"); waitUntilLeader(shard); - String transactionID = "tx1"; - MutableCompositeModification modification = new MutableCompositeModification(); - modification.addModification(new DeleteModification(YangInstanceIdentifier.builder().build())); - DOMStoreThreePhaseCommitCohort cohort = mock(DOMStoreThreePhaseCommitCohort.class, "cohort1"); - doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort).canCommit(); - doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort).preCommit(); - doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort).commit(); + final LocalHistoryIdentifier historyId = nextHistoryId(); + final TransactionIdentifier transactionID1 = new TransactionIdentifier(historyId, 0); + final TransactionIdentifier transactionID2 = new TransactionIdentifier(historyId, 1); - FiniteDuration duration = duration("5 seconds"); + final FiniteDuration duration = duration("5 seconds"); - // Simulate the ForwardedReadyTransaction messages that would be sent - // by the ShardTransaction. + // Send a BatchedModifications to start a chained write + // transaction and ready it. - shard.tell(new ForwardedReadyTransaction(transactionID, CURRENT_VERSION, - cohort, modification, true), getRef()); + final ContainerNode containerNode = ImmutableNodes.containerNode(TestModel.TEST_QNAME); + final YangInstanceIdentifier path = TestModel.TEST_PATH; + shard.tell(newBatchedModifications(transactionID1, path, containerNode, true, false, 1), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); - // Send the CanCommitTransaction message. + // Create a read Tx on the same chain. - shard.tell(new CanCommitTransaction(transactionID).toSerializable(), getRef()); - CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( - expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); - assertEquals("Can commit", true, canCommitReply.getCanCommit()); + shard.tell(new CreateTransaction(transactionID2, TransactionType.READ_ONLY.ordinal(), + DataStoreVersions.CURRENT_VERSION).toSerializable(), getRef()); - shard.tell(new CommitTransaction(transactionID).toSerializable(), getRef()); - expectMsgClass(duration, ThreePhaseCommitCohortMessages.CommitTransactionReply.class); + final CreateTransactionReply createReply = expectMsgClass(duration("3 seconds"), + CreateTransactionReply.class); - InOrder inOrder = inOrder(cohort); - inOrder.verify(cohort).canCommit(); - inOrder.verify(cohort).preCommit(); - inOrder.verify(cohort).commit(); + getSystem().actorSelection(createReply.getTransactionPath()) + .tell(new ReadData(path, DataStoreVersions.CURRENT_VERSION), getRef()); + final ReadDataReply readReply = expectMsgClass(duration("3 seconds"), ReadDataReply.class); + assertEquals("Read node", containerNode, readReply.getNormalizedNode()); - // Use MBean for verification - // Committed transaction count should increase as usual - assertEquals(1, shard.underlyingActor().getShardMBean().getCommittedTransactionsCount()); + // Commit the write transaction. + + shard.tell(new CanCommitTransaction(transactionID1, CURRENT_VERSION).toSerializable(), getRef()); + final CanCommitTransactionReply canCommitReply = CanCommitTransactionReply + .fromSerializable(expectMsgClass(duration, CanCommitTransactionReply.class)); + assertEquals("Can commit", true, canCommitReply.getCanCommit()); - // Commit index should advance as we do not have an empty modification - assertEquals(0, shard.underlyingActor().getShardMBean().getCommitIndex()); + shard.tell(new CommitTransaction(transactionID1, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, CommitTransactionReply.class); - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); + // Verify data in the data store. + final NormalizedNode actualNode = readStore(shard, path); + assertEquals("Stored node", containerNode, actualNode); } }; } @Test - public void testCommitPhaseFailure() throws Throwable { - new ShardTestKit(getSystem()) {{ - final TestActorRef shard = TestActorRef.create(getSystem(), - newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testCommitPhaseFailure"); + public void testOnBatchedModificationsWhenNotLeader() { + final AtomicBoolean overrideLeaderCalls = new AtomicBoolean(); + new ShardTestKit(getSystem()) { + { + final Creator creator = new Creator() { + private static final long serialVersionUID = 1L; + + @Override + public Shard create() throws Exception { + return new Shard(newShardBuilder()) { + @Override + protected boolean isLeader() { + return overrideLeaderCalls.get() ? false : super.isLeader(); + } + + @Override + public ActorSelection getLeader() { + return overrideLeaderCalls.get() ? getSystem().actorSelection(getRef().path()) + : super.getLeader(); + } + }; + } + }; - waitUntilLeader(shard); + final TestActorRef shard = actorFactory.createTestActor(Props + .create(new DelegatingShardCreator(creator)).withDispatcher(Dispatchers.DefaultDispatcherId()), + "testOnBatchedModificationsWhenNotLeader"); - // Setup 2 simulated transactions with mock cohorts. The first one fails in the - // commit phase. + waitUntilLeader(shard); - String transactionID1 = "tx1"; - MutableCompositeModification modification1 = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort1 = mock(DOMStoreThreePhaseCommitCohort.class, "cohort1"); - doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort1).canCommit(); - doReturn(Futures.immediateFuture(null)).when(cohort1).preCommit(); - doReturn(Futures.immediateFailedFuture(new IllegalStateException("mock"))).when(cohort1).commit(); + overrideLeaderCalls.set(true); - String transactionID2 = "tx2"; - MutableCompositeModification modification2 = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort2 = mock(DOMStoreThreePhaseCommitCohort.class, "cohort2"); - doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort2).canCommit(); + final BatchedModifications batched = new BatchedModifications(nextTransactionId(), + DataStoreVersions.CURRENT_VERSION); - FiniteDuration duration = duration("5 seconds"); - final Timeout timeout = new Timeout(duration); + shard.tell(batched, ActorRef.noSender()); - // Simulate the ForwardedReadyTransaction messages that would be sent - // by the ShardTransaction. + expectMsgEquals(batched); + } + }; + } - shard.tell(new ForwardedReadyTransaction(transactionID1, CURRENT_VERSION, - cohort1, modification1, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.class); + @Test + public void testTransactionMessagesWithNoLeader() { + new ShardTestKit(getSystem()) { + { + dataStoreContextBuilder.customRaftPolicyImplementation(DisableElectionsRaftPolicy.class.getName()) + .shardHeartbeatIntervalInMillis(50).shardElectionTimeoutFactor(1); + final TestActorRef shard = actorFactory.createTestActor( + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testTransactionMessagesWithNoLeader"); - shard.tell(new ForwardedReadyTransaction(transactionID2, CURRENT_VERSION, - cohort2, modification2, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.class); + waitUntilNoLeader(shard); - // Send the CanCommitTransaction message for the first Tx. + final TransactionIdentifier txId = nextTransactionId(); + shard.tell(new BatchedModifications(txId, DataStoreVersions.CURRENT_VERSION), getRef()); + Failure failure = expectMsgClass(Failure.class); + assertEquals("Failure cause type", NoShardLeaderException.class, failure.cause().getClass()); - shard.tell(new CanCommitTransaction(transactionID1).toSerializable(), getRef()); - CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( - expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); - assertEquals("Can commit", true, canCommitReply.getCanCommit()); + shard.tell(prepareForwardedReadyTransaction(shard, txId, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true), getRef()); + failure = expectMsgClass(Failure.class); + assertEquals("Failure cause type", NoShardLeaderException.class, failure.cause().getClass()); - // Send the CanCommitTransaction message for the 2nd Tx. This should get queued and - // processed after the first Tx completes. + shard.tell(new ReadyLocalTransaction(txId, mock(DataTreeModification.class), true), getRef()); + failure = expectMsgClass(Failure.class); + assertEquals("Failure cause type", NoShardLeaderException.class, failure.cause().getClass()); + } + }; + } - Future canCommitFuture = Patterns.ask(shard, - new CanCommitTransaction(transactionID2).toSerializable(), timeout); + @Test + public void testReadyWithReadWriteImmediateCommit() throws Exception { + testReadyWithImmediateCommit(true); + } - // Send the CommitTransaction message for the first Tx. This should send back an error - // and trigger the 2nd Tx to proceed. + @Test + public void testReadyWithWriteOnlyImmediateCommit() throws Exception { + testReadyWithImmediateCommit(false); + } - shard.tell(new CommitTransaction(transactionID1).toSerializable(), getRef()); - expectMsgClass(duration, akka.actor.Status.Failure.class); + private void testReadyWithImmediateCommit(final boolean readWrite) throws Exception { + new ShardTestKit(getSystem()) { + { + final TestActorRef shard = actorFactory.createTestActor( + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testReadyWithImmediateCommit-" + readWrite); - // Wait for the 2nd Tx to complete the canCommit phase. + waitUntilLeader(shard); - final CountDownLatch latch = new CountDownLatch(1); - canCommitFuture.onComplete(new OnComplete() { - @Override - public void onComplete(final Throwable t, final Object resp) { - latch.countDown(); + final TransactionIdentifier transactionID = nextTransactionId(); + final NormalizedNode containerNode = ImmutableNodes.containerNode(TestModel.TEST_QNAME); + if (readWrite) { + shard.tell(prepareForwardedReadyTransaction(shard, transactionID, TestModel.TEST_PATH, + containerNode, true), getRef()); + } else { + shard.tell(prepareBatchedModifications(transactionID, TestModel.TEST_PATH, containerNode, true), + getRef()); } - }, getSystem().dispatcher()); - assertEquals("2nd CanCommit complete", true, latch.await(5, TimeUnit.SECONDS)); + expectMsgClass(duration("5 seconds"), CommitTransactionReply.class); - InOrder inOrder = inOrder(cohort1, cohort2); - inOrder.verify(cohort1).canCommit(); - inOrder.verify(cohort1).preCommit(); - inOrder.verify(cohort1).commit(); - inOrder.verify(cohort2).canCommit(); - - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); - }}; + final NormalizedNode actualNode = readStore(shard, TestModel.TEST_PATH); + assertEquals(TestModel.TEST_QNAME.getLocalName(), containerNode, actualNode); + } + }; } @Test - public void testPreCommitPhaseFailure() throws Throwable { - new ShardTestKit(getSystem()) {{ - final TestActorRef shard = TestActorRef.create(getSystem(), - newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testPreCommitPhaseFailure"); - - waitUntilLeader(shard); - - String transactionID = "tx1"; - MutableCompositeModification modification = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort = mock(DOMStoreThreePhaseCommitCohort.class, "cohort1"); - doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort).canCommit(); - doReturn(Futures.immediateFailedFuture(new IllegalStateException("mock"))).when(cohort).preCommit(); - - FiniteDuration duration = duration("5 seconds"); + public void testReadyLocalTransactionWithImmediateCommit() throws Exception { + new ShardTestKit(getSystem()) { + { + final TestActorRef shard = actorFactory.createTestActor( + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testReadyLocalTransactionWithImmediateCommit"); - // Simulate the ForwardedReadyTransaction messages that would be sent - // by the ShardTransaction. + waitUntilLeader(shard); - shard.tell(new ForwardedReadyTransaction(transactionID, CURRENT_VERSION, - cohort, modification, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.class); + final ShardDataTree dataStore = shard.underlyingActor().getDataStore(); - // Send the CanCommitTransaction message. + final DataTreeModification modification = dataStore.newModification(); - shard.tell(new CanCommitTransaction(transactionID).toSerializable(), getRef()); - CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( - expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); - assertEquals("Can commit", true, canCommitReply.getCanCommit()); + final ContainerNode writeData = ImmutableNodes.containerNode(TestModel.TEST_QNAME); + new WriteModification(TestModel.TEST_PATH, writeData).apply(modification); + final MapNode mergeData = ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build(); + new MergeModification(TestModel.OUTER_LIST_PATH, mergeData).apply(modification); - // Send the CommitTransaction message. This should send back an error - // for preCommit failure. + final TransactionIdentifier txId = nextTransactionId(); + modification.ready(); + final ReadyLocalTransaction readyMessage = new ReadyLocalTransaction(txId, modification, true); - shard.tell(new CommitTransaction(transactionID).toSerializable(), getRef()); - expectMsgClass(duration, akka.actor.Status.Failure.class); + shard.tell(readyMessage, getRef()); - InOrder inOrder = inOrder(cohort); - inOrder.verify(cohort).canCommit(); - inOrder.verify(cohort).preCommit(); + expectMsgClass(CommitTransactionReply.class); - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); - }}; + final NormalizedNode actualNode = readStore(shard, TestModel.OUTER_LIST_PATH); + assertEquals(TestModel.OUTER_LIST_QNAME.getLocalName(), mergeData, actualNode); + } + }; } @Test - public void testCanCommitPhaseFailure() throws Throwable { - new ShardTestKit(getSystem()) {{ - final TestActorRef shard = TestActorRef.create(getSystem(), - newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testCanCommitPhaseFailure"); - - waitUntilLeader(shard); - - final FiniteDuration duration = duration("5 seconds"); - - String transactionID = "tx1"; - MutableCompositeModification modification = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort = mock(DOMStoreThreePhaseCommitCohort.class, "cohort1"); - doReturn(Futures.immediateFailedFuture(new IllegalStateException("mock"))).when(cohort).canCommit(); - - // Simulate the ForwardedReadyTransaction messages that would be sent - // by the ShardTransaction. + public void testReadyLocalTransactionWithThreePhaseCommit() throws Exception { + new ShardTestKit(getSystem()) { + { + final TestActorRef shard = actorFactory.createTestActor( + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testReadyLocalTransactionWithThreePhaseCommit"); - shard.tell(new ForwardedReadyTransaction(transactionID, CURRENT_VERSION, - cohort, modification, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.class); + waitUntilLeader(shard); - // Send the CanCommitTransaction message. + final ShardDataTree dataStore = shard.underlyingActor().getDataStore(); - shard.tell(new CanCommitTransaction(transactionID).toSerializable(), getRef()); - expectMsgClass(duration, akka.actor.Status.Failure.class); + final DataTreeModification modification = dataStore.newModification(); - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); - }}; - } + final ContainerNode writeData = ImmutableNodes.containerNode(TestModel.TEST_QNAME); + new WriteModification(TestModel.TEST_PATH, writeData).apply(modification); + final MapNode mergeData = ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build(); + new MergeModification(TestModel.OUTER_LIST_PATH, mergeData).apply(modification); - @Test - public void testAbortBeforeFinishCommit() throws Throwable { - new ShardTestKit(getSystem()) {{ - final TestActorRef shard = TestActorRef.create(getSystem(), - newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testAbortBeforeFinishCommit"); - - waitUntilLeader(shard); - - final FiniteDuration duration = duration("5 seconds"); - InMemoryDOMDataStore dataStore = shard.underlyingActor().getDataStore(); - - final String transactionID = "tx1"; - Function> preCommit = - new Function>() { - @Override - public ListenableFuture apply(final DOMStoreThreePhaseCommitCohort cohort) { - ListenableFuture preCommitFuture = cohort.preCommit(); - - // Simulate an AbortTransaction message occurring during replication, after - // persisting and before finishing the commit to the in-memory store. - // We have no followers so due to optimizations in the RaftActor, it does not - // attempt replication and thus we can't send an AbortTransaction message b/c - // it would be processed too late after CommitTransaction completes. So we'll - // simulate an AbortTransaction message occurring during replication by calling - // the shard directly. - // - shard.underlyingActor().doAbortTransaction(transactionID, null); - - return preCommitFuture; - } - }; + final TransactionIdentifier txId = nextTransactionId(); + modification.ready(); + final ReadyLocalTransaction readyMessage = new ReadyLocalTransaction(txId, modification, false); - MutableCompositeModification modification = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort = setupMockWriteTransaction("cohort1", dataStore, - TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME), - modification, preCommit); + shard.tell(readyMessage, getRef()); - shard.tell(new ForwardedReadyTransaction(transactionID, CURRENT_VERSION, - cohort, modification, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.class); + expectMsgClass(ReadyTransactionReply.class); - shard.tell(new CanCommitTransaction(transactionID).toSerializable(), getRef()); - CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( - expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); - assertEquals("Can commit", true, canCommitReply.getCanCommit()); + // Send the CanCommitTransaction message. - shard.tell(new CommitTransaction(transactionID).toSerializable(), getRef()); - expectMsgClass(duration, CommitTransactionReply.SERIALIZABLE_CLASS); + shard.tell(new CanCommitTransaction(txId, CURRENT_VERSION).toSerializable(), getRef()); + final CanCommitTransactionReply canCommitReply = CanCommitTransactionReply + .fromSerializable(expectMsgClass(CanCommitTransactionReply.class)); + assertEquals("Can commit", true, canCommitReply.getCanCommit()); - NormalizedNode node = readStore(shard, TestModel.TEST_PATH); + // Send the CanCommitTransaction message. - // Since we're simulating an abort occurring during replication and before finish commit, - // the data should still get written to the in-memory store since we've gotten past - // canCommit and preCommit and persisted the data. - assertNotNull(TestModel.TEST_QNAME.getLocalName() + " not found", node); + shard.tell(new CommitTransaction(txId, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(CommitTransactionReply.class); - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); - }}; + final NormalizedNode actualNode = readStore(shard, TestModel.OUTER_LIST_PATH); + assertEquals(TestModel.OUTER_LIST_QNAME.getLocalName(), mergeData, actualNode); + } + }; } @Test - public void testTransactionCommitTimeout() throws Throwable { - dataStoreContextBuilder.shardTransactionCommitTimeoutInSeconds(1); + public void testReadWriteCommitWithPersistenceDisabled() throws Exception { + dataStoreContextBuilder.persistent(false); + new ShardTestKit(getSystem()) { + { + final TestActorRef shard = actorFactory.createTestActor( + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testCommitWithPersistenceDisabled"); + + waitUntilLeader(shard); + + // Setup a simulated transactions with a mock cohort. + + final FiniteDuration duration = duration("5 seconds"); - new ShardTestKit(getSystem()) {{ - final TestActorRef shard = TestActorRef.create(getSystem(), - newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testTransactionCommitTimeout"); + final TransactionIdentifier transactionID = nextTransactionId(); + final NormalizedNode containerNode = ImmutableNodes.containerNode(TestModel.TEST_QNAME); + shard.tell(prepareBatchedModifications(transactionID, TestModel.TEST_PATH, containerNode, false), + getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); + + // Send the CanCommitTransaction message. - waitUntilLeader(shard); + shard.tell(new CanCommitTransaction(transactionID, CURRENT_VERSION).toSerializable(), getRef()); + final CanCommitTransactionReply canCommitReply = CanCommitTransactionReply + .fromSerializable(expectMsgClass(duration, CanCommitTransactionReply.class)); + assertEquals("Can commit", true, canCommitReply.getCanCommit()); - final FiniteDuration duration = duration("5 seconds"); + // Send the CanCommitTransaction message. - InMemoryDOMDataStore dataStore = shard.underlyingActor().getDataStore(); + shard.tell(new CommitTransaction(transactionID, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, CommitTransactionReply.class); - writeToStore(shard, TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); - writeToStore(shard, TestModel.OUTER_LIST_PATH, - ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build()); + final NormalizedNode actualNode = readStore(shard, TestModel.TEST_PATH); + assertEquals(TestModel.TEST_QNAME.getLocalName(), containerNode, actualNode); + } + }; + } - // Create 1st Tx - will timeout + @Test + public void testReadWriteCommitWhenTransactionHasNoModifications() { + testCommitWhenTransactionHasNoModifications(true); + } - String transactionID1 = "tx1"; - MutableCompositeModification modification1 = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort1 = setupMockWriteTransaction("cohort1", dataStore, - YangInstanceIdentifier.builder(TestModel.OUTER_LIST_PATH) - .nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1).build(), - ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1), - modification1); + @Test + public void testWriteOnlyCommitWhenTransactionHasNoModifications() { + testCommitWhenTransactionHasNoModifications(false); + } - // Create 2nd Tx + private void testCommitWhenTransactionHasNoModifications(final boolean readWrite) { + // Note that persistence is enabled which would normally result in the + // entry getting written to the journal + // but here that need not happen + new ShardTestKit(getSystem()) { + { + final TestActorRef shard = actorFactory.createTestActor( + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testCommitWhenTransactionHasNoModifications-" + readWrite); - String transactionID2 = "tx3"; - MutableCompositeModification modification2 = new MutableCompositeModification(); - YangInstanceIdentifier listNodePath = YangInstanceIdentifier.builder(TestModel.OUTER_LIST_PATH) - .nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 2).build(); - DOMStoreThreePhaseCommitCohort cohort2 = setupMockWriteTransaction("cohort3", dataStore, - listNodePath, - ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 2), - modification2); + waitUntilLeader(shard); - // Ready the Tx's + final TransactionIdentifier transactionID = nextTransactionId(); - shard.tell(new ForwardedReadyTransaction(transactionID1, CURRENT_VERSION, - cohort1, modification1, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.class); + final FiniteDuration duration = duration("5 seconds"); - shard.tell(new ForwardedReadyTransaction(transactionID2, CURRENT_VERSION, - cohort2, modification2, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.class); + if (readWrite) { + final ReadWriteShardDataTreeTransaction rwTx = shard.underlyingActor().getDataStore() + .newReadWriteTransaction(transactionID); + shard.tell(new ForwardedReadyTransaction(transactionID, CURRENT_VERSION, rwTx, false), getRef()); + } else { + shard.tell(prepareBatchedModifications(transactionID, new MutableCompositeModification()), + getRef()); + } - // canCommit 1st Tx. We don't send the commit so it should timeout. + expectMsgClass(duration, ReadyTransactionReply.class); - shard.tell(new CanCommitTransaction(transactionID1).toSerializable(), getRef()); - expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS); + // Send the CanCommitTransaction message. - // canCommit the 2nd Tx - it should complete after the 1st Tx times out. + shard.tell(new CanCommitTransaction(transactionID, CURRENT_VERSION).toSerializable(), getRef()); + final CanCommitTransactionReply canCommitReply = CanCommitTransactionReply + .fromSerializable(expectMsgClass(duration, CanCommitTransactionReply.class)); + assertEquals("Can commit", true, canCommitReply.getCanCommit()); - shard.tell(new CanCommitTransaction(transactionID2).toSerializable(), getRef()); - expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS); + shard.tell(new CommitTransaction(transactionID, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, CommitTransactionReply.class); - // Commit the 2nd Tx. + shard.tell(Shard.GET_SHARD_MBEAN_MESSAGE, getRef()); + final ShardStats shardStats = expectMsgClass(duration, ShardStats.class); - shard.tell(new CommitTransaction(transactionID2).toSerializable(), getRef()); - expectMsgClass(duration, CommitTransactionReply.SERIALIZABLE_CLASS); + // Use MBean for verification + // Committed transaction count should increase as usual + assertEquals(1, shardStats.getCommittedTransactionsCount()); - NormalizedNode node = readStore(shard, listNodePath); - assertNotNull(listNodePath + " not found", node); + // Commit index should not advance because this does not go into + // the journal + assertEquals(-1, shardStats.getCommitIndex()); + } + }; + } - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); - }}; + @Test + public void testReadWriteCommitWhenTransactionHasModifications() throws Exception { + testCommitWhenTransactionHasModifications(true); } @Test - public void testTransactionCommitQueueCapacityExceeded() throws Throwable { - dataStoreContextBuilder.shardTransactionCommitQueueCapacity(1); + public void testWriteOnlyCommitWhenTransactionHasModifications() throws Exception { + testCommitWhenTransactionHasModifications(false); + } + + private void testCommitWhenTransactionHasModifications(final boolean readWrite) throws Exception { + new ShardTestKit(getSystem()) { + { + final TipProducingDataTree dataTree = createDelegatingMockDataTree(); + final TestActorRef shard = actorFactory.createTestActor( + newShardBuilder().dataTree(dataTree).props().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testCommitWhenTransactionHasModifications-" + readWrite); + + waitUntilLeader(shard); + + final FiniteDuration duration = duration("5 seconds"); + final TransactionIdentifier transactionID = nextTransactionId(); + + if (readWrite) { + shard.tell(prepareForwardedReadyTransaction(shard, transactionID, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), false), getRef()); + } else { + shard.tell(prepareBatchedModifications(transactionID, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), false), getRef()); + } + + expectMsgClass(duration, ReadyTransactionReply.class); + + // Send the CanCommitTransaction message. + + shard.tell(new CanCommitTransaction(transactionID, CURRENT_VERSION).toSerializable(), getRef()); + final CanCommitTransactionReply canCommitReply = CanCommitTransactionReply + .fromSerializable(expectMsgClass(duration, CanCommitTransactionReply.class)); + assertEquals("Can commit", true, canCommitReply.getCanCommit()); + + shard.tell(new CommitTransaction(transactionID, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, CommitTransactionReply.class); + + final InOrder inOrder = inOrder(dataTree); + inOrder.verify(dataTree).validate(any(DataTreeModification.class)); + inOrder.verify(dataTree).prepare(any(DataTreeModification.class)); + inOrder.verify(dataTree).commit(any(DataTreeCandidate.class)); + + shard.tell(Shard.GET_SHARD_MBEAN_MESSAGE, getRef()); + final ShardStats shardStats = expectMsgClass(duration, ShardStats.class); - new ShardTestKit(getSystem()) {{ - final TestActorRef shard = TestActorRef.create(getSystem(), - newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testTransactionCommitQueueCapacityExceeded"); + // Use MBean for verification + // Committed transaction count should increase as usual + assertEquals(1, shardStats.getCommittedTransactionsCount()); + + // Commit index should advance as we do not have an empty + // modification + assertEquals(0, shardStats.getCommitIndex()); + } + }; + } - waitUntilLeader(shard); + @Test + public void testCommitPhaseFailure() throws Exception { + new ShardTestKit(getSystem()) { + { + final TipProducingDataTree dataTree = createDelegatingMockDataTree(); + final TestActorRef shard = actorFactory.createTestActor( + newShardBuilder().dataTree(dataTree).props().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testCommitPhaseFailure"); - final FiniteDuration duration = duration("5 seconds"); + waitUntilLeader(shard); - InMemoryDOMDataStore dataStore = shard.underlyingActor().getDataStore(); + final FiniteDuration duration = duration("5 seconds"); + final Timeout timeout = new Timeout(duration); - String transactionID1 = "tx1"; - MutableCompositeModification modification1 = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort1 = setupMockWriteTransaction("cohort1", dataStore, - TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME), modification1); + // Setup 2 simulated transactions with mock cohorts. The first + // one fails in the + // commit phase. - String transactionID2 = "tx2"; - MutableCompositeModification modification2 = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort2 = setupMockWriteTransaction("cohort2", dataStore, - TestModel.OUTER_LIST_PATH, - ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build(), - modification2); + doThrow(new RuntimeException("mock commit failure")).when(dataTree) + .commit(any(DataTreeCandidate.class)); - String transactionID3 = "tx3"; - MutableCompositeModification modification3 = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort3 = setupMockWriteTransaction("cohort3", dataStore, - TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME), modification3); + final TransactionIdentifier transactionID1 = nextTransactionId(); + shard.tell(newBatchedModifications(transactionID1, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true, false, 1), getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); - // Ready the Tx's + final TransactionIdentifier transactionID2 = nextTransactionId(); + shard.tell(newBatchedModifications(transactionID2, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true, false, 1), getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); - shard.tell(new ForwardedReadyTransaction(transactionID1, CURRENT_VERSION, - cohort1, modification1, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.class); + // Send the CanCommitTransaction message for the first Tx. - shard.tell(new ForwardedReadyTransaction(transactionID2, CURRENT_VERSION, - cohort2, modification2, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.class); + shard.tell(new CanCommitTransaction(transactionID1, CURRENT_VERSION).toSerializable(), getRef()); + final CanCommitTransactionReply canCommitReply = CanCommitTransactionReply + .fromSerializable(expectMsgClass(duration, CanCommitTransactionReply.class)); + assertEquals("Can commit", true, canCommitReply.getCanCommit()); - shard.tell(new ForwardedReadyTransaction(transactionID3, CURRENT_VERSION, - cohort3, modification3, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.class); + // Send the CanCommitTransaction message for the 2nd Tx. This + // should get queued and + // processed after the first Tx completes. - // canCommit 1st Tx. + final Future canCommitFuture = Patterns.ask(shard, + new CanCommitTransaction(transactionID2, CURRENT_VERSION).toSerializable(), timeout); - shard.tell(new CanCommitTransaction(transactionID1).toSerializable(), getRef()); - expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS); + // Send the CommitTransaction message for the first Tx. This + // should send back an error + // and trigger the 2nd Tx to proceed. - // canCommit the 2nd Tx - it should get queued. + shard.tell(new CommitTransaction(transactionID1, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, akka.actor.Status.Failure.class); - shard.tell(new CanCommitTransaction(transactionID2).toSerializable(), getRef()); + // Wait for the 2nd Tx to complete the canCommit phase. - // canCommit the 3rd Tx - should exceed queue capacity and fail. + final CountDownLatch latch = new CountDownLatch(1); + canCommitFuture.onComplete(new OnComplete() { + @Override + public void onComplete(final Throwable failure, final Object resp) { + latch.countDown(); + } + }, getSystem().dispatcher()); - shard.tell(new CanCommitTransaction(transactionID3).toSerializable(), getRef()); - expectMsgClass(duration, akka.actor.Status.Failure.class); + assertEquals("2nd CanCommit complete", true, latch.await(5, TimeUnit.SECONDS)); - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); - }}; + final InOrder inOrder = inOrder(dataTree); + inOrder.verify(dataTree).validate(any(DataTreeModification.class)); + inOrder.verify(dataTree).prepare(any(DataTreeModification.class)); + inOrder.verify(dataTree).commit(any(DataTreeCandidate.class)); + inOrder.verify(dataTree).validate(any(DataTreeModification.class)); + } + }; } @Test - public void testCanCommitBeforeReadyFailure() throws Throwable { - new ShardTestKit(getSystem()) {{ - final TestActorRef shard = TestActorRef.create(getSystem(), - newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testCanCommitBeforeReadyFailure"); + public void testPreCommitPhaseFailure() throws Exception { + new ShardTestKit(getSystem()) { + { + final TipProducingDataTree dataTree = createDelegatingMockDataTree(); + final TestActorRef shard = actorFactory.createTestActor( + newShardBuilder().dataTree(dataTree).props().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testPreCommitPhaseFailure"); - shard.tell(new CanCommitTransaction("tx").toSerializable(), getRef()); - expectMsgClass(duration("5 seconds"), akka.actor.Status.Failure.class); + waitUntilLeader(shard); + + final FiniteDuration duration = duration("5 seconds"); + final Timeout timeout = new Timeout(duration); + + doThrow(new RuntimeException("mock preCommit failure")).when(dataTree) + .prepare(any(DataTreeModification.class)); + + final TransactionIdentifier transactionID1 = nextTransactionId(); + shard.tell(newBatchedModifications(transactionID1, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true, false, 1), getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); + + final TransactionIdentifier transactionID2 = nextTransactionId(); + shard.tell(newBatchedModifications(transactionID2, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true, false, 1), getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); + + // Send the CanCommitTransaction message for the first Tx. + + shard.tell(new CanCommitTransaction(transactionID1, CURRENT_VERSION).toSerializable(), getRef()); + final CanCommitTransactionReply canCommitReply = CanCommitTransactionReply + .fromSerializable(expectMsgClass(duration, CanCommitTransactionReply.class)); + assertEquals("Can commit", true, canCommitReply.getCanCommit()); + + // Send the CanCommitTransaction message for the 2nd Tx. This + // should get queued and + // processed after the first Tx completes. + + final Future canCommitFuture = Patterns.ask(shard, + new CanCommitTransaction(transactionID2, CURRENT_VERSION).toSerializable(), timeout); - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); - }}; + // Send the CommitTransaction message for the first Tx. This + // should send back an error + // and trigger the 2nd Tx to proceed. + + shard.tell(new CommitTransaction(transactionID1, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, akka.actor.Status.Failure.class); + + // Wait for the 2nd Tx to complete the canCommit phase. + + final CountDownLatch latch = new CountDownLatch(1); + canCommitFuture.onComplete(new OnComplete() { + @Override + public void onComplete(final Throwable failure, final Object resp) { + latch.countDown(); + } + }, getSystem().dispatcher()); + + assertEquals("2nd CanCommit complete", true, latch.await(5, TimeUnit.SECONDS)); + + final InOrder inOrder = inOrder(dataTree); + inOrder.verify(dataTree).validate(any(DataTreeModification.class)); + inOrder.verify(dataTree).prepare(any(DataTreeModification.class)); + inOrder.verify(dataTree).validate(any(DataTreeModification.class)); + } + }; } @Test - public void testAbortTransaction() throws Throwable { - new ShardTestKit(getSystem()) {{ - final TestActorRef shard = TestActorRef.create(getSystem(), - newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testAbortTransaction"); + public void testCanCommitPhaseFailure() throws Exception { + new ShardTestKit(getSystem()) { + { + final TipProducingDataTree dataTree = createDelegatingMockDataTree(); + final TestActorRef shard = actorFactory.createTestActor( + newShardBuilder().dataTree(dataTree).props().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testCanCommitPhaseFailure"); - waitUntilLeader(shard); + waitUntilLeader(shard); - // Setup 2 simulated transactions with mock cohorts. The first one will be aborted. + final FiniteDuration duration = duration("5 seconds"); + final TransactionIdentifier transactionID1 = nextTransactionId(); - String transactionID1 = "tx1"; - MutableCompositeModification modification1 = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort1 = mock(DOMStoreThreePhaseCommitCohort.class, "cohort1"); - doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort1).canCommit(); - doReturn(Futures.immediateFuture(null)).when(cohort1).abort(); + doThrow(new DataValidationFailedException(YangInstanceIdentifier.EMPTY, "mock canCommit failure")) + .doNothing().when(dataTree).validate(any(DataTreeModification.class)); - String transactionID2 = "tx2"; - MutableCompositeModification modification2 = new MutableCompositeModification(); - DOMStoreThreePhaseCommitCohort cohort2 = mock(DOMStoreThreePhaseCommitCohort.class, "cohort2"); - doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort2).canCommit(); + shard.tell(newBatchedModifications(transactionID1, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true, false, 1), getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); - FiniteDuration duration = duration("5 seconds"); - final Timeout timeout = new Timeout(duration); + // Send the CanCommitTransaction message. + + shard.tell(new CanCommitTransaction(transactionID1, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, akka.actor.Status.Failure.class); + + // Send another can commit to ensure the failed one got cleaned + // up. - // Simulate the ForwardedReadyTransaction messages that would be sent - // by the ShardTransaction. + final TransactionIdentifier transactionID2 = nextTransactionId(); + shard.tell(newBatchedModifications(transactionID2, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true, false, 1), getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); - shard.tell(new ForwardedReadyTransaction(transactionID1, CURRENT_VERSION, - cohort1, modification1, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.class); + shard.tell(new CanCommitTransaction(transactionID2, CURRENT_VERSION).toSerializable(), getRef()); + final CanCommitTransactionReply reply = CanCommitTransactionReply + .fromSerializable(expectMsgClass(CanCommitTransactionReply.class)); + assertEquals("getCanCommit", true, reply.getCanCommit()); + } + }; + } - shard.tell(new ForwardedReadyTransaction(transactionID2, CURRENT_VERSION, - cohort2, modification2, true), getRef()); - expectMsgClass(duration, ReadyTransactionReply.class); + @Test + public void testImmediateCommitWithCanCommitPhaseFailure() throws Exception { + testImmediateCommitWithCanCommitPhaseFailure(true); + testImmediateCommitWithCanCommitPhaseFailure(false); + } - // Send the CanCommitTransaction message for the first Tx. + private void testImmediateCommitWithCanCommitPhaseFailure(final boolean readWrite) throws Exception { + new ShardTestKit(getSystem()) { + { + final TipProducingDataTree dataTree = createDelegatingMockDataTree(); + final TestActorRef shard = actorFactory.createTestActor( + newShardBuilder().dataTree(dataTree).props().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testImmediateCommitWithCanCommitPhaseFailure-" + readWrite); - shard.tell(new CanCommitTransaction(transactionID1).toSerializable(), getRef()); - CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( - expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); - assertEquals("Can commit", true, canCommitReply.getCanCommit()); + waitUntilLeader(shard); - // Send the CanCommitTransaction message for the 2nd Tx. This should get queued and - // processed after the first Tx completes. + doThrow(new DataValidationFailedException(YangInstanceIdentifier.EMPTY, "mock canCommit failure")) + .doNothing().when(dataTree).validate(any(DataTreeModification.class)); - Future canCommitFuture = Patterns.ask(shard, - new CanCommitTransaction(transactionID2).toSerializable(), timeout); + final FiniteDuration duration = duration("5 seconds"); - // Send the AbortTransaction message for the first Tx. This should trigger the 2nd - // Tx to proceed. + final TransactionIdentifier transactionID1 = nextTransactionId(); - shard.tell(new AbortTransaction(transactionID1).toSerializable(), getRef()); - expectMsgClass(duration, AbortTransactionReply.SERIALIZABLE_CLASS); + if (readWrite) { + shard.tell(prepareForwardedReadyTransaction(shard, transactionID1, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true), getRef()); + } else { + shard.tell(prepareBatchedModifications(transactionID1, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true), getRef()); + } - // Wait for the 2nd Tx to complete the canCommit phase. + expectMsgClass(duration, akka.actor.Status.Failure.class); - Await.ready(canCommitFuture, duration); + // Send another can commit to ensure the failed one got cleaned + // up. - InOrder inOrder = inOrder(cohort1, cohort2); - inOrder.verify(cohort1).canCommit(); - inOrder.verify(cohort2).canCommit(); + final TransactionIdentifier transactionID2 = nextTransactionId(); + if (readWrite) { + shard.tell(prepareForwardedReadyTransaction(shard, transactionID2, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true), getRef()); + } else { + shard.tell(prepareBatchedModifications(transactionID2, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true), getRef()); + } - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); - }}; + expectMsgClass(duration, CommitTransactionReply.class); + } + }; } @Test - public void testCreateSnapshot() throws Exception { - testCreateSnapshot(true, "testCreateSnapshot"); + public void testAbortWithCommitPending() throws Exception { + new ShardTestKit(getSystem()) { + { + final Creator creator = () -> new Shard(newShardBuilder()) { + @Override + void persistPayload(final TransactionIdentifier transactionId, final Payload payload) { + // Simulate an AbortTransaction message occurring during + // replication, after + // persisting and before finishing the commit to the + // in-memory store. + + doAbortTransaction(transactionId, null); + super.persistPayload(transactionId, payload); + } + }; + + final TestActorRef shard = actorFactory.createTestActor(Props + .create(new DelegatingShardCreator(creator)).withDispatcher(Dispatchers.DefaultDispatcherId()), + "testAbortWithCommitPending"); + + waitUntilLeader(shard); + + final FiniteDuration duration = duration("5 seconds"); + + final TransactionIdentifier transactionID = nextTransactionId(); + + shard.tell(prepareBatchedModifications(transactionID, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), false), getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); + + shard.tell(new CanCommitTransaction(transactionID, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, CanCommitTransactionReply.class); + + shard.tell(new CommitTransaction(transactionID, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, CommitTransactionReply.class); + + final NormalizedNode node = readStore(shard, TestModel.TEST_PATH); + + // Since we're simulating an abort occurring during replication + // and before finish commit, + // the data should still get written to the in-memory store + // since we've gotten past + // canCommit and preCommit and persisted the data. + assertNotNull(TestModel.TEST_QNAME.getLocalName() + " not found", node); + } + }; + } + + @Test + public void testTransactionCommitTimeout() throws Exception { + dataStoreContextBuilder.shardTransactionCommitTimeoutInSeconds(1); + new ShardTestKit(getSystem()) { + { + final TestActorRef shard = actorFactory.createTestActor( + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testTransactionCommitTimeout"); + + waitUntilLeader(shard); + + final FiniteDuration duration = duration("5 seconds"); + + writeToStore(shard, TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); + writeToStore(shard, TestModel.OUTER_LIST_PATH, + ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build()); + + // Ready 2 Tx's - the first will timeout + + final TransactionIdentifier transactionID1 = nextTransactionId(); + shard.tell( + prepareBatchedModifications(transactionID1, + YangInstanceIdentifier.builder(TestModel.OUTER_LIST_PATH) + .nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1).build(), + ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1), false), + getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); + + final TransactionIdentifier transactionID2 = nextTransactionId(); + final YangInstanceIdentifier listNodePath = YangInstanceIdentifier.builder(TestModel.OUTER_LIST_PATH) + .nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 2).build(); + shard.tell( + prepareBatchedModifications(transactionID2, listNodePath, + ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 2), false), + getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); + + // canCommit 1st Tx. We don't send the commit so it should + // timeout. + + shard.tell(new CanCommitTransaction(transactionID1, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, CanCommitTransactionReply.class); + + // canCommit the 2nd Tx - it should complete after the 1st Tx + // times out. + + shard.tell(new CanCommitTransaction(transactionID2, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, CanCommitTransactionReply.class); + + // Try to commit the 1st Tx - should fail as it's not the + // current Tx. + + shard.tell(new CommitTransaction(transactionID1, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, akka.actor.Status.Failure.class); + + // Commit the 2nd Tx. + + shard.tell(new CommitTransaction(transactionID2, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, CommitTransactionReply.class); + + final NormalizedNode node = readStore(shard, listNodePath); + assertNotNull(listNodePath + " not found", node); + } + }; + } + +// @Test +// @Ignore +// public void testTransactionCommitQueueCapacityExceeded() throws Throwable { +// dataStoreContextBuilder.shardTransactionCommitQueueCapacity(2); +// +// new ShardTestKit(getSystem()) {{ +// final TestActorRef shard = actorFactory.createTestActor( +// newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), +// "testTransactionCommitQueueCapacityExceeded"); +// +// waitUntilLeader(shard); +// +// final FiniteDuration duration = duration("5 seconds"); +// +// final ShardDataTree dataStore = shard.underlyingActor().getDataStore(); +// +// final TransactionIdentifier transactionID1 = nextTransactionId(); +// final MutableCompositeModification modification1 = new MutableCompositeModification(); +// final ShardDataTreeCohort cohort1 = setupMockWriteTransaction("cohort1", dataStore, +// TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME), transactionID1, +// modification1); +// +// final TransactionIdentifier transactionID2 = nextTransactionId(); +// final MutableCompositeModification modification2 = new MutableCompositeModification(); +// final ShardDataTreeCohort cohort2 = setupMockWriteTransaction("cohort2", dataStore, +// TestModel.OUTER_LIST_PATH, +// ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build(), transactionID2, +// modification2); +// +// final TransactionIdentifier transactionID3 = nextTransactionId(); +// final MutableCompositeModification modification3 = new MutableCompositeModification(); +// final ShardDataTreeCohort cohort3 = setupMockWriteTransaction("cohort3", dataStore, +// TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME), transactionID3, +// modification3); +// +// // Ready the Tx's +// +// shard.tell(prepareReadyTransactionMessage(false, shard.underlyingActor(), cohort1, transactionID1, +// modification1), getRef()); +// expectMsgClass(duration, ReadyTransactionReply.class); +// +// shard.tell(prepareReadyTransactionMessage(false, shard.underlyingActor(), cohort2, transactionID2, +// modification2), getRef()); +// expectMsgClass(duration, ReadyTransactionReply.class); +// +// // The 3rd Tx should exceed queue capacity and fail. +// +// shard.tell(prepareReadyTransactionMessage(false, shard.underlyingActor(), cohort3, transactionID3, +// modification3), getRef()); +// expectMsgClass(duration, akka.actor.Status.Failure.class); +// +// // canCommit 1st Tx. +// +// shard.tell(new CanCommitTransaction(transactionID1, CURRENT_VERSION).toSerializable(), getRef()); +// expectMsgClass(duration, CanCommitTransactionReply.class); +// +// // canCommit the 2nd Tx - it should get queued. +// +// shard.tell(new CanCommitTransaction(transactionID2, CURRENT_VERSION).toSerializable(), getRef()); +// +// // canCommit the 3rd Tx - should exceed queue capacity and fail. +// +// shard.tell(new CanCommitTransaction(transactionID3, CURRENT_VERSION).toSerializable(), getRef()); +// expectMsgClass(duration, akka.actor.Status.Failure.class); +// }}; +// } + + @Test + public void testTransactionCommitWithPriorExpiredCohortEntries() throws Exception { + dataStoreContextBuilder.shardTransactionCommitTimeoutInSeconds(1); + new ShardTestKit(getSystem()) { + { + final TestActorRef shard = actorFactory.createTestActor( + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testTransactionCommitWithPriorExpiredCohortEntries"); + + waitUntilLeader(shard); + + final FiniteDuration duration = duration("5 seconds"); + + final TransactionIdentifier transactionID1 = nextTransactionId(); + shard.tell(newBatchedModifications(transactionID1, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true, false, 1), getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); + + final TransactionIdentifier transactionID2 = nextTransactionId(); + shard.tell(newBatchedModifications(transactionID2, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true, false, 1), getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); + + final TransactionIdentifier transactionID3 = nextTransactionId(); + shard.tell(newBatchedModifications(transactionID3, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true, false, 1), getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); + + // All Tx's are readied. We'll send canCommit for the last one + // but not the others. The others + // should expire from the queue and the last one should be + // processed. + + shard.tell(new CanCommitTransaction(transactionID3, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, CanCommitTransactionReply.class); + } + }; + } + + @Test + public void testTransactionCommitWithSubsequentExpiredCohortEntry() throws Exception { + dataStoreContextBuilder.shardTransactionCommitTimeoutInSeconds(1); + new ShardTestKit(getSystem()) { + { + final TestActorRef shard = actorFactory.createTestActor( + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testTransactionCommitWithSubsequentExpiredCohortEntry"); + + waitUntilLeader(shard); + + final FiniteDuration duration = duration("5 seconds"); + + final ShardDataTree dataStore = shard.underlyingActor().getDataStore(); + + final TransactionIdentifier transactionID1 = nextTransactionId(); + shard.tell(prepareBatchedModifications(transactionID1, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), false), getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); + + // CanCommit the first Tx so it's the current in-progress Tx. + + shard.tell(new CanCommitTransaction(transactionID1, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, CanCommitTransactionReply.class); + + // Ready the second Tx. + + final TransactionIdentifier transactionID2 = nextTransactionId(); + shard.tell(prepareBatchedModifications(transactionID2, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), false), getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); + + // Ready the third Tx. + + final TransactionIdentifier transactionID3 = nextTransactionId(); + final DataTreeModification modification3 = dataStore.newModification(); + new WriteModification(TestModel.TEST2_PATH, ImmutableNodes.containerNode(TestModel.TEST2_QNAME)) + .apply(modification3); + modification3.ready(); + final ReadyLocalTransaction readyMessage = new ReadyLocalTransaction(transactionID3, modification3, + true); + shard.tell(readyMessage, getRef()); + + // Commit the first Tx. After completing, the second should + // expire from the queue and the third + // Tx committed. + + shard.tell(new CommitTransaction(transactionID1, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, CommitTransactionReply.class); + + // Expect commit reply from the third Tx. + + expectMsgClass(duration, CommitTransactionReply.class); + + final NormalizedNode node = readStore(shard, TestModel.TEST2_PATH); + assertNotNull(TestModel.TEST2_PATH + " not found", node); + } + }; + } + + @Test + public void testCanCommitBeforeReadyFailure() throws Exception { + new ShardTestKit(getSystem()) { + { + final TestActorRef shard = actorFactory.createTestActor( + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testCanCommitBeforeReadyFailure"); + + shard.tell(new CanCommitTransaction(nextTransactionId(), CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration("5 seconds"), akka.actor.Status.Failure.class); + } + }; + } + + @Test + public void testAbortAfterCanCommit() throws Exception { + new ShardTestKit(getSystem()) { + { + final TestActorRef shard = actorFactory.createTestActor( + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), "testAbortAfterCanCommit"); + + waitUntilLeader(shard); + + final FiniteDuration duration = duration("5 seconds"); + final Timeout timeout = new Timeout(duration); + + // Ready 2 transactions - the first one will be aborted. + + final TransactionIdentifier transactionID1 = nextTransactionId(); + shard.tell(newBatchedModifications(transactionID1, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true, false, 1), getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); + + final TransactionIdentifier transactionID2 = nextTransactionId(); + shard.tell(newBatchedModifications(transactionID2, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true, false, 1), getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); + + // Send the CanCommitTransaction message for the first Tx. + + shard.tell(new CanCommitTransaction(transactionID1, CURRENT_VERSION).toSerializable(), getRef()); + CanCommitTransactionReply canCommitReply = CanCommitTransactionReply + .fromSerializable(expectMsgClass(duration, CanCommitTransactionReply.class)); + assertEquals("Can commit", true, canCommitReply.getCanCommit()); + + // Send the CanCommitTransaction message for the 2nd Tx. This + // should get queued and + // processed after the first Tx completes. + + final Future canCommitFuture = Patterns.ask(shard, + new CanCommitTransaction(transactionID2, CURRENT_VERSION).toSerializable(), timeout); + + // Send the AbortTransaction message for the first Tx. This + // should trigger the 2nd + // Tx to proceed. + + shard.tell(new AbortTransaction(transactionID1, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, AbortTransactionReply.class); + + // Wait for the 2nd Tx to complete the canCommit phase. + + canCommitReply = (CanCommitTransactionReply) Await.result(canCommitFuture, duration); + assertEquals("Can commit", true, canCommitReply.getCanCommit()); + } + }; + } + + @Test + public void testAbortAfterReady() throws Exception { + dataStoreContextBuilder.shardTransactionCommitTimeoutInSeconds(1); + new ShardTestKit(getSystem()) { + { + final TestActorRef shard = actorFactory.createTestActor( + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), "testAbortAfterReady"); + + waitUntilLeader(shard); + + final FiniteDuration duration = duration("5 seconds"); + + // Ready a tx. + + final TransactionIdentifier transactionID1 = nextTransactionId(); + shard.tell(newBatchedModifications(transactionID1, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true, false, 1), getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); + + // Send the AbortTransaction message. + + shard.tell(new AbortTransaction(transactionID1, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, AbortTransactionReply.class); + + assertEquals("getPendingTxCommitQueueSize", 0, shard.underlyingActor().getPendingTxCommitQueueSize()); + + // Now send CanCommitTransaction - should fail. + + shard.tell(new CanCommitTransaction(transactionID1, CURRENT_VERSION).toSerializable(), getRef()); + final Throwable failure = expectMsgClass(duration, akka.actor.Status.Failure.class).cause(); + assertTrue("Failure type", failure instanceof IllegalStateException); + + // Ready and CanCommit another and verify success. + + final TransactionIdentifier transactionID2 = nextTransactionId(); + shard.tell(newBatchedModifications(transactionID2, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true, false, 1), getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); + + shard.tell(new CanCommitTransaction(transactionID2, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, CanCommitTransactionReply.class); + } + }; + } + + @Test + public void testAbortQueuedTransaction() throws Exception { + new ShardTestKit(getSystem()) { + { + final TestActorRef shard = actorFactory.createTestActor( + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), "testAbortAfterReady"); + + waitUntilLeader(shard); + + final FiniteDuration duration = duration("5 seconds"); + + // Ready 3 tx's. + + final TransactionIdentifier transactionID1 = nextTransactionId(); + shard.tell(newBatchedModifications(transactionID1, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true, false, 1), getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); + + final TransactionIdentifier transactionID2 = nextTransactionId(); + shard.tell(newBatchedModifications(transactionID2, TestModel.TEST_PATH, + ImmutableNodes.containerNode(TestModel.TEST_QNAME), true, false, 1), getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); + + final TransactionIdentifier transactionID3 = nextTransactionId(); + shard.tell( + newBatchedModifications(transactionID3, TestModel.OUTER_LIST_PATH, + ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build(), true, false, 1), + getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); + + // Abort the second tx while it's queued. + + shard.tell(new AbortTransaction(transactionID2, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, AbortTransactionReply.class); + + // Commit the other 2. + + shard.tell(new CanCommitTransaction(transactionID1, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, CanCommitTransactionReply.class); + + shard.tell(new CommitTransaction(transactionID1, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, CommitTransactionReply.class); + + shard.tell(new CanCommitTransaction(transactionID3, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, CanCommitTransactionReply.class); + + shard.tell(new CommitTransaction(transactionID3, CURRENT_VERSION).toSerializable(), getRef()); + expectMsgClass(duration, CommitTransactionReply.class); + + assertEquals("getPendingTxCommitQueueSize", 0, shard.underlyingActor().getPendingTxCommitQueueSize()); + } + }; } @Test @@ -1408,244 +1963,408 @@ public class ShardTest extends AbstractShardTest { testCreateSnapshot(false, "testCreateSnapshotWithNonPersistentData"); } - @SuppressWarnings("serial") - public void testCreateSnapshot(final boolean persistent, final String shardActorName) throws Exception{ + @Test + public void testCreateSnapshot() throws Exception { + testCreateSnapshot(true, "testCreateSnapshot"); + } + private void testCreateSnapshot(final boolean persistent, final String shardActorName) throws Exception { final AtomicReference latch = new AtomicReference<>(new CountDownLatch(1)); final AtomicReference savedSnapshot = new AtomicReference<>(); class TestPersistentDataProvider extends DelegatingPersistentDataProvider { - TestPersistentDataProvider(DataPersistenceProvider delegate) { + TestPersistentDataProvider(final DataPersistenceProvider delegate) { super(delegate); } @Override - public void saveSnapshot(Object o) { - savedSnapshot.set(o); - super.saveSnapshot(o); + public void saveSnapshot(final Object obj) { + savedSnapshot.set(obj); + super.saveSnapshot(obj); } } dataStoreContextBuilder.persistent(persistent); - new ShardTestKit(getSystem()) {{ - class TestShard extends Shard { + class TestShard extends Shard { - protected TestShard(ShardIdentifier name, Map peerAddresses, - DatastoreContext datastoreContext, SchemaContext schemaContext) { - super(name, peerAddresses, datastoreContext, schemaContext); - setPersistence(new TestPersistentDataProvider(super.persistence())); - } - - @Override - public void handleCommand(Object message) { - super.handleCommand(message); + protected TestShard(final AbstractBuilder builder) { + super(builder); + setPersistence(new TestPersistentDataProvider(super.persistence())); + } - if (message instanceof SaveSnapshotSuccess || message.equals("commit_snapshot")) { - latch.get().countDown(); - } - } + @Override + public void handleCommand(final Object message) { + super.handleCommand(message); - @Override - public RaftActorContext getRaftActorContext() { - return super.getRaftActorContext(); + // XXX: commit_snapshot equality check references RaftActorSnapshotMessageSupport.COMMIT_SNAPSHOT + if (message instanceof SaveSnapshotSuccess || "commit_snapshot".equals(message.toString())) { + latch.get().countDown(); } } - Creator creator = new Creator() { - @Override - public Shard create() throws Exception { - return new TestShard(shardID, Collections.emptyMap(), - newDatastoreContext(), SCHEMA_CONTEXT); - } - }; - - TestActorRef shard = TestActorRef.create(getSystem(), - Props.create(new DelegatingShardCreator(creator)), shardActorName); - - waitUntilLeader(shard); - - writeToStore(shard, TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); - - NormalizedNode expectedRoot = readStore(shard, YangInstanceIdentifier.builder().build()); - - // Trigger creation of a snapshot by ensuring - RaftActorContext raftActorContext = ((TestShard) shard.underlyingActor()).getRaftActorContext(); - raftActorContext.getSnapshotManager().capture(mock(ReplicatedLogEntry.class), -1); - - assertEquals("Snapshot saved", true, latch.get().await(5, TimeUnit.SECONDS)); + @Override + public RaftActorContext getRaftActorContext() { + return super.getRaftActorContext(); + } + } - assertTrue("Invalid saved snapshot " + savedSnapshot.get(), - savedSnapshot.get() instanceof Snapshot); + new ShardTestKit(getSystem()) { + { + final Creator creator = () -> new TestShard(newShardBuilder()); - verifySnapshot((Snapshot)savedSnapshot.get(), expectedRoot); + final TestActorRef shard = actorFactory.createTestActor(Props + .create(new DelegatingShardCreator(creator)).withDispatcher(Dispatchers.DefaultDispatcherId()), + shardActorName); - latch.set(new CountDownLatch(1)); - savedSnapshot.set(null); + waitUntilLeader(shard); + writeToStore(shard, TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); - raftActorContext.getSnapshotManager().capture(mock(ReplicatedLogEntry.class), -1); + final NormalizedNode expectedRoot = readStore(shard, YangInstanceIdentifier.EMPTY); - assertEquals("Snapshot saved", true, latch.get().await(5, TimeUnit.SECONDS)); + // Trigger creation of a snapshot by ensuring + final RaftActorContext raftActorContext = ((TestShard) shard.underlyingActor()).getRaftActorContext(); + raftActorContext.getSnapshotManager().capture(mock(ReplicatedLogEntry.class), -1); + awaitAndValidateSnapshot(expectedRoot); - assertTrue("Invalid saved snapshot " + savedSnapshot.get(), - savedSnapshot.get() instanceof Snapshot); + raftActorContext.getSnapshotManager().capture(mock(ReplicatedLogEntry.class), -1); + awaitAndValidateSnapshot(expectedRoot); + } - verifySnapshot((Snapshot)savedSnapshot.get(), expectedRoot); + private void awaitAndValidateSnapshot(final NormalizedNode expectedRoot) + throws InterruptedException, IOException { + assertEquals("Snapshot saved", true, latch.get().await(5, TimeUnit.SECONDS)); - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); - } + assertTrue("Invalid saved snapshot " + savedSnapshot.get(), savedSnapshot.get() instanceof Snapshot); - private void verifySnapshot(Snapshot snapshot, NormalizedNode expectedRoot) { + verifySnapshot((Snapshot) savedSnapshot.get(), expectedRoot); - NormalizedNode actual = SerializationUtils.deserializeNormalizedNode(snapshot.getState()); - assertEquals("Root node", expectedRoot, actual); + latch.set(new CountDownLatch(1)); + savedSnapshot.set(null); + } - }}; + private void verifySnapshot(final Snapshot snapshot, final NormalizedNode expectedRoot) + throws IOException { + final NormalizedNode actual = ShardDataTreeSnapshot.deserialize(snapshot.getState()).getRootNode() + .get(); + assertEquals("Root node", expectedRoot, actual); + } + }; } /** - * This test simply verifies that the applySnapShot logic will work - * @throws ReadFailedException + * This test simply verifies that the applySnapShot logic will work. */ @Test - public void testInMemoryDataStoreRestore() throws ReadFailedException { - InMemoryDOMDataStore store = new InMemoryDOMDataStore("test", MoreExecutors.sameThreadExecutor()); + public void testInMemoryDataTreeRestore() throws ReadFailedException, DataValidationFailedException { + final DataTree store = InMemoryDataTreeFactory.getInstance().create(TreeType.OPERATIONAL); + store.setSchemaContext(SCHEMA_CONTEXT); - store.onGlobalContextUpdated(SCHEMA_CONTEXT); - - DOMStoreWriteTransaction putTransaction = store.newWriteOnlyTransaction(); + final DataTreeModification putTransaction = store.takeSnapshot().newModification(); putTransaction.write(TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); - commitTransaction(putTransaction); + commitTransaction(store, putTransaction); - NormalizedNode expected = readStore(store); + final NormalizedNode expected = readStore(store, YangInstanceIdentifier.EMPTY); - DOMStoreWriteTransaction writeTransaction = store.newWriteOnlyTransaction(); + final DataTreeModification writeTransaction = store.takeSnapshot().newModification(); - writeTransaction.delete(YangInstanceIdentifier.builder().build()); - writeTransaction.write(YangInstanceIdentifier.builder().build(), expected); + writeTransaction.delete(YangInstanceIdentifier.EMPTY); + writeTransaction.write(YangInstanceIdentifier.EMPTY, expected); - commitTransaction(writeTransaction); + commitTransaction(store, writeTransaction); - NormalizedNode actual = readStore(store); + final NormalizedNode actual = readStore(store, YangInstanceIdentifier.EMPTY); assertEquals(expected, actual); } @Test - public void testRecoveryApplicable(){ - - final DatastoreContext persistentContext = DatastoreContext.newBuilder(). - shardJournalRecoveryLogBatchSize(3).shardSnapshotBatchCount(5000).persistent(true).build(); - - final Props persistentProps = Shard.props(shardID, Collections.emptyMap(), - persistentContext, SCHEMA_CONTEXT); + public void testRecoveryApplicable() { - final DatastoreContext nonPersistentContext = DatastoreContext.newBuilder(). - shardJournalRecoveryLogBatchSize(3).shardSnapshotBatchCount(5000).persistent(false).build(); + final DatastoreContext persistentContext = DatastoreContext.newBuilder() + .shardJournalRecoveryLogBatchSize(3).shardSnapshotBatchCount(5000).persistent(true).build(); - final Props nonPersistentProps = Shard.props(shardID, Collections.emptyMap(), - nonPersistentContext, SCHEMA_CONTEXT); + final Props persistentProps = Shard.builder().id(shardID).datastoreContext(persistentContext) + .schemaContext(SCHEMA_CONTEXT).props(); - new ShardTestKit(getSystem()) {{ - TestActorRef shard1 = TestActorRef.create(getSystem(), - persistentProps, "testPersistence1"); + final DatastoreContext nonPersistentContext = DatastoreContext.newBuilder() + .shardJournalRecoveryLogBatchSize(3).shardSnapshotBatchCount(5000).persistent(false).build(); - assertTrue("Recovery Applicable", shard1.underlyingActor().persistence().isRecoveryApplicable()); + final Props nonPersistentProps = Shard.builder().id(shardID).datastoreContext(nonPersistentContext) + .schemaContext(SCHEMA_CONTEXT).props(); - shard1.tell(PoisonPill.getInstance(), ActorRef.noSender()); - - TestActorRef shard2 = TestActorRef.create(getSystem(), - nonPersistentProps, "testPersistence2"); - - assertFalse("Recovery Not Applicable", shard2.underlyingActor().persistence().isRecoveryApplicable()); + new ShardTestKit(getSystem()) { + { + final TestActorRef shard1 = actorFactory.createTestActor(persistentProps, "testPersistence1"); - shard2.tell(PoisonPill.getInstance(), ActorRef.noSender()); + assertTrue("Recovery Applicable", shard1.underlyingActor().persistence().isRecoveryApplicable()); - }}; + final TestActorRef shard2 = actorFactory.createTestActor(nonPersistentProps, "testPersistence2"); + assertFalse("Recovery Not Applicable", shard2.underlyingActor().persistence().isRecoveryApplicable()); + } + }; } @Test public void testOnDatastoreContext() { - new ShardTestKit(getSystem()) {{ - dataStoreContextBuilder.persistent(true); - - TestActorRef shard = TestActorRef.create(getSystem(), newShardProps(), "testOnDatastoreContext"); + new ShardTestKit(getSystem()) { + { + dataStoreContextBuilder.persistent(true); - assertEquals("isRecoveryApplicable", true, - shard.underlyingActor().persistence().isRecoveryApplicable()); + final TestActorRef shard = actorFactory.createTestActor(newShardProps(), + "testOnDatastoreContext"); - waitUntilLeader(shard); + assertEquals("isRecoveryApplicable", true, + shard.underlyingActor().persistence().isRecoveryApplicable()); - shard.tell(dataStoreContextBuilder.persistent(false).build(), ActorRef.noSender()); + waitUntilLeader(shard); - assertEquals("isRecoveryApplicable", false, - shard.underlyingActor().persistence().isRecoveryApplicable()); + shard.tell(dataStoreContextBuilder.persistent(false).build(), ActorRef.noSender()); - shard.tell(dataStoreContextBuilder.persistent(true).build(), ActorRef.noSender()); + assertEquals("isRecoveryApplicable", false, + shard.underlyingActor().persistence().isRecoveryApplicable()); - assertEquals("isRecoveryApplicable", true, - shard.underlyingActor().persistence().isRecoveryApplicable()); + shard.tell(dataStoreContextBuilder.persistent(true).build(), ActorRef.noSender()); - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); - }}; + assertEquals("isRecoveryApplicable", true, + shard.underlyingActor().persistence().isRecoveryApplicable()); + } + }; } @Test public void testRegisterRoleChangeListener() throws Exception { new ShardTestKit(getSystem()) { { - final TestActorRef shard = TestActorRef.create(getSystem(), + final TestActorRef shard = actorFactory.createTestActor( newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), "testRegisterRoleChangeListener"); waitUntilLeader(shard); - TestActorRef listener = + final TestActorRef listener = TestActorRef.create(getSystem(), Props.create(MessageCollectorActor.class)); shard.tell(new RegisterRoleChangeListener(), listener); - // TODO: MessageCollectorActor exists as a test util in both the akka-raft and distributed-datastore - // projects. Need to move it to commons as a regular utility and then we can get rid of this arbitrary - // sleep. - Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS); + MessageCollectorActor.expectFirstMatching(listener, RegisterRoleChangeListenerReply.class); + + ShardLeaderStateChanged leaderStateChanged = MessageCollectorActor.expectFirstMatching(listener, + ShardLeaderStateChanged.class); + assertEquals("getLocalShardDataTree present", true, + leaderStateChanged.getLocalShardDataTree().isPresent()); + assertSame("getLocalShardDataTree", shard.underlyingActor().getDataStore().getDataTree(), + leaderStateChanged.getLocalShardDataTree().get()); + + MessageCollectorActor.clearMessages(listener); + + // Force a leader change - List allMatching = MessageCollectorActor.getAllMatching(listener, RegisterRoleChangeListenerReply.class); + shard.tell(new RequestVote(10000, "member2", 50, 50), getRef()); - assertEquals(1, allMatching.size()); + leaderStateChanged = MessageCollectorActor.expectFirstMatching(listener, + ShardLeaderStateChanged.class); + assertEquals("getLocalShardDataTree present", false, + leaderStateChanged.getLocalShardDataTree().isPresent()); } }; } @Test public void testFollowerInitialSyncStatus() throws Exception { - final TestActorRef shard = TestActorRef.create(getSystem(), + final TestActorRef shard = actorFactory.createTestActor( newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), "testFollowerInitialSyncStatus"); - shard.underlyingActor().onReceiveCommand(new FollowerInitialSyncUpStatus(false, "member-1-shard-inventory-operational")); + shard.underlyingActor().handleNonRaftCommand(new FollowerInitialSyncUpStatus(false, + "member-1-shard-inventory-operational")); assertEquals(false, shard.underlyingActor().getShardMBean().getFollowerInitialSyncStatus()); - shard.underlyingActor().onReceiveCommand(new FollowerInitialSyncUpStatus(true, "member-1-shard-inventory-operational")); + shard.underlyingActor().handleNonRaftCommand(new FollowerInitialSyncUpStatus(true, + "member-1-shard-inventory-operational")); assertEquals(true, shard.underlyingActor().getShardMBean().getFollowerInitialSyncStatus()); + } + + @Test + public void testClusteredDataChangeListenerDelayedRegistration() throws Exception { + new ShardTestKit(getSystem()) { + { + final String testName = "testClusteredDataChangeListenerDelayedRegistration"; + dataStoreContextBuilder.shardElectionTimeoutFactor(1000) + .customRaftPolicyImplementation(DisableElectionsRaftPolicy.class.getName()); + + final MockDataChangeListener listener = new MockDataChangeListener(1); + final ActorRef dclActor = actorFactory.createActor(DataChangeListener.props(listener), + actorFactory.generateActorId(testName + "-DataChangeListener")); + + setupInMemorySnapshotStore(); + + final TestActorRef shard = actorFactory.createTestActor( + newShardBuilder().props().withDispatcher(Dispatchers.DefaultDispatcherId()), + actorFactory.generateActorId(testName + "-shard")); + + waitUntilNoLeader(shard); + + final YangInstanceIdentifier path = TestModel.TEST_PATH; + + shard.tell(new RegisterChangeListener(path, dclActor, AsyncDataBroker.DataChangeScope.BASE, true), + getRef()); + final RegisterChangeListenerReply reply = expectMsgClass(duration("5 seconds"), + RegisterChangeListenerReply.class); + assertNotNull("getListenerRegistrationPath", reply.getListenerRegistrationPath()); + + shard.tell(DatastoreContext.newBuilderFrom(dataStoreContextBuilder.build()) + .customRaftPolicyImplementation(null).build(), ActorRef.noSender()); - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); + listener.waitForChangeEvents(); + } + }; } - private void commitTransaction(final DOMStoreWriteTransaction transaction) { - DOMStoreThreePhaseCommitCohort commitCohort = transaction.ready(); - ListenableFuture future = - commitCohort.preCommit(); - try { - future.get(); - future = commitCohort.commit(); - future.get(); - } catch (InterruptedException | ExecutionException e) { - } + @Test + public void testClusteredDataChangeListenerRegistration() throws Exception { + new ShardTestKit(getSystem()) { + { + final String testName = "testClusteredDataChangeListenerRegistration"; + final ShardIdentifier followerShardID = ShardIdentifier.create("inventory", + MemberName.forName(actorFactory.generateActorId(testName + "-follower")), "config"); + + final ShardIdentifier leaderShardID = ShardIdentifier.create("inventory", + MemberName.forName(actorFactory.generateActorId(testName + "-leader")), "config"); + + final TestActorRef followerShard = actorFactory + .createTestActor(Shard.builder().id(followerShardID) + .datastoreContext(dataStoreContextBuilder.shardElectionTimeoutFactor(1000).build()) + .peerAddresses(Collections.singletonMap(leaderShardID.toString(), + "akka://test/user/" + leaderShardID.toString())) + .schemaContext(SCHEMA_CONTEXT).props() + .withDispatcher(Dispatchers.DefaultDispatcherId()), followerShardID.toString()); + + final TestActorRef leaderShard = actorFactory + .createTestActor(Shard.builder().id(leaderShardID).datastoreContext(newDatastoreContext()) + .peerAddresses(Collections.singletonMap(followerShardID.toString(), + "akka://test/user/" + followerShardID.toString())) + .schemaContext(SCHEMA_CONTEXT).props() + .withDispatcher(Dispatchers.DefaultDispatcherId()), leaderShardID.toString()); + + leaderShard.tell(TimeoutNow.INSTANCE, ActorRef.noSender()); + final String leaderPath = waitUntilLeader(followerShard); + assertEquals("Shard leader path", leaderShard.path().toString(), leaderPath); + + final YangInstanceIdentifier path = TestModel.TEST_PATH; + final MockDataChangeListener listener = new MockDataChangeListener(1); + final ActorRef dclActor = actorFactory.createActor(DataChangeListener.props(listener), + actorFactory.generateActorId(testName + "-DataChangeListener")); + + followerShard.tell( + new RegisterChangeListener(path, dclActor, AsyncDataBroker.DataChangeScope.BASE, true), + getRef()); + final RegisterChangeListenerReply reply = expectMsgClass(duration("5 seconds"), + RegisterChangeListenerReply.class); + assertNotNull("getListenerRegistratioznPath", reply.getListenerRegistrationPath()); + + writeToStore(followerShard, path, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); + + listener.waitForChangeEvents(); + } + }; + } + + @Test + public void testClusteredDataTreeChangeListenerDelayedRegistration() throws Exception { + new ShardTestKit(getSystem()) { + { + final String testName = "testClusteredDataTreeChangeListenerDelayedRegistration"; + dataStoreContextBuilder.shardElectionTimeoutFactor(1000) + .customRaftPolicyImplementation(DisableElectionsRaftPolicy.class.getName()); + + final MockDataTreeChangeListener listener = new MockDataTreeChangeListener(1); + final ActorRef dclActor = actorFactory.createActor(DataTreeChangeListenerActor.props(listener), + actorFactory.generateActorId(testName + "-DataTreeChangeListener")); + + setupInMemorySnapshotStore(); + + final TestActorRef shard = actorFactory.createTestActor( + newShardBuilder().props().withDispatcher(Dispatchers.DefaultDispatcherId()), + actorFactory.generateActorId(testName + "-shard")); + + waitUntilNoLeader(shard); + + shard.tell(new RegisterDataTreeChangeListener(TestModel.TEST_PATH, dclActor, true), getRef()); + final RegisterDataTreeChangeListenerReply reply = expectMsgClass(duration("5 seconds"), + RegisterDataTreeChangeListenerReply.class); + assertNotNull("getListenerRegistrationPath", reply.getListenerRegistrationPath()); + + shard.tell(DatastoreContext.newBuilderFrom(dataStoreContextBuilder.build()) + .customRaftPolicyImplementation(null).build(), ActorRef.noSender()); + + listener.waitForChangeEvents(); + } + }; + } + + @Test + public void testClusteredDataTreeChangeListenerRegistration() throws Exception { + new ShardTestKit(getSystem()) { + { + final String testName = "testClusteredDataTreeChangeListenerRegistration"; + final ShardIdentifier followerShardID = ShardIdentifier.create("inventory", + MemberName.forName(actorFactory.generateActorId(testName + "-follower")), "config"); + + final ShardIdentifier leaderShardID = ShardIdentifier.create("inventory", + MemberName.forName(actorFactory.generateActorId(testName + "-leader")), "config"); + + final TestActorRef followerShard = actorFactory + .createTestActor(Shard.builder().id(followerShardID) + .datastoreContext(dataStoreContextBuilder.shardElectionTimeoutFactor(1000).build()) + .peerAddresses(Collections.singletonMap(leaderShardID.toString(), + "akka://test/user/" + leaderShardID.toString())) + .schemaContext(SCHEMA_CONTEXT).props() + .withDispatcher(Dispatchers.DefaultDispatcherId()), followerShardID.toString()); + + final TestActorRef leaderShard = actorFactory + .createTestActor(Shard.builder().id(leaderShardID).datastoreContext(newDatastoreContext()) + .peerAddresses(Collections.singletonMap(followerShardID.toString(), + "akka://test/user/" + followerShardID.toString())) + .schemaContext(SCHEMA_CONTEXT).props() + .withDispatcher(Dispatchers.DefaultDispatcherId()), leaderShardID.toString()); + + leaderShard.tell(TimeoutNow.INSTANCE, ActorRef.noSender()); + final String leaderPath = waitUntilLeader(followerShard); + assertEquals("Shard leader path", leaderShard.path().toString(), leaderPath); + + final YangInstanceIdentifier path = TestModel.TEST_PATH; + final MockDataTreeChangeListener listener = new MockDataTreeChangeListener(1); + final ActorRef dclActor = actorFactory.createActor(DataTreeChangeListenerActor.props(listener), + actorFactory.generateActorId(testName + "-DataTreeChangeListener")); + + followerShard.tell(new RegisterDataTreeChangeListener(TestModel.TEST_PATH, dclActor, true), getRef()); + final RegisterDataTreeChangeListenerReply reply = expectMsgClass(duration("5 seconds"), + RegisterDataTreeChangeListenerReply.class); + assertNotNull("getListenerRegistrationPath", reply.getListenerRegistrationPath()); + + writeToStore(followerShard, path, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); + + listener.waitForChangeEvents(); + } + }; + } + + @Test + public void testServerRemoved() throws Exception { + final TestActorRef parent = actorFactory.createTestActor(MessageCollectorActor.props()); + + final ActorRef shard = parent.underlyingActor().context().actorOf( + newShardBuilder().props().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testServerRemoved"); + + shard.tell(new ServerRemoved("test"), ActorRef.noSender()); + + MessageCollectorActor.expectFirstMatching(parent, ServerRemoved.class); } }