X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?p=controller.git;a=blobdiff_plain;f=opendaylight%2Fmd-sal%2Fsal-distributed-datastore%2Fsrc%2Ftest%2Fjava%2Forg%2Fopendaylight%2Fcontroller%2Fcluster%2Fdatastore%2FShardTest.java;h=0121a15338a2ed807e910a793e6655a062ba35de;hp=70869ee68a55644cf7aa8c5103031a72a881d703;hb=7ec296abdea94bbdc336276731ee545a2fb13e7c;hpb=376df7be9839100e31e6916d6e685dda9f8bd030 diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/ShardTest.java b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/ShardTest.java index 70869ee68a..0121a15338 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/ShardTest.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/ShardTest.java @@ -1,3 +1,11 @@ +/* + * Copyright (c) 2014, 2015 Cisco Systems, Inc. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ + package org.opendaylight.controller.cluster.datastore; import static org.junit.Assert.assertEquals; @@ -10,6 +18,7 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.verify; import static org.opendaylight.controller.cluster.datastore.DataStoreVersions.CURRENT_VERSION; import akka.actor.ActorRef; import akka.actor.ActorSelection; @@ -28,10 +37,8 @@ import com.google.common.base.Optional; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.Uninterruptibles; -import java.io.IOException; import java.util.Collections; import java.util.HashSet; -import java.util.Map; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -41,6 +48,7 @@ import org.junit.Test; import org.mockito.InOrder; import org.opendaylight.controller.cluster.DataPersistenceProvider; import org.opendaylight.controller.cluster.DelegatingPersistentDataProvider; +import org.opendaylight.controller.cluster.datastore.exceptions.NoShardLeaderException; import org.opendaylight.controller.cluster.datastore.identifiers.ShardIdentifier; import org.opendaylight.controller.cluster.datastore.jmx.mbeans.shard.ShardStats; import org.opendaylight.controller.cluster.datastore.messages.AbortTransaction; @@ -52,7 +60,6 @@ import org.opendaylight.controller.cluster.datastore.messages.CanCommitTransacti import org.opendaylight.controller.cluster.datastore.messages.CommitTransaction; import org.opendaylight.controller.cluster.datastore.messages.CommitTransactionReply; import org.opendaylight.controller.cluster.datastore.messages.CreateTransaction; -import org.opendaylight.controller.cluster.datastore.messages.ForwardedReadyTransaction; import org.opendaylight.controller.cluster.datastore.messages.PeerAddressResolved; import org.opendaylight.controller.cluster.datastore.messages.ReadData; import org.opendaylight.controller.cluster.datastore.messages.ReadDataReply; @@ -67,7 +74,6 @@ import org.opendaylight.controller.cluster.datastore.messages.UpdateSchemaContex import org.opendaylight.controller.cluster.datastore.modification.DeleteModification; import org.opendaylight.controller.cluster.datastore.modification.MergeModification; import org.opendaylight.controller.cluster.datastore.modification.Modification; -import org.opendaylight.controller.cluster.datastore.modification.ModificationPayload; import org.opendaylight.controller.cluster.datastore.modification.MutableCompositeModification; import org.opendaylight.controller.cluster.datastore.modification.WriteModification; import org.opendaylight.controller.cluster.datastore.utils.MockDataChangeListener; @@ -80,13 +86,14 @@ import org.opendaylight.controller.cluster.raft.ReplicatedLogEntry; import org.opendaylight.controller.cluster.raft.ReplicatedLogImplEntry; import org.opendaylight.controller.cluster.raft.Snapshot; import org.opendaylight.controller.cluster.raft.base.messages.ApplyJournalEntries; -import org.opendaylight.controller.cluster.raft.base.messages.ApplySnapshot; import org.opendaylight.controller.cluster.raft.base.messages.ApplyState; import org.opendaylight.controller.cluster.raft.base.messages.ElectionTimeout; import org.opendaylight.controller.cluster.raft.base.messages.FollowerInitialSyncUpStatus; import org.opendaylight.controller.cluster.raft.client.messages.FindLeader; import org.opendaylight.controller.cluster.raft.client.messages.FindLeaderReply; import org.opendaylight.controller.cluster.raft.messages.RequestVote; +import org.opendaylight.controller.cluster.raft.messages.ServerRemoved; +import org.opendaylight.controller.cluster.raft.policy.DisableElectionsRaftPolicy; import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal; import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore; import org.opendaylight.controller.cluster.raft.utils.MessageCollectorActor; @@ -108,13 +115,13 @@ import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTree; import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidate; import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidateNode; import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidateTip; -import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidates; import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeModification; import org.opendaylight.yangtools.yang.data.api.schema.tree.DataValidationFailedException; import org.opendaylight.yangtools.yang.data.api.schema.tree.ModificationType; +import org.opendaylight.yangtools.yang.data.api.schema.tree.TreeType; import org.opendaylight.yangtools.yang.data.impl.schema.ImmutableNodes; +import org.opendaylight.yangtools.yang.data.impl.schema.builder.impl.ImmutableContainerNodeBuilder; import org.opendaylight.yangtools.yang.data.impl.schema.tree.InMemoryDataTreeFactory; -import org.opendaylight.yangtools.yang.model.api.SchemaContext; import scala.concurrent.Await; import scala.concurrent.Future; import scala.concurrent.duration.FiniteDuration; @@ -124,53 +131,30 @@ public class ShardTest extends AbstractShardTest { private static final String DUMMY_DATA = "Dummy data as snapshot sequence number is set to 0 in InMemorySnapshotStore and journal recovery seq number will start from 1"; - final CountDownLatch recoveryComplete = new CountDownLatch(1); - - protected Props newShardPropsWithRecoveryComplete() { - - Creator creator = new Creator() { - @Override - public Shard create() throws Exception { - return new Shard(shardID, Collections.emptyMap(), - newDatastoreContext(), SCHEMA_CONTEXT) { - @Override - protected void onRecoveryComplete() { - try { - super.onRecoveryComplete(); - } finally { - recoveryComplete.countDown(); - } - } - }; - } - }; - return Props.create(new DelegatingShardCreator(creator)).withDispatcher(Dispatchers.DefaultDispatcherId()); - } - @Test public void testRegisterChangeListener() throws Exception { new ShardTestKit(getSystem()) {{ - TestActorRef shard = TestActorRef.create(getSystem(), + final TestActorRef shard = TestActorRef.create(getSystem(), newShardProps(), "testRegisterChangeListener"); waitUntilLeader(shard); shard.tell(new UpdateSchemaContext(SchemaContextHelper.full()), ActorRef.noSender()); - MockDataChangeListener listener = new MockDataChangeListener(1); - ActorRef dclActor = getSystem().actorOf(DataChangeListener.props(listener), + final MockDataChangeListener listener = new MockDataChangeListener(1); + final ActorRef dclActor = getSystem().actorOf(DataChangeListener.props(listener), "testRegisterChangeListener-DataChangeListener"); shard.tell(new RegisterChangeListener(TestModel.TEST_PATH, - dclActor, AsyncDataBroker.DataChangeScope.BASE), getRef()); + dclActor, AsyncDataBroker.DataChangeScope.BASE, true), getRef()); - RegisterChangeListenerReply reply = expectMsgClass(duration("3 seconds"), + final RegisterChangeListenerReply reply = expectMsgClass(duration("3 seconds"), RegisterChangeListenerReply.class); - String replyPath = reply.getListenerRegistrationPath().toString(); + final String replyPath = reply.getListenerRegistrationPath().toString(); assertTrue("Incorrect reply path: " + replyPath, replyPath.matches( "akka:\\/\\/test\\/user\\/testRegisterChangeListener\\/\\$.*")); - YangInstanceIdentifier path = TestModel.TEST_PATH; + final YangInstanceIdentifier path = TestModel.TEST_PATH; writeToStore(shard, path, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); listener.waitForChangeEvents(path); @@ -195,7 +179,7 @@ public class ShardTest extends AbstractShardTest { // ElectionTimeout with the sending of the RegisterChangeListener message. final CountDownLatch onFirstElectionTimeout = new CountDownLatch(1); final CountDownLatch onChangeListenerRegistered = new CountDownLatch(1); - Creator creator = new Creator() { + final Creator creator = new Creator() { boolean firstElectionTimeout = true; @Override @@ -204,8 +188,7 @@ public class ShardTest extends AbstractShardTest { // this will cause all other messages to not be queued properly after that. // The basic issue is that you cannot use TestActorRef with a persistent actor (at least when // it does do a persist) - return new Shard(shardID, Collections.emptyMap(), - dataStoreContextBuilder.persistent(false).build(), SCHEMA_CONTEXT) { + return new Shard(newShardBuilder()) { @Override public void onReceiveCommand(final Object message) throws Exception { if(message instanceof ElectionTimeout && firstElectionTimeout) { @@ -237,16 +220,16 @@ public class ShardTest extends AbstractShardTest { } }; - MockDataChangeListener listener = new MockDataChangeListener(1); - ActorRef dclActor = getSystem().actorOf(DataChangeListener.props(listener), + final MockDataChangeListener listener = new MockDataChangeListener(1); + final ActorRef dclActor = getSystem().actorOf(DataChangeListener.props(listener), "testRegisterChangeListenerWhenNotLeaderInitially-DataChangeListener"); - TestActorRef shard = TestActorRef.create(getSystem(), - Props.create(new DelegatingShardCreator(creator)), + final TestActorRef shard = TestActorRef.create(getSystem(), + Props.create(new DelegatingShardCreator(creator)).withDispatcher(Dispatchers.DefaultDispatcherId()), "testRegisterChangeListenerWhenNotLeaderInitially"); // Write initial data into the in-memory store. - YangInstanceIdentifier path = TestModel.TEST_PATH; + final YangInstanceIdentifier path = TestModel.TEST_PATH; writeToStore(shard, path, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); // Wait until the shard receives the first ElectionTimeout message. @@ -255,15 +238,15 @@ public class ShardTest extends AbstractShardTest { // Now send the RegisterChangeListener and wait for the reply. shard.tell(new RegisterChangeListener(path, dclActor, - AsyncDataBroker.DataChangeScope.SUBTREE), getRef()); + AsyncDataBroker.DataChangeScope.SUBTREE, false), getRef()); - RegisterChangeListenerReply reply = expectMsgClass(duration("5 seconds"), + final RegisterChangeListenerReply reply = expectMsgClass(duration("5 seconds"), RegisterChangeListenerReply.class); assertNotNull("getListenerRegistrationPath", reply.getListenerRegistrationPath()); // Sanity check - verify the shard is not the leader yet. shard.tell(new FindLeader(), getRef()); - FindLeaderReply findLeadeReply = + final FindLeaderReply findLeadeReply = expectMsgClass(duration("5 seconds"), FindLeaderReply.class); assertNull("Expected the shard not to be the leader", findLeadeReply.getLeaderActor()); @@ -283,26 +266,26 @@ public class ShardTest extends AbstractShardTest { @Test public void testRegisterDataTreeChangeListener() throws Exception { new ShardTestKit(getSystem()) {{ - TestActorRef shard = TestActorRef.create(getSystem(), + final TestActorRef shard = TestActorRef.create(getSystem(), newShardProps(), "testRegisterDataTreeChangeListener"); waitUntilLeader(shard); shard.tell(new UpdateSchemaContext(SchemaContextHelper.full()), ActorRef.noSender()); - MockDataTreeChangeListener listener = new MockDataTreeChangeListener(1); - ActorRef dclActor = getSystem().actorOf(DataTreeChangeListenerActor.props(listener), + final MockDataTreeChangeListener listener = new MockDataTreeChangeListener(1); + final ActorRef dclActor = getSystem().actorOf(DataTreeChangeListenerActor.props(listener), "testRegisterDataTreeChangeListener-DataTreeChangeListener"); - shard.tell(new RegisterDataTreeChangeListener(TestModel.TEST_PATH, dclActor), getRef()); + shard.tell(new RegisterDataTreeChangeListener(TestModel.TEST_PATH, dclActor, false), getRef()); - RegisterDataTreeChangeListenerReply reply = expectMsgClass(duration("3 seconds"), + final RegisterDataTreeChangeListenerReply reply = expectMsgClass(duration("3 seconds"), RegisterDataTreeChangeListenerReply.class); - String replyPath = reply.getListenerRegistrationPath().toString(); + final String replyPath = reply.getListenerRegistrationPath().toString(); assertTrue("Incorrect reply path: " + replyPath, replyPath.matches( "akka:\\/\\/test\\/user\\/testRegisterDataTreeChangeListener\\/\\$.*")); - YangInstanceIdentifier path = TestModel.TEST_PATH; + final YangInstanceIdentifier path = TestModel.TEST_PATH; writeToStore(shard, path, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); listener.waitForChangeEvents(); @@ -318,13 +301,13 @@ public class ShardTest extends AbstractShardTest { new ShardTestKit(getSystem()) {{ final CountDownLatch onFirstElectionTimeout = new CountDownLatch(1); final CountDownLatch onChangeListenerRegistered = new CountDownLatch(1); - Creator creator = new Creator() { + final Creator creator = new Creator() { boolean firstElectionTimeout = true; @Override public Shard create() throws Exception { - return new Shard(shardID, Collections.emptyMap(), - dataStoreContextBuilder.persistent(false).build(), SCHEMA_CONTEXT) { + return new Shard(Shard.builder().id(shardID).datastoreContext( + dataStoreContextBuilder.persistent(false).build()).schemaContext(SCHEMA_CONTEXT)) { @Override public void onReceiveCommand(final Object message) throws Exception { if(message instanceof ElectionTimeout && firstElectionTimeout) { @@ -348,27 +331,27 @@ public class ShardTest extends AbstractShardTest { } }; - MockDataTreeChangeListener listener = new MockDataTreeChangeListener(1); - ActorRef dclActor = getSystem().actorOf(DataTreeChangeListenerActor.props(listener), + final MockDataTreeChangeListener listener = new MockDataTreeChangeListener(1); + final ActorRef dclActor = getSystem().actorOf(DataTreeChangeListenerActor.props(listener), "testDataTreeChangeListenerNotifiedWhenNotTheLeaderOnRegistration-DataChangeListener"); - TestActorRef shard = TestActorRef.create(getSystem(), - Props.create(new DelegatingShardCreator(creator)), + final TestActorRef shard = TestActorRef.create(getSystem(), + Props.create(new DelegatingShardCreator(creator)).withDispatcher(Dispatchers.DefaultDispatcherId()), "testDataTreeChangeListenerNotifiedWhenNotTheLeaderOnRegistration"); - YangInstanceIdentifier path = TestModel.TEST_PATH; + final YangInstanceIdentifier path = TestModel.TEST_PATH; writeToStore(shard, path, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); assertEquals("Got first ElectionTimeout", true, - onFirstElectionTimeout.await(5, TimeUnit.SECONDS)); + onFirstElectionTimeout.await(5, TimeUnit.SECONDS)); - shard.tell(new RegisterDataTreeChangeListener(path, dclActor), getRef()); - RegisterDataTreeChangeListenerReply reply = expectMsgClass(duration("5 seconds"), - RegisterDataTreeChangeListenerReply.class); + shard.tell(new RegisterDataTreeChangeListener(path, dclActor, false), getRef()); + final RegisterDataTreeChangeListenerReply reply = expectMsgClass(duration("5 seconds"), + RegisterDataTreeChangeListenerReply.class); assertNotNull("getListenerRegistratioznPath", reply.getListenerRegistrationPath()); shard.tell(new FindLeader(), getRef()); - FindLeaderReply findLeadeReply = + final FindLeaderReply findLeadeReply = expectMsgClass(duration("5 seconds"), FindLeaderReply.class); assertNull("Expected the shard not to be the leader", findLeadeReply.getLeaderActor()); @@ -387,7 +370,7 @@ public class ShardTest extends AbstractShardTest { @Test public void testCreateTransaction(){ new ShardTestKit(getSystem()) {{ - ActorRef shard = getSystem().actorOf(newShardProps(), "testCreateTransaction"); + final ActorRef shard = getSystem().actorOf(newShardProps(), "testCreateTransaction"); waitUntilLeader(shard); @@ -396,10 +379,10 @@ public class ShardTest extends AbstractShardTest { shard.tell(new CreateTransaction("txn-1", TransactionType.READ_ONLY.ordinal() ).toSerializable(), getRef()); - CreateTransactionReply reply = expectMsgClass(duration("3 seconds"), + final CreateTransactionReply reply = expectMsgClass(duration("3 seconds"), CreateTransactionReply.class); - String path = reply.getTransactionActorPath().toString(); + final String path = reply.getTransactionActorPath().toString(); assertTrue("Unexpected transaction path " + path, path.contains("akka://test/user/testCreateTransaction/shard-txn-1")); @@ -418,10 +401,10 @@ public class ShardTest extends AbstractShardTest { TransactionType.READ_ONLY.ordinal() , "foobar").toSerializable(), getRef()); - CreateTransactionReply reply = expectMsgClass(duration("3 seconds"), + final CreateTransactionReply reply = expectMsgClass(duration("3 seconds"), CreateTransactionReply.class); - String path = reply.getTransactionActorPath().toString(); + final String path = reply.getTransactionActorPath().toString(); assertTrue("Unexpected transaction path " + path, path.contains("akka://test/user/testCreateTransactionOnChain/shard-txn-1")); @@ -436,12 +419,13 @@ public class ShardTest extends AbstractShardTest { final CountDownLatch recoveryComplete = new CountDownLatch(1); class TestShard extends Shard { TestShard() { - super(shardID, Collections.singletonMap(shardID.toString(), null), - newDatastoreContext(), SCHEMA_CONTEXT); + super(Shard.builder().id(shardID).datastoreContext(newDatastoreContext()). + peerAddresses(Collections.singletonMap(shardID.toString(), null)). + schemaContext(SCHEMA_CONTEXT)); } - Map getPeerAddresses() { - return getRaftActorContext().getPeerAddresses(); + String getPeerAddress(String id) { + return getRaftActorContext().getPeerAddress(id); } @Override @@ -462,15 +446,14 @@ public class ShardTest extends AbstractShardTest { } })), "testPeerAddressResolved"); - //waitUntilLeader(shard); assertEquals("Recovery complete", true, - Uninterruptibles.awaitUninterruptibly(recoveryComplete, 5, TimeUnit.SECONDS)); + Uninterruptibles.awaitUninterruptibly(recoveryComplete, 5, TimeUnit.SECONDS)); - String address = "akka://foobar"; + final String address = "akka://foobar"; shard.underlyingActor().onReceiveCommand(new PeerAddressResolved(shardID.toString(), address)); - assertEquals("getPeerAddresses", address, - ((TestShard)shard.underlyingActor()).getPeerAddresses().get(shardID.toString())); + assertEquals("getPeerAddress", address, + ((TestShard) shard.underlyingActor()).getPeerAddress(shardID.toString())); shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); }}; @@ -478,24 +461,31 @@ public class ShardTest extends AbstractShardTest { @Test public void testApplySnapshot() throws Exception { - TestActorRef shard = TestActorRef.create(getSystem(), newShardProps(), + + final TestActorRef shard = TestActorRef.create(getSystem(), newShardProps(), "testApplySnapshot"); - DataTree store = InMemoryDataTreeFactory.getInstance().create(); + ShardTestKit.waitUntilLeader(shard); + + final DataTree store = InMemoryDataTreeFactory.getInstance().create(TreeType.OPERATIONAL); store.setSchemaContext(SCHEMA_CONTEXT); - writeToStore(store, TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); + final ContainerNode container = ImmutableContainerNodeBuilder.create().withNodeIdentifier( + new YangInstanceIdentifier.NodeIdentifier(TestModel.TEST_QNAME)). + withChild(ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).addChild( + ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1)).build()).build(); + + writeToStore(store, TestModel.TEST_PATH, container); - YangInstanceIdentifier root = YangInstanceIdentifier.builder().build(); - NormalizedNode expected = readStore(store, root); + final YangInstanceIdentifier root = YangInstanceIdentifier.builder().build(); + final NormalizedNode expected = readStore(store, root); - ApplySnapshot applySnapshot = new ApplySnapshot(Snapshot.create( - SerializationUtils.serializeNormalizedNode(expected), - Collections.emptyList(), 1, 2, 3, 4)); + final Snapshot snapshot = Snapshot.create(SerializationUtils.serializeNormalizedNode(expected), + Collections.emptyList(), 1, 2, 3, 4); - shard.underlyingActor().onReceiveCommand(applySnapshot); + shard.underlyingActor().getRaftActorSnapshotCohort().applySnapshot(snapshot.getState()); - NormalizedNode actual = readStore(shard, root); + final NormalizedNode actual = readStore(shard, root); assertEquals("Root node", expected, actual); @@ -504,50 +494,30 @@ public class ShardTest extends AbstractShardTest { @Test public void testApplyState() throws Exception { + final TestActorRef shard = TestActorRef.create(getSystem(), newShardProps(), "testApplyState"); - TestActorRef shard = TestActorRef.create(getSystem(), newShardProps(), "testApplyState"); - - NormalizedNode node = ImmutableNodes.containerNode(TestModel.TEST_QNAME); - - ApplyState applyState = new ApplyState(null, "test", new ReplicatedLogImplEntry(1, 2, - newModificationPayload(new WriteModification(TestModel.TEST_PATH, node)))); - - shard.underlyingActor().onReceiveCommand(applyState); - - NormalizedNode actual = readStore(shard, TestModel.TEST_PATH); - assertEquals("Applied state", node, actual); - - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); - } - - @Test - public void testApplyStateWithCandidatePayload() throws Exception { - - TestActorRef shard = TestActorRef.create(getSystem(), newShardPropsWithRecoveryComplete(), "testApplyState"); + ShardTestKit.waitUntilLeader(shard); - recoveryComplete.await(5, TimeUnit.SECONDS); + final NormalizedNode node = ImmutableNodes.containerNode(TestModel.TEST_QNAME); - NormalizedNode node = ImmutableNodes.containerNode(TestModel.TEST_QNAME); - DataTreeCandidate candidate = DataTreeCandidates.fromNormalizedNode(TestModel.TEST_PATH, node); - - ApplyState applyState = new ApplyState(null, "test", new ReplicatedLogImplEntry(1, 2, - DataTreeCandidatePayload.create(candidate))); + final ApplyState applyState = new ApplyState(null, "test", new ReplicatedLogImplEntry(1, 2, + newDataTreeCandidatePayload(new WriteModification(TestModel.TEST_PATH, node)))); shard.underlyingActor().onReceiveCommand(applyState); - NormalizedNode actual = readStore(shard, TestModel.TEST_PATH); + final NormalizedNode actual = readStore(shard, TestModel.TEST_PATH); assertEquals("Applied state", node, actual); shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); } DataTree setupInMemorySnapshotStore() throws DataValidationFailedException { - DataTree testStore = InMemoryDataTreeFactory.getInstance().create(); + final DataTree testStore = InMemoryDataTreeFactory.getInstance().create(TreeType.OPERATIONAL); testStore.setSchemaContext(SCHEMA_CONTEXT); writeToStore(testStore, TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); - NormalizedNode root = readStore(testStore, YangInstanceIdentifier.builder().build()); + final NormalizedNode root = readStore(testStore, YangInstanceIdentifier.builder().build()); InMemorySnapshotStore.addSnapshot(shardID.toString(), Snapshot.create( SerializationUtils.serializeNormalizedNode(root), @@ -555,7 +525,7 @@ public class ShardTest extends AbstractShardTest { return testStore; } - private static DataTreeCandidatePayload payloadForModification(DataTree source, DataTreeModification mod) throws DataValidationFailedException { + private static DataTreeCandidatePayload payloadForModification(final DataTree source, final DataTreeModification mod) throws DataValidationFailedException { source.validate(mod); final DataTreeCandidate candidate = source.prepare(mod); source.commit(candidate); @@ -569,31 +539,31 @@ public class ShardTest extends AbstractShardTest { final DataTreeModification writeMod = source.takeSnapshot().newModification(); writeMod.write(TestModel.OUTER_LIST_PATH, ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build()); - + writeMod.ready(); InMemoryJournal.addEntry(shardID.toString(), 0, DUMMY_DATA); // Set up the InMemoryJournal. InMemoryJournal.addEntry(shardID.toString(), 1, new ReplicatedLogImplEntry(0, 1, payloadForModification(source, writeMod))); - int nListEntries = 16; - Set listEntryKeys = new HashSet<>(); + final int nListEntries = 16; + final Set listEntryKeys = new HashSet<>(); // Add some ModificationPayload entries for (int i = 1; i <= nListEntries; i++) { listEntryKeys.add(Integer.valueOf(i)); - YangInstanceIdentifier path = YangInstanceIdentifier.builder(TestModel.OUTER_LIST_PATH) + final YangInstanceIdentifier path = YangInstanceIdentifier.builder(TestModel.OUTER_LIST_PATH) .nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, i).build(); final DataTreeModification mod = source.takeSnapshot().newModification(); mod.merge(path, ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, i)); - + mod.ready(); InMemoryJournal.addEntry(shardID.toString(), i+1, new ReplicatedLogImplEntry(i, 1, payloadForModification(source, mod))); } InMemoryJournal.addEntry(shardID.toString(), nListEntries + 2, - new ApplyJournalEntries(nListEntries)); + new ApplyJournalEntries(nListEntries)); testRecovery(listEntryKeys); } @@ -608,22 +578,26 @@ public class ShardTest extends AbstractShardTest { InMemoryJournal.addEntry(shardID.toString(), 0, DUMMY_DATA); - InMemoryJournal.addEntry(shardID.toString(), 1, new ReplicatedLogImplEntry(0, 1, newModificationPayload( - new WriteModification(TestModel.OUTER_LIST_PATH, - ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build())))); + ShardDataTree shardDataTree = new ShardDataTree(SCHEMA_CONTEXT, TreeType.CONFIGURATION); - int nListEntries = 16; - Set listEntryKeys = new HashSet<>(); + InMemoryJournal.addEntry(shardID.toString(), 1, new ReplicatedLogImplEntry(0, 1, newDataTreeCandidatePayload( + shardDataTree, + new WriteModification(TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME)), + new WriteModification(TestModel.OUTER_LIST_PATH, + ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build())))); + + final int nListEntries = 16; + final Set listEntryKeys = new HashSet<>(); // Add some ModificationPayload entries for(int i = 1; i <= nListEntries; i++) { listEntryKeys.add(Integer.valueOf(i)); - YangInstanceIdentifier path = YangInstanceIdentifier.builder(TestModel.OUTER_LIST_PATH) + final YangInstanceIdentifier path = YangInstanceIdentifier.builder(TestModel.OUTER_LIST_PATH) .nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, i).build(); - Modification mod = new MergeModification(path, + final Modification mod = new MergeModification(path, ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, i)); InMemoryJournal.addEntry(shardID.toString(), i + 1, new ReplicatedLogImplEntry(i, 1, - newModificationPayload(mod))); + newDataTreeCandidatePayload(shardDataTree, mod))); } InMemoryJournal.addEntry(shardID.toString(), nListEntries + 2, @@ -632,13 +606,18 @@ public class ShardTest extends AbstractShardTest { testRecovery(listEntryKeys); } - private static ModificationPayload newModificationPayload(final Modification... mods) throws IOException { - MutableCompositeModification compMod = new MutableCompositeModification(); - for(Modification mod: mods) { - compMod.addModification(mod); + private static DataTreeCandidatePayload newDataTreeCandidatePayload(final Modification... mods) throws Exception { + return newDataTreeCandidatePayload(new ShardDataTree(SCHEMA_CONTEXT, TreeType.CONFIGURATION), mods); + } + + private static DataTreeCandidatePayload newDataTreeCandidatePayload(ShardDataTree shardDataTree, + final Modification... mods) throws Exception { + DataTreeModification dataTreeModification = shardDataTree.newModification(); + for(final Modification mod: mods) { + mod.apply(dataTreeModification); } - return new ModificationPayload(compMod); + return DataTreeCandidatePayload.create(shardDataTree.commit(dataTreeModification)); } @Test @@ -652,65 +631,57 @@ public class ShardTest extends AbstractShardTest { // Setup 3 simulated transactions with mock cohorts backed by real cohorts. - ShardDataTree dataStore = shard.underlyingActor().getDataStore(); + final ShardDataTree dataStore = shard.underlyingActor().getDataStore(); - String transactionID1 = "tx1"; - MutableCompositeModification modification1 = new MutableCompositeModification(); - ShardDataTreeCohort cohort1 = setupMockWriteTransaction("cohort1", dataStore, + final String transactionID1 = "tx1"; + final MutableCompositeModification modification1 = new MutableCompositeModification(); + final ShardDataTreeCohort cohort1 = setupMockWriteTransaction("cohort1", dataStore, TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME), modification1); - String transactionID2 = "tx2"; - MutableCompositeModification modification2 = new MutableCompositeModification(); - ShardDataTreeCohort cohort2 = setupMockWriteTransaction("cohort2", dataStore, + final String transactionID2 = "tx2"; + final MutableCompositeModification modification2 = new MutableCompositeModification(); + final ShardDataTreeCohort cohort2 = setupMockWriteTransaction("cohort2", dataStore, TestModel.OUTER_LIST_PATH, ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build(), modification2); - String transactionID3 = "tx3"; - MutableCompositeModification modification3 = new MutableCompositeModification(); - ShardDataTreeCohort cohort3 = setupMockWriteTransaction("cohort3", dataStore, + final String transactionID3 = "tx3"; + final MutableCompositeModification modification3 = new MutableCompositeModification(); + final ShardDataTreeCohort cohort3 = setupMockWriteTransaction("cohort3", dataStore, YangInstanceIdentifier.builder(TestModel.OUTER_LIST_PATH) .nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1).build(), ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1), modification3); - long timeoutSec = 5; + final long timeoutSec = 5; final FiniteDuration duration = FiniteDuration.create(timeoutSec, TimeUnit.SECONDS); final Timeout timeout = new Timeout(duration); - // Simulate the ForwardedReadyTransaction message for the first Tx that would be sent - // by the ShardTransaction. - - shard.tell(new ForwardedReadyTransaction(transactionID1, CURRENT_VERSION, - cohort1, modification1, true, false), getRef()); - ReadyTransactionReply readyReply = ReadyTransactionReply.fromSerializable( + shard.tell(prepareReadyTransactionMessage(false, shard.underlyingActor(), cohort1, transactionID1, modification1), getRef()); + final ReadyTransactionReply readyReply = ReadyTransactionReply.fromSerializable( expectMsgClass(duration, ReadyTransactionReply.class)); assertEquals("Cohort path", shard.path().toString(), readyReply.getCohortPath()); // Send the CanCommitTransaction message for the first Tx. shard.tell(new CanCommitTransaction(transactionID1).toSerializable(), getRef()); - CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( + final CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); assertEquals("Can commit", true, canCommitReply.getCanCommit()); - // Send the ForwardedReadyTransaction for the next 2 Tx's. - - shard.tell(new ForwardedReadyTransaction(transactionID2, CURRENT_VERSION, - cohort2, modification2, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(false, shard.underlyingActor(), cohort2, transactionID2, modification2), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); - shard.tell(new ForwardedReadyTransaction(transactionID3, CURRENT_VERSION, - cohort3, modification3, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(false, shard.underlyingActor(), cohort3, transactionID3, modification3), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); // Send the CanCommitTransaction message for the next 2 Tx's. These should get queued and // processed after the first Tx completes. - Future canCommitFuture1 = Patterns.ask(shard, + final Future canCommitFuture1 = Patterns.ask(shard, new CanCommitTransaction(transactionID2).toSerializable(), timeout); - Future canCommitFuture2 = Patterns.ask(shard, + final Future canCommitFuture2 = Patterns.ask(shard, new CanCommitTransaction(transactionID3).toSerializable(), timeout); // Send the CommitTransaction message for the first Tx. After it completes, it should @@ -739,7 +710,7 @@ public class ShardTest extends AbstractShardTest { try { assertEquals("Commit response type", expRespType, resp.getClass()); onSuccess(resp); - } catch (Exception e) { + } catch (final Exception e) { caughtEx.set(e); } } @@ -771,11 +742,11 @@ public class ShardTest extends AbstractShardTest { @Override void onSuccess(final Object resp) throws Exception { - CanCommitTransactionReply canCommitReply = + final CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable(resp); assertEquals("Can commit", true, canCommitReply.getCanCommit()); - Future commitFuture = Patterns.ask(shard, + final Future commitFuture = Patterns.ask(shard, new CommitTransaction(transactionID).toSerializable(), timeout); commitFuture.onComplete(new OnCommitFutureComplete(), getSystem().dispatcher()); } @@ -787,7 +758,7 @@ public class ShardTest extends AbstractShardTest { canCommitFuture2.onComplete(new OnCanCommitFutureComplete(transactionID3), getSystem().dispatcher()); - boolean done = commitLatch.await(timeoutSec, TimeUnit.SECONDS); + final boolean done = commitLatch.await(timeoutSec, TimeUnit.SECONDS); if(caughtEx.get() != null) { throw caughtEx.get(); @@ -795,7 +766,7 @@ public class ShardTest extends AbstractShardTest { assertEquals("Commits complete", true, done); - InOrder inOrder = inOrder(cohort1, cohort2, cohort3); + final InOrder inOrder = inOrder(cohort1, cohort2, cohort3); inOrder.verify(cohort1).canCommit(); inOrder.verify(cohort1).preCommit(); inOrder.verify(cohort1).commit(); @@ -816,15 +787,15 @@ public class ShardTest extends AbstractShardTest { }}; } - private static BatchedModifications newBatchedModifications(String transactionID, YangInstanceIdentifier path, - NormalizedNode data, boolean ready, boolean doCommitOnReady, int messagesSent) { + private static BatchedModifications newBatchedModifications(final String transactionID, final YangInstanceIdentifier path, + final NormalizedNode data, final boolean ready, final boolean doCommitOnReady, final int messagesSent) { return newBatchedModifications(transactionID, null, path, data, ready, doCommitOnReady, messagesSent); } - private static BatchedModifications newBatchedModifications(String transactionID, String transactionChainID, - YangInstanceIdentifier path, NormalizedNode data, boolean ready, boolean doCommitOnReady, - int messagesSent) { - BatchedModifications batched = new BatchedModifications(transactionID, CURRENT_VERSION, transactionChainID); + private static BatchedModifications newBatchedModifications(final String transactionID, final String transactionChainID, + final YangInstanceIdentifier path, final NormalizedNode data, final boolean ready, final boolean doCommitOnReady, + final int messagesSent) { + final BatchedModifications batched = new BatchedModifications(transactionID, CURRENT_VERSION, transactionChainID); batched.addModification(new WriteModification(path, data)); batched.setReady(ready); batched.setDoCommitOnReady(doCommitOnReady); @@ -842,12 +813,12 @@ public class ShardTest extends AbstractShardTest { waitUntilLeader(shard); final String transactionID = "tx"; - FiniteDuration duration = duration("5 seconds"); + final FiniteDuration duration = duration("5 seconds"); final AtomicReference mockCohort = new AtomicReference<>(); - ShardCommitCoordinator.CohortDecorator cohortDecorator = new ShardCommitCoordinator.CohortDecorator() { + final ShardCommitCoordinator.CohortDecorator cohortDecorator = new ShardCommitCoordinator.CohortDecorator() { @Override - public ShardDataTreeCohort decorate(String txID, ShardDataTreeCohort actual) { + public ShardDataTreeCohort decorate(final String txID, final ShardDataTreeCohort actual) { if(mockCohort.get() == null) { mockCohort.set(createDelegatingMockCohort("cohort", actual)); } @@ -878,7 +849,7 @@ public class ShardTest extends AbstractShardTest { // Send the CanCommitTransaction message. shard.tell(new CanCommitTransaction(transactionID).toSerializable(), getRef()); - CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( + final CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); assertEquals("Can commit", true, canCommitReply.getCanCommit()); @@ -887,7 +858,7 @@ public class ShardTest extends AbstractShardTest { shard.tell(new CommitTransaction(transactionID).toSerializable(), getRef()); expectMsgClass(duration, CommitTransactionReply.SERIALIZABLE_CLASS); - InOrder inOrder = inOrder(mockCohort.get()); + final InOrder inOrder = inOrder(mockCohort.get()); inOrder.verify(mockCohort.get()).canCommit(); inOrder.verify(mockCohort.get()).preCommit(); inOrder.verify(mockCohort.get()).commit(); @@ -910,12 +881,12 @@ public class ShardTest extends AbstractShardTest { waitUntilLeader(shard); final String transactionID = "tx"; - FiniteDuration duration = duration("5 seconds"); + final FiniteDuration duration = duration("5 seconds"); final AtomicReference mockCohort = new AtomicReference<>(); - ShardCommitCoordinator.CohortDecorator cohortDecorator = new ShardCommitCoordinator.CohortDecorator() { + final ShardCommitCoordinator.CohortDecorator cohortDecorator = new ShardCommitCoordinator.CohortDecorator() { @Override - public ShardDataTreeCohort decorate(String txID, ShardDataTreeCohort actual) { + public ShardDataTreeCohort decorate(final String txID, final ShardDataTreeCohort actual) { if(mockCohort.get() == null) { mockCohort.set(createDelegatingMockCohort("cohort", actual)); } @@ -935,16 +906,16 @@ public class ShardTest extends AbstractShardTest { // Send a couple more BatchedModifications. shard.tell(newBatchedModifications(transactionID, TestModel.OUTER_LIST_PATH, - ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build(), false, false, 2), getRef()); + ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build(), false, false, 2), getRef()); expectMsgClass(duration, BatchedModificationsReply.class); shard.tell(newBatchedModifications(transactionID, YangInstanceIdentifier.builder( TestModel.OUTER_LIST_PATH).nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1).build(), - ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1), true, true, 3), getRef()); + ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1), true, true, 3), getRef()); expectMsgClass(duration, CommitTransactionReply.SERIALIZABLE_CLASS); - InOrder inOrder = inOrder(mockCohort.get()); + final InOrder inOrder = inOrder(mockCohort.get()); inOrder.verify(mockCohort.get()).canCommit(); inOrder.verify(mockCohort.get()).preCommit(); inOrder.verify(mockCohort.get()).commit(); @@ -966,14 +937,14 @@ public class ShardTest extends AbstractShardTest { waitUntilLeader(shard); - String transactionID = "tx1"; - BatchedModifications batched = new BatchedModifications(transactionID, DataStoreVersions.CURRENT_VERSION, null); + final String transactionID = "tx1"; + final BatchedModifications batched = new BatchedModifications(transactionID, DataStoreVersions.CURRENT_VERSION, null); batched.setReady(true); batched.setTotalMessagesSent(2); shard.tell(batched, getRef()); - Failure failure = expectMsgClass(duration("5 seconds"), akka.actor.Status.Failure.class); + final Failure failure = expectMsgClass(duration("5 seconds"), akka.actor.Status.Failure.class); shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); @@ -983,17 +954,55 @@ public class ShardTest extends AbstractShardTest { }}; } + @Test + public void testBatchedModificationsWithOperationFailure() throws Throwable { + new ShardTestKit(getSystem()) {{ + final TestActorRef shard = TestActorRef.create(getSystem(), + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testBatchedModificationsWithOperationFailure"); + + waitUntilLeader(shard); + + // Test merge with invalid data. An exception should occur when the merge is applied. Note that + // write will not validate the children for performance reasons. + + String transactionID = "tx1"; + + ContainerNode invalidData = ImmutableContainerNodeBuilder.create().withNodeIdentifier( + new YangInstanceIdentifier.NodeIdentifier(TestModel.TEST_QNAME)). + withChild(ImmutableNodes.leafNode(TestModel.JUNK_QNAME, "junk")).build(); + + BatchedModifications batched = new BatchedModifications(transactionID, CURRENT_VERSION, null); + batched.addModification(new MergeModification(TestModel.TEST_PATH, invalidData)); + shard.tell(batched, getRef()); + Failure failure = expectMsgClass(duration("5 seconds"), akka.actor.Status.Failure.class); + + Throwable cause = failure.cause(); + + batched = new BatchedModifications(transactionID, DataStoreVersions.CURRENT_VERSION, null); + batched.setReady(true); + batched.setTotalMessagesSent(2); + + shard.tell(batched, getRef()); + + failure = expectMsgClass(duration("5 seconds"), akka.actor.Status.Failure.class); + assertEquals("Failure cause", cause, failure.cause()); + + shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); + }}; + } + @SuppressWarnings("unchecked") - private static void verifyOuterListEntry(final TestActorRef shard, Object expIDValue) throws Exception { - NormalizedNode outerList = readStore(shard, TestModel.OUTER_LIST_PATH); + private static void verifyOuterListEntry(final TestActorRef shard, final Object expIDValue) throws Exception { + final NormalizedNode outerList = readStore(shard, TestModel.OUTER_LIST_PATH); assertNotNull(TestModel.OUTER_LIST_QNAME.getLocalName() + " not found", outerList); assertTrue(TestModel.OUTER_LIST_QNAME.getLocalName() + " value is not Iterable", outerList.getValue() instanceof Iterable); - Object entry = ((Iterable)outerList.getValue()).iterator().next(); + final Object entry = ((Iterable)outerList.getValue()).iterator().next(); assertTrue(TestModel.OUTER_LIST_QNAME.getLocalName() + " entry is not MapEntryNode", entry instanceof MapEntryNode); - MapEntryNode mapEntry = (MapEntryNode)entry; - Optional> idLeaf = + final MapEntryNode mapEntry = (MapEntryNode)entry; + final Optional> idLeaf = mapEntry.getChild(new YangInstanceIdentifier.NodeIdentifier(TestModel.ID_QNAME)); assertTrue("Missing leaf " + TestModel.ID_QNAME.getLocalName(), idLeaf.isPresent()); assertEquals(TestModel.ID_QNAME.getLocalName() + " value", expIDValue, idLeaf.get().getValue()); @@ -1008,16 +1017,16 @@ public class ShardTest extends AbstractShardTest { waitUntilLeader(shard); - String transactionChainID = "txChain"; - String transactionID1 = "tx1"; - String transactionID2 = "tx2"; + final String transactionChainID = "txChain"; + final String transactionID1 = "tx1"; + final String transactionID2 = "tx2"; - FiniteDuration duration = duration("5 seconds"); + final FiniteDuration duration = duration("5 seconds"); // Send a BatchedModifications to start a chained write transaction and ready it. - ContainerNode containerNode = ImmutableNodes.containerNode(TestModel.TEST_QNAME); - YangInstanceIdentifier path = TestModel.TEST_PATH; + final ContainerNode containerNode = ImmutableNodes.containerNode(TestModel.TEST_QNAME); + final YangInstanceIdentifier path = TestModel.TEST_PATH; shard.tell(newBatchedModifications(transactionID1, transactionChainID, path, containerNode, true, false, 1), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); @@ -1027,16 +1036,16 @@ public class ShardTest extends AbstractShardTest { shard.tell(new CreateTransaction(transactionID2, TransactionType.READ_ONLY.ordinal() , transactionChainID).toSerializable(), getRef()); - CreateTransactionReply createReply = expectMsgClass(duration("3 seconds"), CreateTransactionReply.class); + final CreateTransactionReply createReply = expectMsgClass(duration("3 seconds"), CreateTransactionReply.class); getSystem().actorSelection(createReply.getTransactionActorPath()).tell(new ReadData(path), getRef()); - ReadDataReply readReply = expectMsgClass(duration("3 seconds"), ReadDataReply.class); + final ReadDataReply readReply = expectMsgClass(duration("3 seconds"), ReadDataReply.class); assertEquals("Read node", containerNode, readReply.getNormalizedNode()); // Commit the write transaction. shard.tell(new CanCommitTransaction(transactionID1).toSerializable(), getRef()); - CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( + final CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); assertEquals("Can commit", true, canCommitReply.getCanCommit()); @@ -1045,7 +1054,7 @@ public class ShardTest extends AbstractShardTest { // Verify data in the data store. - NormalizedNode actualNode = readStore(shard, path); + final NormalizedNode actualNode = readStore(shard, path); assertEquals("Stored node", containerNode, actualNode); shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); @@ -1056,13 +1065,12 @@ public class ShardTest extends AbstractShardTest { public void testOnBatchedModificationsWhenNotLeader() { final AtomicBoolean overrideLeaderCalls = new AtomicBoolean(); new ShardTestKit(getSystem()) {{ - Creator creator = new Creator() { + final Creator creator = new Creator() { private static final long serialVersionUID = 1L; @Override public Shard create() throws Exception { - return new Shard(shardID, Collections.emptyMap(), - newDatastoreContext(), SCHEMA_CONTEXT) { + return new Shard(newShardBuilder()) { @Override protected boolean isLeader() { return overrideLeaderCalls.get() ? false : super.isLeader(); @@ -1077,14 +1085,14 @@ public class ShardTest extends AbstractShardTest { } }; - TestActorRef shard = TestActorRef.create(getSystem(), + final TestActorRef shard = TestActorRef.create(getSystem(), Props.create(new DelegatingShardCreator(creator)), "testOnBatchedModificationsWhenNotLeader"); waitUntilLeader(shard); overrideLeaderCalls.set(true); - BatchedModifications batched = new BatchedModifications("tx", DataStoreVersions.CURRENT_VERSION, ""); + final BatchedModifications batched = new BatchedModifications("tx", DataStoreVersions.CURRENT_VERSION, ""); shard.tell(batched, ActorRef.noSender()); @@ -1095,38 +1103,65 @@ public class ShardTest extends AbstractShardTest { } @Test - public void testForwardedReadyTransactionWithImmediateCommit() throws Exception{ + public void testTransactionMessagesWithNoLeader() { + new ShardTestKit(getSystem()) {{ + dataStoreContextBuilder.customRaftPolicyImplementation(DisableElectionsRaftPolicy.class.getName()). + shardHeartbeatIntervalInMillis(50).shardElectionTimeoutFactor(1); + final TestActorRef shard = actorFactory.createTestActor( + newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testTransactionMessagesWithNoLeader"); + + waitUntilNoLeader(shard); + + shard.tell(new BatchedModifications("tx", DataStoreVersions.CURRENT_VERSION, ""), getRef()); + Failure failure = expectMsgClass(Failure.class); + assertEquals("Failure cause type", NoShardLeaderException.class, failure.cause().getClass()); + + shard.tell(prepareForwardedReadyTransaction(mock(ShardDataTreeCohort.class), "tx", + DataStoreVersions.CURRENT_VERSION, true), getRef()); + failure = expectMsgClass(Failure.class); + assertEquals("Failure cause type", NoShardLeaderException.class, failure.cause().getClass()); + + shard.tell(new ReadyLocalTransaction("tx", mock(DataTreeModification.class), true), getRef()); + failure = expectMsgClass(Failure.class); + assertEquals("Failure cause type", NoShardLeaderException.class, failure.cause().getClass()); + }}; + } + + @Test + public void testReadyWithImmediateCommit() throws Exception{ + testReadyWithImmediateCommit(true); + testReadyWithImmediateCommit(false); + } + + public void testReadyWithImmediateCommit(final boolean readWrite) throws Exception{ new ShardTestKit(getSystem()) {{ final TestActorRef shard = TestActorRef.create(getSystem(), newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testForwardedReadyTransactionWithImmediateCommit"); + "testReadyWithImmediateCommit-" + readWrite); waitUntilLeader(shard); - ShardDataTree dataStore = shard.underlyingActor().getDataStore(); + final ShardDataTree dataStore = shard.underlyingActor().getDataStore(); - String transactionID = "tx1"; - MutableCompositeModification modification = new MutableCompositeModification(); - NormalizedNode containerNode = ImmutableNodes.containerNode(TestModel.TEST_QNAME); - ShardDataTreeCohort cohort = setupMockWriteTransaction("cohort", dataStore, + final String transactionID = "tx1"; + final MutableCompositeModification modification = new MutableCompositeModification(); + final NormalizedNode containerNode = ImmutableNodes.containerNode(TestModel.TEST_QNAME); + final ShardDataTreeCohort cohort = setupMockWriteTransaction("cohort", dataStore, TestModel.TEST_PATH, containerNode, modification); - FiniteDuration duration = duration("5 seconds"); - - // Simulate the ForwardedReadyTransaction messages that would be sent - // by the ShardTransaction. + final FiniteDuration duration = duration("5 seconds"); - shard.tell(new ForwardedReadyTransaction(transactionID, CURRENT_VERSION, - cohort, modification, true, true), getRef()); + shard.tell(prepareReadyTransactionMessage(readWrite, shard.underlyingActor(), cohort, transactionID, modification, true), getRef()); expectMsgClass(duration, ThreePhaseCommitCohortMessages.CommitTransactionReply.class); - InOrder inOrder = inOrder(cohort); + final InOrder inOrder = inOrder(cohort); inOrder.verify(cohort).canCommit(); inOrder.verify(cohort).preCommit(); inOrder.verify(cohort).commit(); - NormalizedNode actualNode = readStore(shard, TestModel.TEST_PATH); + final NormalizedNode actualNode = readStore(shard, TestModel.TEST_PATH); assertEquals(TestModel.TEST_QNAME.getLocalName(), containerNode, actualNode); shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); @@ -1142,23 +1177,24 @@ public class ShardTest extends AbstractShardTest { waitUntilLeader(shard); - ShardDataTree dataStore = shard.underlyingActor().getDataStore(); + final ShardDataTree dataStore = shard.underlyingActor().getDataStore(); - DataTreeModification modification = dataStore.getDataTree().takeSnapshot().newModification(); + final DataTreeModification modification = dataStore.newModification(); - ContainerNode writeData = ImmutableNodes.containerNode(TestModel.TEST_QNAME); + final ContainerNode writeData = ImmutableNodes.containerNode(TestModel.TEST_QNAME); new WriteModification(TestModel.TEST_PATH, writeData).apply(modification); - MapNode mergeData = ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build(); + final MapNode mergeData = ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build(); new MergeModification(TestModel.OUTER_LIST_PATH, mergeData).apply(modification); - String txId = "tx1"; - ReadyLocalTransaction readyMessage = new ReadyLocalTransaction(txId, modification, true); + final String txId = "tx1"; + modification.ready(); + final ReadyLocalTransaction readyMessage = new ReadyLocalTransaction(txId, modification, true); shard.tell(readyMessage, getRef()); expectMsgClass(CommitTransactionReply.SERIALIZABLE_CLASS); - NormalizedNode actualNode = readStore(shard, TestModel.OUTER_LIST_PATH); + final NormalizedNode actualNode = readStore(shard, TestModel.OUTER_LIST_PATH); assertEquals(TestModel.OUTER_LIST_QNAME.getLocalName(), mergeData, actualNode); shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); @@ -1174,17 +1210,18 @@ public class ShardTest extends AbstractShardTest { waitUntilLeader(shard); - ShardDataTree dataStore = shard.underlyingActor().getDataStore(); + final ShardDataTree dataStore = shard.underlyingActor().getDataStore(); - DataTreeModification modification = dataStore.getDataTree().takeSnapshot().newModification(); + final DataTreeModification modification = dataStore.newModification(); - ContainerNode writeData = ImmutableNodes.containerNode(TestModel.TEST_QNAME); + final ContainerNode writeData = ImmutableNodes.containerNode(TestModel.TEST_QNAME); new WriteModification(TestModel.TEST_PATH, writeData).apply(modification); - MapNode mergeData = ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build(); + final MapNode mergeData = ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build(); new MergeModification(TestModel.OUTER_LIST_PATH, mergeData).apply(modification); - String txId = "tx1"; - ReadyLocalTransaction readyMessage = new ReadyLocalTransaction(txId, modification, false); + final String txId = "tx1"; + modification.ready(); + final ReadyLocalTransaction readyMessage = new ReadyLocalTransaction(txId, modification, false); shard.tell(readyMessage, getRef()); @@ -1193,7 +1230,7 @@ public class ShardTest extends AbstractShardTest { // Send the CanCommitTransaction message. shard.tell(new CanCommitTransaction(txId).toSerializable(), getRef()); - CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( + final CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( expectMsgClass(CanCommitTransactionReply.SERIALIZABLE_CLASS)); assertEquals("Can commit", true, canCommitReply.getCanCommit()); @@ -1202,7 +1239,7 @@ public class ShardTest extends AbstractShardTest { shard.tell(new CommitTransaction(txId).toSerializable(), getRef()); expectMsgClass(CommitTransactionReply.SERIALIZABLE_CLASS); - NormalizedNode actualNode = readStore(shard, TestModel.OUTER_LIST_PATH); + final NormalizedNode actualNode = readStore(shard, TestModel.OUTER_LIST_PATH); assertEquals(TestModel.OUTER_LIST_QNAME.getLocalName(), mergeData, actualNode); shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); @@ -1211,37 +1248,38 @@ public class ShardTest extends AbstractShardTest { @Test public void testCommitWithPersistenceDisabled() throws Throwable { + testCommitWithPersistenceDisabled(true); + testCommitWithPersistenceDisabled(false); + } + + public void testCommitWithPersistenceDisabled(final boolean readWrite) throws Throwable { dataStoreContextBuilder.persistent(false); new ShardTestKit(getSystem()) {{ final TestActorRef shard = TestActorRef.create(getSystem(), newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testCommitWithPersistenceDisabled"); + "testCommitWithPersistenceDisabled-" + readWrite); waitUntilLeader(shard); - ShardDataTree dataStore = shard.underlyingActor().getDataStore(); + final ShardDataTree dataStore = shard.underlyingActor().getDataStore(); // Setup a simulated transactions with a mock cohort. - String transactionID = "tx"; - MutableCompositeModification modification = new MutableCompositeModification(); - NormalizedNode containerNode = ImmutableNodes.containerNode(TestModel.TEST_QNAME); - ShardDataTreeCohort cohort = setupMockWriteTransaction("cohort", dataStore, - TestModel.TEST_PATH, containerNode, modification); - - FiniteDuration duration = duration("5 seconds"); + final String transactionID = "tx"; + final MutableCompositeModification modification = new MutableCompositeModification(); + final NormalizedNode containerNode = ImmutableNodes.containerNode(TestModel.TEST_QNAME); + final ShardDataTreeCohort cohort = setupMockWriteTransaction("cohort", dataStore, + TestModel.TEST_PATH, containerNode, modification); - // Simulate the ForwardedReadyTransaction messages that would be sent - // by the ShardTransaction. + final FiniteDuration duration = duration("5 seconds"); - shard.tell(new ForwardedReadyTransaction(transactionID, CURRENT_VERSION, - cohort, modification, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(readWrite, shard.underlyingActor(), cohort, transactionID, modification), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); // Send the CanCommitTransaction message. shard.tell(new CanCommitTransaction(transactionID).toSerializable(), getRef()); - CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( + final CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); assertEquals("Can commit", true, canCommitReply.getCanCommit()); @@ -1250,12 +1288,12 @@ public class ShardTest extends AbstractShardTest { shard.tell(new CommitTransaction(transactionID).toSerializable(), getRef()); expectMsgClass(duration, CommitTransactionReply.SERIALIZABLE_CLASS); - InOrder inOrder = inOrder(cohort); + final InOrder inOrder = inOrder(cohort); inOrder.verify(cohort).canCommit(); inOrder.verify(cohort).preCommit(); inOrder.verify(cohort).commit(); - NormalizedNode actualNode = readStore(shard, TestModel.TEST_PATH); + final NormalizedNode actualNode = readStore(shard, TestModel.TEST_PATH); assertEquals(TestModel.TEST_QNAME.getLocalName(), containerNode, actualNode); shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); @@ -1263,8 +1301,8 @@ public class ShardTest extends AbstractShardTest { } private static DataTreeCandidateTip mockCandidate(final String name) { - DataTreeCandidateTip mockCandidate = mock(DataTreeCandidateTip.class, name); - DataTreeCandidateNode mockCandidateNode = mock(DataTreeCandidateNode.class, name + "-node"); + final DataTreeCandidateTip mockCandidate = mock(DataTreeCandidateTip.class, name); + final DataTreeCandidateNode mockCandidateNode = mock(DataTreeCandidateNode.class, name + "-node"); doReturn(ModificationType.WRITE).when(mockCandidateNode).getModificationType(); doReturn(Optional.of(ImmutableNodes.containerNode(CARS_QNAME))).when(mockCandidateNode).getDataAfter(); doReturn(YangInstanceIdentifier.builder().build()).when(mockCandidate).getRootPath(); @@ -1273,8 +1311,8 @@ public class ShardTest extends AbstractShardTest { } private static DataTreeCandidateTip mockUnmodifiedCandidate(final String name) { - DataTreeCandidateTip mockCandidate = mock(DataTreeCandidateTip.class, name); - DataTreeCandidateNode mockCandidateNode = mock(DataTreeCandidateNode.class, name + "-node"); + final DataTreeCandidateTip mockCandidate = mock(DataTreeCandidateTip.class, name); + final DataTreeCandidateNode mockCandidateNode = mock(DataTreeCandidateNode.class, name + "-node"); doReturn(ModificationType.UNMODIFIED).when(mockCandidateNode).getModificationType(); doReturn(YangInstanceIdentifier.builder().build()).when(mockCandidate).getRootPath(); doReturn(mockCandidateNode).when(mockCandidate).getRootNode(); @@ -1282,51 +1320,52 @@ public class ShardTest extends AbstractShardTest { } @Test - public void testCommitWhenTransactionHasNoModifications(){ + public void testCommitWhenTransactionHasNoModifications() { + testCommitWhenTransactionHasNoModifications(true); + testCommitWhenTransactionHasNoModifications(false); + } + + public void testCommitWhenTransactionHasNoModifications(final boolean readWrite){ // Note that persistence is enabled which would normally result in the entry getting written to the journal // but here that need not happen new ShardTestKit(getSystem()) { { final TestActorRef shard = TestActorRef.create(getSystem(), newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testCommitWhenTransactionHasNoModifications"); + "testCommitWhenTransactionHasNoModifications-" + readWrite); waitUntilLeader(shard); - String transactionID = "tx1"; - MutableCompositeModification modification = new MutableCompositeModification(); - ShardDataTreeCohort cohort = mock(ShardDataTreeCohort.class, "cohort1"); + final String transactionID = "tx1"; + final MutableCompositeModification modification = new MutableCompositeModification(); + final ShardDataTreeCohort cohort = mock(ShardDataTreeCohort.class, "cohort1"); doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort).canCommit(); doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort).preCommit(); doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort).commit(); doReturn(mockUnmodifiedCandidate("cohort1-candidate")).when(cohort).getCandidate(); - FiniteDuration duration = duration("5 seconds"); + final FiniteDuration duration = duration("5 seconds"); - // Simulate the ForwardedReadyTransaction messages that would be sent - // by the ShardTransaction. - - shard.tell(new ForwardedReadyTransaction(transactionID, CURRENT_VERSION, - cohort, modification, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(readWrite, shard.underlyingActor(), cohort, transactionID, modification), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); // Send the CanCommitTransaction message. shard.tell(new CanCommitTransaction(transactionID).toSerializable(), getRef()); - CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( + final CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); assertEquals("Can commit", true, canCommitReply.getCanCommit()); shard.tell(new CommitTransaction(transactionID).toSerializable(), getRef()); expectMsgClass(duration, ThreePhaseCommitCohortMessages.CommitTransactionReply.class); - InOrder inOrder = inOrder(cohort); + final InOrder inOrder = inOrder(cohort); inOrder.verify(cohort).canCommit(); inOrder.verify(cohort).preCommit(); inOrder.verify(cohort).commit(); shard.tell(Shard.GET_SHARD_MBEAN_MESSAGE, getRef()); - ShardStats shardStats = expectMsgClass(duration, ShardStats.class); + final ShardStats shardStats = expectMsgClass(duration, ShardStats.class); // Use MBean for verification // Committed transaction count should increase as usual @@ -1342,50 +1381,51 @@ public class ShardTest extends AbstractShardTest { } @Test - public void testCommitWhenTransactionHasModifications(){ + public void testCommitWhenTransactionHasModifications() { + testCommitWhenTransactionHasModifications(true); + testCommitWhenTransactionHasModifications(false); + } + + public void testCommitWhenTransactionHasModifications(final boolean readWrite){ new ShardTestKit(getSystem()) { { final TestActorRef shard = TestActorRef.create(getSystem(), newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testCommitWhenTransactionHasModifications"); + "testCommitWhenTransactionHasModifications-" + readWrite); waitUntilLeader(shard); - String transactionID = "tx1"; - MutableCompositeModification modification = new MutableCompositeModification(); + final String transactionID = "tx1"; + final MutableCompositeModification modification = new MutableCompositeModification(); modification.addModification(new DeleteModification(YangInstanceIdentifier.builder().build())); - ShardDataTreeCohort cohort = mock(ShardDataTreeCohort.class, "cohort1"); + final ShardDataTreeCohort cohort = mock(ShardDataTreeCohort.class, "cohort1"); doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort).canCommit(); doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort).preCommit(); doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort).commit(); doReturn(mockCandidate("cohort1-candidate")).when(cohort).getCandidate(); - FiniteDuration duration = duration("5 seconds"); - - // Simulate the ForwardedReadyTransaction messages that would be sent - // by the ShardTransaction. + final FiniteDuration duration = duration("5 seconds"); - shard.tell(new ForwardedReadyTransaction(transactionID, CURRENT_VERSION, - cohort, modification, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(readWrite, shard.underlyingActor(), cohort, transactionID, modification), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); // Send the CanCommitTransaction message. shard.tell(new CanCommitTransaction(transactionID).toSerializable(), getRef()); - CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( + final CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); assertEquals("Can commit", true, canCommitReply.getCanCommit()); shard.tell(new CommitTransaction(transactionID).toSerializable(), getRef()); expectMsgClass(duration, ThreePhaseCommitCohortMessages.CommitTransactionReply.class); - InOrder inOrder = inOrder(cohort); + final InOrder inOrder = inOrder(cohort); inOrder.verify(cohort).canCommit(); inOrder.verify(cohort).preCommit(); inOrder.verify(cohort).commit(); shard.tell(Shard.GET_SHARD_MBEAN_MESSAGE, getRef()); - ShardStats shardStats = expectMsgClass(duration, ShardStats.class); + final ShardStats shardStats = expectMsgClass(duration, ShardStats.class); // Use MBean for verification // Committed transaction count should increase as usual @@ -1402,54 +1442,54 @@ public class ShardTest extends AbstractShardTest { @Test public void testCommitPhaseFailure() throws Throwable { + testCommitPhaseFailure(true); + testCommitPhaseFailure(false); + } + + public void testCommitPhaseFailure(final boolean readWrite) throws Throwable { new ShardTestKit(getSystem()) {{ final TestActorRef shard = TestActorRef.create(getSystem(), newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testCommitPhaseFailure"); + "testCommitPhaseFailure-" + readWrite); waitUntilLeader(shard); // Setup 2 simulated transactions with mock cohorts. The first one fails in the // commit phase. - String transactionID1 = "tx1"; - MutableCompositeModification modification1 = new MutableCompositeModification(); - ShardDataTreeCohort cohort1 = mock(ShardDataTreeCohort.class, "cohort1"); + final String transactionID1 = "tx1"; + final MutableCompositeModification modification1 = new MutableCompositeModification(); + final ShardDataTreeCohort cohort1 = mock(ShardDataTreeCohort.class, "cohort1"); doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort1).canCommit(); doReturn(Futures.immediateFuture(null)).when(cohort1).preCommit(); doReturn(Futures.immediateFailedFuture(new IllegalStateException("mock"))).when(cohort1).commit(); doReturn(mockCandidate("cohort1-candidate")).when(cohort1).getCandidate(); - String transactionID2 = "tx2"; - MutableCompositeModification modification2 = new MutableCompositeModification(); - ShardDataTreeCohort cohort2 = mock(ShardDataTreeCohort.class, "cohort2"); + final String transactionID2 = "tx2"; + final MutableCompositeModification modification2 = new MutableCompositeModification(); + final ShardDataTreeCohort cohort2 = mock(ShardDataTreeCohort.class, "cohort2"); doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort2).canCommit(); - FiniteDuration duration = duration("5 seconds"); + final FiniteDuration duration = duration("5 seconds"); final Timeout timeout = new Timeout(duration); - // Simulate the ForwardedReadyTransaction messages that would be sent - // by the ShardTransaction. - - shard.tell(new ForwardedReadyTransaction(transactionID1, CURRENT_VERSION, - cohort1, modification1, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(readWrite, shard.underlyingActor(), cohort1, transactionID1, modification1), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); - shard.tell(new ForwardedReadyTransaction(transactionID2, CURRENT_VERSION, - cohort2, modification2, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(readWrite, shard.underlyingActor(), cohort2, transactionID2, modification2), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); // Send the CanCommitTransaction message for the first Tx. shard.tell(new CanCommitTransaction(transactionID1).toSerializable(), getRef()); - CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( + final CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); assertEquals("Can commit", true, canCommitReply.getCanCommit()); // Send the CanCommitTransaction message for the 2nd Tx. This should get queued and // processed after the first Tx completes. - Future canCommitFuture = Patterns.ask(shard, + final Future canCommitFuture = Patterns.ask(shard, new CanCommitTransaction(transactionID2).toSerializable(), timeout); // Send the CommitTransaction message for the first Tx. This should send back an error @@ -1470,7 +1510,7 @@ public class ShardTest extends AbstractShardTest { assertEquals("2nd CanCommit complete", true, latch.await(5, TimeUnit.SECONDS)); - InOrder inOrder = inOrder(cohort1, cohort2); + final InOrder inOrder = inOrder(cohort1, cohort2); inOrder.verify(cohort1).canCommit(); inOrder.verify(cohort1).preCommit(); inOrder.verify(cohort1).commit(); @@ -1482,49 +1522,49 @@ public class ShardTest extends AbstractShardTest { @Test public void testPreCommitPhaseFailure() throws Throwable { + testPreCommitPhaseFailure(true); + testPreCommitPhaseFailure(false); + } + + public void testPreCommitPhaseFailure(final boolean readWrite) throws Throwable { new ShardTestKit(getSystem()) {{ final TestActorRef shard = TestActorRef.create(getSystem(), newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testPreCommitPhaseFailure"); + "testPreCommitPhaseFailure-" + readWrite); waitUntilLeader(shard); - String transactionID1 = "tx1"; - MutableCompositeModification modification1 = new MutableCompositeModification(); - ShardDataTreeCohort cohort1 = mock(ShardDataTreeCohort.class, "cohort1"); + final String transactionID1 = "tx1"; + final MutableCompositeModification modification1 = new MutableCompositeModification(); + final ShardDataTreeCohort cohort1 = mock(ShardDataTreeCohort.class, "cohort1"); doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort1).canCommit(); doReturn(Futures.immediateFailedFuture(new IllegalStateException("mock"))).when(cohort1).preCommit(); - String transactionID2 = "tx2"; - MutableCompositeModification modification2 = new MutableCompositeModification(); - ShardDataTreeCohort cohort2 = mock(ShardDataTreeCohort.class, "cohort2"); + final String transactionID2 = "tx2"; + final MutableCompositeModification modification2 = new MutableCompositeModification(); + final ShardDataTreeCohort cohort2 = mock(ShardDataTreeCohort.class, "cohort2"); doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort2).canCommit(); - FiniteDuration duration = duration("5 seconds"); + final FiniteDuration duration = duration("5 seconds"); final Timeout timeout = new Timeout(duration); - // Simulate the ForwardedReadyTransaction messages that would be sent - // by the ShardTransaction. - - shard.tell(new ForwardedReadyTransaction(transactionID1, CURRENT_VERSION, - cohort1, modification1, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(readWrite, shard.underlyingActor(), cohort1, transactionID1, modification1), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); - shard.tell(new ForwardedReadyTransaction(transactionID2, CURRENT_VERSION, - cohort2, modification2, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(readWrite, shard.underlyingActor(), cohort2, transactionID2, modification2), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); // Send the CanCommitTransaction message for the first Tx. shard.tell(new CanCommitTransaction(transactionID1).toSerializable(), getRef()); - CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( - expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); + final CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( + expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); assertEquals("Can commit", true, canCommitReply.getCanCommit()); // Send the CanCommitTransaction message for the 2nd Tx. This should get queued and // processed after the first Tx completes. - Future canCommitFuture = Patterns.ask(shard, + final Future canCommitFuture = Patterns.ask(shard, new CanCommitTransaction(transactionID2).toSerializable(), timeout); // Send the CommitTransaction message for the first Tx. This should send back an error @@ -1545,7 +1585,7 @@ public class ShardTest extends AbstractShardTest { assertEquals("2nd CanCommit complete", true, latch.await(5, TimeUnit.SECONDS)); - InOrder inOrder = inOrder(cohort1, cohort2); + final InOrder inOrder = inOrder(cohort1, cohort2); inOrder.verify(cohort1).canCommit(); inOrder.verify(cohort1).preCommit(); inOrder.verify(cohort2).canCommit(); @@ -1556,25 +1596,26 @@ public class ShardTest extends AbstractShardTest { @Test public void testCanCommitPhaseFailure() throws Throwable { + testCanCommitPhaseFailure(true); + testCanCommitPhaseFailure(false); + } + + public void testCanCommitPhaseFailure(final boolean readWrite) throws Throwable { new ShardTestKit(getSystem()) {{ final TestActorRef shard = TestActorRef.create(getSystem(), newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testCanCommitPhaseFailure"); + "testCanCommitPhaseFailure-" + readWrite); waitUntilLeader(shard); final FiniteDuration duration = duration("5 seconds"); - String transactionID1 = "tx1"; - MutableCompositeModification modification = new MutableCompositeModification(); - ShardDataTreeCohort cohort = mock(ShardDataTreeCohort.class, "cohort1"); + final String transactionID1 = "tx1"; + final MutableCompositeModification modification = new MutableCompositeModification(); + final ShardDataTreeCohort cohort = mock(ShardDataTreeCohort.class, "cohort1"); doReturn(Futures.immediateFailedFuture(new IllegalStateException("mock"))).when(cohort).canCommit(); - // Simulate the ForwardedReadyTransaction messages that would be sent - // by the ShardTransaction. - - shard.tell(new ForwardedReadyTransaction(transactionID1, CURRENT_VERSION, - cohort, modification, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(readWrite, shard.underlyingActor(), cohort, transactionID1, modification), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); // Send the CanCommitTransaction message. @@ -1586,15 +1627,14 @@ public class ShardTest extends AbstractShardTest { reset(cohort); - String transactionID2 = "tx2"; + final String transactionID2 = "tx2"; doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort).canCommit(); - shard.tell(new ForwardedReadyTransaction(transactionID2, CURRENT_VERSION, - cohort, modification, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(readWrite, shard.underlyingActor(), cohort, transactionID2, modification), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); shard.tell(new CanCommitTransaction(transactionID2).toSerializable(), getRef()); - CanCommitTransactionReply reply = CanCommitTransactionReply.fromSerializable( + final CanCommitTransactionReply reply = CanCommitTransactionReply.fromSerializable( expectMsgClass(CanCommitTransactionReply.SERIALIZABLE_CLASS)); assertEquals("getCanCommit", true, reply.getCanCommit()); @@ -1604,25 +1644,26 @@ public class ShardTest extends AbstractShardTest { @Test public void testCanCommitPhaseFalseResponse() throws Throwable { + testCanCommitPhaseFalseResponse(true); + testCanCommitPhaseFalseResponse(false); + } + + public void testCanCommitPhaseFalseResponse(final boolean readWrite) throws Throwable { new ShardTestKit(getSystem()) {{ final TestActorRef shard = TestActorRef.create(getSystem(), newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testCanCommitPhaseFalseResponse"); + "testCanCommitPhaseFalseResponse-" + readWrite); waitUntilLeader(shard); final FiniteDuration duration = duration("5 seconds"); - String transactionID1 = "tx1"; - MutableCompositeModification modification = new MutableCompositeModification(); - ShardDataTreeCohort cohort = mock(ShardDataTreeCohort.class, "cohort1"); + final String transactionID1 = "tx1"; + final MutableCompositeModification modification = new MutableCompositeModification(); + final ShardDataTreeCohort cohort = mock(ShardDataTreeCohort.class, "cohort1"); doReturn(Futures.immediateFuture(Boolean.FALSE)).when(cohort).canCommit(); - // Simulate the ForwardedReadyTransaction messages that would be sent - // by the ShardTransaction. - - shard.tell(new ForwardedReadyTransaction(transactionID1, CURRENT_VERSION, - cohort, modification, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(readWrite, shard.underlyingActor(), cohort, transactionID1, modification), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); // Send the CanCommitTransaction message. @@ -1636,11 +1677,10 @@ public class ShardTest extends AbstractShardTest { reset(cohort); - String transactionID2 = "tx2"; + final String transactionID2 = "tx2"; doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort).canCommit(); - shard.tell(new ForwardedReadyTransaction(transactionID2, CURRENT_VERSION, - cohort, modification, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(readWrite, shard.underlyingActor(), cohort, transactionID2, modification), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); shard.tell(new CanCommitTransaction(transactionID2).toSerializable(), getRef()); @@ -1654,25 +1694,26 @@ public class ShardTest extends AbstractShardTest { @Test public void testImmediateCommitWithCanCommitPhaseFailure() throws Throwable { + testImmediateCommitWithCanCommitPhaseFailure(true); + testImmediateCommitWithCanCommitPhaseFailure(false); + } + + public void testImmediateCommitWithCanCommitPhaseFailure(final boolean readWrite) throws Throwable { new ShardTestKit(getSystem()) {{ final TestActorRef shard = TestActorRef.create(getSystem(), newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testImmediateCommitWithCanCommitPhaseFailure"); + "testImmediateCommitWithCanCommitPhaseFailure-" + readWrite); waitUntilLeader(shard); final FiniteDuration duration = duration("5 seconds"); - String transactionID1 = "tx1"; - MutableCompositeModification modification = new MutableCompositeModification(); - ShardDataTreeCohort cohort = mock(ShardDataTreeCohort.class, "cohort1"); + final String transactionID1 = "tx1"; + final MutableCompositeModification modification = new MutableCompositeModification(); + final ShardDataTreeCohort cohort = mock(ShardDataTreeCohort.class, "cohort1"); doReturn(Futures.immediateFailedFuture(new IllegalStateException("mock"))).when(cohort).canCommit(); - // Simulate the ForwardedReadyTransaction messages that would be sent - // by the ShardTransaction. - - shard.tell(new ForwardedReadyTransaction(transactionID1, CURRENT_VERSION, - cohort, modification, true, true), getRef()); + shard.tell(prepareReadyTransactionMessage(readWrite, shard.underlyingActor(), cohort, transactionID1, modification, true), getRef()); expectMsgClass(duration, akka.actor.Status.Failure.class); @@ -1680,18 +1721,17 @@ public class ShardTest extends AbstractShardTest { reset(cohort); - String transactionID2 = "tx2"; + final String transactionID2 = "tx2"; doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort).canCommit(); doReturn(Futures.immediateFuture(null)).when(cohort).preCommit(); doReturn(Futures.immediateFuture(null)).when(cohort).commit(); - DataTreeCandidateTip candidate = mock(DataTreeCandidateTip.class); - DataTreeCandidateNode candidateRoot = mock(DataTreeCandidateNode.class); + final DataTreeCandidateTip candidate = mock(DataTreeCandidateTip.class); + final DataTreeCandidateNode candidateRoot = mock(DataTreeCandidateNode.class); doReturn(ModificationType.UNMODIFIED).when(candidateRoot).getModificationType(); doReturn(candidateRoot).when(candidate).getRootNode(); doReturn(candidate).when(cohort).getCandidate(); - shard.tell(new ForwardedReadyTransaction(transactionID2, CURRENT_VERSION, - cohort, modification, true, true), getRef()); + shard.tell(prepareReadyTransactionMessage(readWrite, shard.underlyingActor(), cohort, transactionID2, modification, true), getRef()); expectMsgClass(duration, CommitTransactionReply.SERIALIZABLE_CLASS); @@ -1701,25 +1741,26 @@ public class ShardTest extends AbstractShardTest { @Test public void testImmediateCommitWithCanCommitPhaseFalseResponse() throws Throwable { + testImmediateCommitWithCanCommitPhaseFalseResponse(true); + testImmediateCommitWithCanCommitPhaseFalseResponse(false); + } + + public void testImmediateCommitWithCanCommitPhaseFalseResponse(final boolean readWrite) throws Throwable { new ShardTestKit(getSystem()) {{ final TestActorRef shard = TestActorRef.create(getSystem(), newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testImmediateCommitWithCanCommitPhaseFalseResponse"); + "testImmediateCommitWithCanCommitPhaseFalseResponse-" + readWrite); waitUntilLeader(shard); final FiniteDuration duration = duration("5 seconds"); - String transactionID = "tx1"; - MutableCompositeModification modification = new MutableCompositeModification(); - ShardDataTreeCohort cohort = mock(ShardDataTreeCohort.class, "cohort1"); + final String transactionID = "tx1"; + final MutableCompositeModification modification = new MutableCompositeModification(); + final ShardDataTreeCohort cohort = mock(ShardDataTreeCohort.class, "cohort1"); doReturn(Futures.immediateFuture(Boolean.FALSE)).when(cohort).canCommit(); - // Simulate the ForwardedReadyTransaction messages that would be sent - // by the ShardTransaction. - - shard.tell(new ForwardedReadyTransaction(transactionID, CURRENT_VERSION, - cohort, modification, true, true), getRef()); + shard.tell(prepareReadyTransactionMessage(readWrite, shard.underlyingActor(), cohort, transactionID, modification, true), getRef()); expectMsgClass(duration, akka.actor.Status.Failure.class); @@ -1727,18 +1768,17 @@ public class ShardTest extends AbstractShardTest { reset(cohort); - String transactionID2 = "tx2"; + final String transactionID2 = "tx2"; doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort).canCommit(); doReturn(Futures.immediateFuture(null)).when(cohort).preCommit(); doReturn(Futures.immediateFuture(null)).when(cohort).commit(); - DataTreeCandidateTip candidate = mock(DataTreeCandidateTip.class); - DataTreeCandidateNode candidateRoot = mock(DataTreeCandidateNode.class); + final DataTreeCandidateTip candidate = mock(DataTreeCandidateTip.class); + final DataTreeCandidateNode candidateRoot = mock(DataTreeCandidateNode.class); doReturn(ModificationType.UNMODIFIED).when(candidateRoot).getModificationType(); doReturn(candidateRoot).when(candidate).getRootNode(); doReturn(candidate).when(cohort).getCandidate(); - shard.tell(new ForwardedReadyTransaction(transactionID2, CURRENT_VERSION, - cohort, modification, true, true), getRef()); + shard.tell(prepareReadyTransactionMessage(readWrite, shard.underlyingActor(), cohort, transactionID2, modification, true), getRef()); expectMsgClass(duration, CommitTransactionReply.SERIALIZABLE_CLASS); @@ -1748,22 +1788,27 @@ public class ShardTest extends AbstractShardTest { @Test public void testAbortBeforeFinishCommit() throws Throwable { + testAbortBeforeFinishCommit(true); + testAbortBeforeFinishCommit(false); + } + + public void testAbortBeforeFinishCommit(final boolean readWrite) throws Throwable { new ShardTestKit(getSystem()) {{ final TestActorRef shard = TestActorRef.create(getSystem(), newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testAbortBeforeFinishCommit"); + "testAbortBeforeFinishCommit-" + readWrite); waitUntilLeader(shard); final FiniteDuration duration = duration("5 seconds"); - ShardDataTree dataStore = shard.underlyingActor().getDataStore(); + final ShardDataTree dataStore = shard.underlyingActor().getDataStore(); final String transactionID = "tx1"; - Function> preCommit = + final Function> preCommit = new Function>() { @Override public ListenableFuture apply(final ShardDataTreeCohort cohort) { - ListenableFuture preCommitFuture = cohort.preCommit(); + final ListenableFuture preCommitFuture = cohort.preCommit(); // Simulate an AbortTransaction message occurring during replication, after // persisting and before finishing the commit to the in-memory store. @@ -1779,24 +1824,23 @@ public class ShardTest extends AbstractShardTest { } }; - MutableCompositeModification modification = new MutableCompositeModification(); - ShardDataTreeCohort cohort = setupMockWriteTransaction("cohort1", dataStore, + final MutableCompositeModification modification = new MutableCompositeModification(); + final ShardDataTreeCohort cohort = setupMockWriteTransaction("cohort1", dataStore, TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME), modification, preCommit); - shard.tell(new ForwardedReadyTransaction(transactionID, CURRENT_VERSION, - cohort, modification, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(readWrite, shard.underlyingActor(), cohort, transactionID, modification), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); shard.tell(new CanCommitTransaction(transactionID).toSerializable(), getRef()); - CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( - expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); + final CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( + expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); assertEquals("Can commit", true, canCommitReply.getCanCommit()); shard.tell(new CommitTransaction(transactionID).toSerializable(), getRef()); expectMsgClass(duration, CommitTransactionReply.SERIALIZABLE_CLASS); - NormalizedNode node = readStore(shard, TestModel.TEST_PATH); + final NormalizedNode node = readStore(shard, TestModel.TEST_PATH); // Since we're simulating an abort occurring during replication and before finish commit, // the data should still get written to the in-memory store since we've gotten past @@ -1809,18 +1853,23 @@ public class ShardTest extends AbstractShardTest { @Test public void testTransactionCommitTimeout() throws Throwable { + testTransactionCommitTimeout(true); + testTransactionCommitTimeout(false); + } + + public void testTransactionCommitTimeout(final boolean readWrite) throws Throwable { dataStoreContextBuilder.shardTransactionCommitTimeoutInSeconds(1); new ShardTestKit(getSystem()) {{ final TestActorRef shard = TestActorRef.create(getSystem(), newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testTransactionCommitTimeout"); + "testTransactionCommitTimeout-" + readWrite); waitUntilLeader(shard); final FiniteDuration duration = duration("5 seconds"); - ShardDataTree dataStore = shard.underlyingActor().getDataStore(); + final ShardDataTree dataStore = shard.underlyingActor().getDataStore(); writeToStore(shard, TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); writeToStore(shard, TestModel.OUTER_LIST_PATH, @@ -1828,9 +1877,9 @@ public class ShardTest extends AbstractShardTest { // Create 1st Tx - will timeout - String transactionID1 = "tx1"; - MutableCompositeModification modification1 = new MutableCompositeModification(); - ShardDataTreeCohort cohort1 = setupMockWriteTransaction("cohort1", dataStore, + final String transactionID1 = "tx1"; + final MutableCompositeModification modification1 = new MutableCompositeModification(); + final ShardDataTreeCohort cohort1 = setupMockWriteTransaction("cohort1", dataStore, YangInstanceIdentifier.builder(TestModel.OUTER_LIST_PATH) .nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1).build(), ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 1), @@ -1838,23 +1887,21 @@ public class ShardTest extends AbstractShardTest { // Create 2nd Tx - String transactionID2 = "tx3"; - MutableCompositeModification modification2 = new MutableCompositeModification(); - YangInstanceIdentifier listNodePath = YangInstanceIdentifier.builder(TestModel.OUTER_LIST_PATH) + final String transactionID2 = "tx3"; + final MutableCompositeModification modification2 = new MutableCompositeModification(); + final YangInstanceIdentifier listNodePath = YangInstanceIdentifier.builder(TestModel.OUTER_LIST_PATH) .nodeWithKey(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 2).build(); - ShardDataTreeCohort cohort2 = setupMockWriteTransaction("cohort3", dataStore, + final ShardDataTreeCohort cohort2 = setupMockWriteTransaction("cohort3", dataStore, listNodePath, ImmutableNodes.mapEntry(TestModel.OUTER_LIST_QNAME, TestModel.ID_QNAME, 2), modification2); // Ready the Tx's - shard.tell(new ForwardedReadyTransaction(transactionID1, CURRENT_VERSION, - cohort1, modification1, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(readWrite, shard.underlyingActor(), cohort1, transactionID1, modification1), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); - shard.tell(new ForwardedReadyTransaction(transactionID2, CURRENT_VERSION, - cohort2, modification2, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(readWrite, shard.underlyingActor(), cohort2, transactionID2, modification2), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); // canCommit 1st Tx. We don't send the commit so it should timeout. @@ -1877,7 +1924,7 @@ public class ShardTest extends AbstractShardTest { shard.tell(new CommitTransaction(transactionID2).toSerializable(), getRef()); expectMsgClass(duration, CommitTransactionReply.SERIALIZABLE_CLASS); - NormalizedNode node = readStore(shard, listNodePath); + final NormalizedNode node = readStore(shard, listNodePath); assertNotNull(listNodePath + " not found", node); shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); @@ -1897,39 +1944,36 @@ public class ShardTest extends AbstractShardTest { final FiniteDuration duration = duration("5 seconds"); - ShardDataTree dataStore = shard.underlyingActor().getDataStore(); + final ShardDataTree dataStore = shard.underlyingActor().getDataStore(); - String transactionID1 = "tx1"; - MutableCompositeModification modification1 = new MutableCompositeModification(); - ShardDataTreeCohort cohort1 = setupMockWriteTransaction("cohort1", dataStore, + final String transactionID1 = "tx1"; + final MutableCompositeModification modification1 = new MutableCompositeModification(); + final ShardDataTreeCohort cohort1 = setupMockWriteTransaction("cohort1", dataStore, TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME), modification1); - String transactionID2 = "tx2"; - MutableCompositeModification modification2 = new MutableCompositeModification(); - ShardDataTreeCohort cohort2 = setupMockWriteTransaction("cohort2", dataStore, + final String transactionID2 = "tx2"; + final MutableCompositeModification modification2 = new MutableCompositeModification(); + final ShardDataTreeCohort cohort2 = setupMockWriteTransaction("cohort2", dataStore, TestModel.OUTER_LIST_PATH, ImmutableNodes.mapNodeBuilder(TestModel.OUTER_LIST_QNAME).build(), modification2); - String transactionID3 = "tx3"; - MutableCompositeModification modification3 = new MutableCompositeModification(); - ShardDataTreeCohort cohort3 = setupMockWriteTransaction("cohort3", dataStore, + final String transactionID3 = "tx3"; + final MutableCompositeModification modification3 = new MutableCompositeModification(); + final ShardDataTreeCohort cohort3 = setupMockWriteTransaction("cohort3", dataStore, TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME), modification3); // Ready the Tx's - shard.tell(new ForwardedReadyTransaction(transactionID1, CURRENT_VERSION, - cohort1, modification1, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(false, shard.underlyingActor(), cohort1, transactionID1, modification1), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); - shard.tell(new ForwardedReadyTransaction(transactionID2, CURRENT_VERSION, - cohort2, modification2, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(false, shard.underlyingActor(), cohort2, transactionID2, modification2), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); // The 3rd Tx should exceed queue capacity and fail. - shard.tell(new ForwardedReadyTransaction(transactionID3, CURRENT_VERSION, - cohort3, modification3, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(false, shard.underlyingActor(), cohort3, transactionID3, modification3), getRef()); expectMsgClass(duration, akka.actor.Status.Failure.class); // canCommit 1st Tx. @@ -1963,33 +2007,30 @@ public class ShardTest extends AbstractShardTest { final FiniteDuration duration = duration("5 seconds"); - ShardDataTree dataStore = shard.underlyingActor().getDataStore(); + final ShardDataTree dataStore = shard.underlyingActor().getDataStore(); - String transactionID1 = "tx1"; - MutableCompositeModification modification1 = new MutableCompositeModification(); - ShardDataTreeCohort cohort1 = setupMockWriteTransaction("cohort1", dataStore, + final String transactionID1 = "tx1"; + final MutableCompositeModification modification1 = new MutableCompositeModification(); + final ShardDataTreeCohort cohort1 = setupMockWriteTransaction("cohort1", dataStore, TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME), modification1); - shard.tell(new ForwardedReadyTransaction(transactionID1, CURRENT_VERSION, - cohort1, modification1, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(false, shard.underlyingActor(), cohort1, transactionID1, modification1), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); - String transactionID2 = "tx2"; - MutableCompositeModification modification2 = new MutableCompositeModification(); - ShardDataTreeCohort cohort2 = setupMockWriteTransaction("cohort2", dataStore, + final String transactionID2 = "tx2"; + final MutableCompositeModification modification2 = new MutableCompositeModification(); + final ShardDataTreeCohort cohort2 = setupMockWriteTransaction("cohort2", dataStore, TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME), modification2); - shard.tell(new ForwardedReadyTransaction(transactionID2, CURRENT_VERSION, - cohort2, modification2, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(false, shard.underlyingActor(), cohort2, transactionID2, modification2), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); - String transactionID3 = "tx3"; - MutableCompositeModification modification3 = new MutableCompositeModification(); - ShardDataTreeCohort cohort3 = setupMockWriteTransaction("cohort3", dataStore, + final String transactionID3 = "tx3"; + final MutableCompositeModification modification3 = new MutableCompositeModification(); + final ShardDataTreeCohort cohort3 = setupMockWriteTransaction("cohort3", dataStore, TestModel.TEST2_PATH, ImmutableNodes.containerNode(TestModel.TEST2_QNAME), modification3); - shard.tell(new ForwardedReadyTransaction(transactionID3, CURRENT_VERSION, - cohort3, modification3, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(false, shard.underlyingActor(), cohort3, transactionID3, modification3), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); // All Tx's are readied. We'll send canCommit for the last one but not the others. The others @@ -2015,15 +2056,14 @@ public class ShardTest extends AbstractShardTest { final FiniteDuration duration = duration("5 seconds"); - ShardDataTree dataStore = shard.underlyingActor().getDataStore(); + final ShardDataTree dataStore = shard.underlyingActor().getDataStore(); - String transactionID1 = "tx1"; - MutableCompositeModification modification1 = new MutableCompositeModification(); - ShardDataTreeCohort cohort1 = setupMockWriteTransaction("cohort1", dataStore, + final String transactionID1 = "tx1"; + final MutableCompositeModification modification1 = new MutableCompositeModification(); + final ShardDataTreeCohort cohort1 = setupMockWriteTransaction("cohort1", dataStore, TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME), modification1); - shard.tell(new ForwardedReadyTransaction(transactionID1, CURRENT_VERSION, - cohort1, modification1, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(false, shard.underlyingActor(), cohort1, transactionID1, modification1), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); // CanCommit the first one so it's the current in-progress CohortEntry. @@ -2033,22 +2073,22 @@ public class ShardTest extends AbstractShardTest { // Ready the second Tx. - String transactionID2 = "tx2"; - MutableCompositeModification modification2 = new MutableCompositeModification(); - ShardDataTreeCohort cohort2 = setupMockWriteTransaction("cohort2", dataStore, + final String transactionID2 = "tx2"; + final MutableCompositeModification modification2 = new MutableCompositeModification(); + final ShardDataTreeCohort cohort2 = setupMockWriteTransaction("cohort2", dataStore, TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME), modification2); - shard.tell(new ForwardedReadyTransaction(transactionID2, CURRENT_VERSION, - cohort2, modification2, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(false, shard.underlyingActor(), cohort2, transactionID2, modification2), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); // Ready the third Tx. - String transactionID3 = "tx3"; - DataTreeModification modification3 = dataStore.getDataTree().takeSnapshot().newModification(); + final String transactionID3 = "tx3"; + final DataTreeModification modification3 = dataStore.newModification(); new WriteModification(TestModel.TEST2_PATH, ImmutableNodes.containerNode(TestModel.TEST2_QNAME)) .apply(modification3); - ReadyLocalTransaction readyMessage = new ReadyLocalTransaction(transactionID3, modification3, true); + modification3.ready(); + final ReadyLocalTransaction readyMessage = new ReadyLocalTransaction(transactionID3, modification3, true); shard.tell(readyMessage, getRef()); @@ -2062,7 +2102,7 @@ public class ShardTest extends AbstractShardTest { expectMsgClass(duration, CommitTransactionReply.SERIALIZABLE_CLASS); - NormalizedNode node = readStore(shard, TestModel.TEST2_PATH); + final NormalizedNode node = readStore(shard, TestModel.TEST2_PATH); assertNotNull(TestModel.TEST2_PATH + " not found", node); shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); @@ -2084,52 +2124,52 @@ public class ShardTest extends AbstractShardTest { } @Test - public void testAbortTransaction() throws Throwable { + public void testAbortCurrentTransaction() throws Throwable { + testAbortCurrentTransaction(true); + testAbortCurrentTransaction(false); + } + + public void testAbortCurrentTransaction(final boolean readWrite) throws Throwable { new ShardTestKit(getSystem()) {{ final TestActorRef shard = TestActorRef.create(getSystem(), newShardProps().withDispatcher(Dispatchers.DefaultDispatcherId()), - "testAbortTransaction"); + "testAbortCurrentTransaction-" + readWrite); waitUntilLeader(shard); // Setup 2 simulated transactions with mock cohorts. The first one will be aborted. - String transactionID1 = "tx1"; - MutableCompositeModification modification1 = new MutableCompositeModification(); - ShardDataTreeCohort cohort1 = mock(ShardDataTreeCohort.class, "cohort1"); + final String transactionID1 = "tx1"; + final MutableCompositeModification modification1 = new MutableCompositeModification(); + final ShardDataTreeCohort cohort1 = mock(ShardDataTreeCohort.class, "cohort1"); doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort1).canCommit(); doReturn(Futures.immediateFuture(null)).when(cohort1).abort(); - String transactionID2 = "tx2"; - MutableCompositeModification modification2 = new MutableCompositeModification(); - ShardDataTreeCohort cohort2 = mock(ShardDataTreeCohort.class, "cohort2"); + final String transactionID2 = "tx2"; + final MutableCompositeModification modification2 = new MutableCompositeModification(); + final ShardDataTreeCohort cohort2 = mock(ShardDataTreeCohort.class, "cohort2"); doReturn(Futures.immediateFuture(Boolean.TRUE)).when(cohort2).canCommit(); - FiniteDuration duration = duration("5 seconds"); + final FiniteDuration duration = duration("5 seconds"); final Timeout timeout = new Timeout(duration); - // Simulate the ForwardedReadyTransaction messages that would be sent - // by the ShardTransaction. - - shard.tell(new ForwardedReadyTransaction(transactionID1, CURRENT_VERSION, - cohort1, modification1, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(readWrite, shard.underlyingActor(), cohort1, transactionID1, modification1), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); - shard.tell(new ForwardedReadyTransaction(transactionID2, CURRENT_VERSION, - cohort2, modification2, true, false), getRef()); + shard.tell(prepareReadyTransactionMessage(readWrite, shard.underlyingActor(), cohort2, transactionID2, modification2), getRef()); expectMsgClass(duration, ReadyTransactionReply.class); // Send the CanCommitTransaction message for the first Tx. shard.tell(new CanCommitTransaction(transactionID1).toSerializable(), getRef()); - CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( + final CanCommitTransactionReply canCommitReply = CanCommitTransactionReply.fromSerializable( expectMsgClass(duration, CanCommitTransactionReply.SERIALIZABLE_CLASS)); assertEquals("Can commit", true, canCommitReply.getCanCommit()); // Send the CanCommitTransaction message for the 2nd Tx. This should get queued and // processed after the first Tx completes. - Future canCommitFuture = Patterns.ask(shard, + final Future canCommitFuture = Patterns.ask(shard, new CanCommitTransaction(transactionID2).toSerializable(), timeout); // Send the AbortTransaction message for the first Tx. This should trigger the 2nd @@ -2142,7 +2182,7 @@ public class ShardTest extends AbstractShardTest { Await.ready(canCommitFuture, duration); - InOrder inOrder = inOrder(cohort1, cohort2); + final InOrder inOrder = inOrder(cohort1, cohort2); inOrder.verify(cohort1).canCommit(); inOrder.verify(cohort2).canCommit(); @@ -2150,6 +2190,81 @@ public class ShardTest extends AbstractShardTest { }}; } + @Test + public void testAbortQueuedTransaction() throws Throwable { + testAbortQueuedTransaction(true); + testAbortQueuedTransaction(false); + } + + public void testAbortQueuedTransaction(final boolean readWrite) throws Throwable { + dataStoreContextBuilder.shardTransactionCommitTimeoutInSeconds(1); + new ShardTestKit(getSystem()) {{ + final AtomicReference cleaupCheckLatch = new AtomicReference<>(); + @SuppressWarnings("serial") + final Creator creator = new Creator() { + @Override + public Shard create() throws Exception { + return new Shard(newShardBuilder()) { + @Override + public void onReceiveCommand(final Object message) throws Exception { + super.onReceiveCommand(message); + if(message.equals(TX_COMMIT_TIMEOUT_CHECK_MESSAGE)) { + if(cleaupCheckLatch.get() != null) { + cleaupCheckLatch.get().countDown(); + } + } + } + }; + } + }; + + final TestActorRef shard = TestActorRef.create(getSystem(), + Props.create(new DelegatingShardCreator(creator)).withDispatcher( + Dispatchers.DefaultDispatcherId()), "testAbortQueuedTransaction-" + readWrite); + + waitUntilLeader(shard); + + final String transactionID = "tx1"; + + final MutableCompositeModification modification = new MutableCompositeModification(); + final ShardDataTreeCohort cohort = mock(ShardDataTreeCohort.class, "cohort"); + doReturn(Futures.immediateFuture(null)).when(cohort).abort(); + + final FiniteDuration duration = duration("5 seconds"); + + // Ready the tx. + + shard.tell(prepareReadyTransactionMessage(readWrite, shard.underlyingActor(), cohort, transactionID, modification), getRef()); + expectMsgClass(duration, ReadyTransactionReply.class); + + assertEquals("getPendingTxCommitQueueSize", 1, shard.underlyingActor().getPendingTxCommitQueueSize()); + + // Send the AbortTransaction message. + + shard.tell(new AbortTransaction(transactionID).toSerializable(), getRef()); + expectMsgClass(duration, AbortTransactionReply.SERIALIZABLE_CLASS); + + verify(cohort).abort(); + + // Verify the tx cohort is removed from queue at the cleanup check interval. + + cleaupCheckLatch.set(new CountDownLatch(1)); + assertEquals("TX_COMMIT_TIMEOUT_CHECK_MESSAGE received", true, + cleaupCheckLatch.get().await(5, TimeUnit.SECONDS)); + + assertEquals("getPendingTxCommitQueueSize", 0, shard.underlyingActor().getPendingTxCommitQueueSize()); + + // Now send CanCommitTransaction - should fail. + + shard.tell(new CanCommitTransaction(transactionID).toSerializable(), getRef()); + + Throwable failure = expectMsgClass(duration, akka.actor.Status.Failure.class).cause(); + assertTrue("Failure type", failure instanceof IllegalStateException); + + shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); + }}; + } + @Test public void testCreateSnapshot() throws Exception { testCreateSnapshot(true, "testCreateSnapshot"); @@ -2167,12 +2282,12 @@ public class ShardTest extends AbstractShardTest { final AtomicReference savedSnapshot = new AtomicReference<>(); class TestPersistentDataProvider extends DelegatingPersistentDataProvider { - TestPersistentDataProvider(DataPersistenceProvider delegate) { + TestPersistentDataProvider(final DataPersistenceProvider delegate) { super(delegate); } @Override - public void saveSnapshot(Object o) { + public void saveSnapshot(final Object o) { savedSnapshot.set(o); super.saveSnapshot(o); } @@ -2183,14 +2298,13 @@ public class ShardTest extends AbstractShardTest { new ShardTestKit(getSystem()) {{ class TestShard extends Shard { - protected TestShard(ShardIdentifier name, Map peerAddresses, - DatastoreContext datastoreContext, SchemaContext schemaContext) { - super(name, peerAddresses, datastoreContext, schemaContext); + protected TestShard(AbstractBuilder builder) { + super(builder); setPersistence(new TestPersistentDataProvider(super.persistence())); } @Override - public void handleCommand(Object message) { + public void handleCommand(final Object message) { super.handleCommand(message); if (message instanceof SaveSnapshotSuccess || message.equals("commit_snapshot")) { @@ -2204,55 +2318,53 @@ public class ShardTest extends AbstractShardTest { } } - Creator creator = new Creator() { + final Creator creator = new Creator() { @Override public Shard create() throws Exception { - return new TestShard(shardID, Collections.emptyMap(), - newDatastoreContext(), SCHEMA_CONTEXT); + return new TestShard(newShardBuilder()); } }; - TestActorRef shard = TestActorRef.create(getSystem(), + final TestActorRef shard = TestActorRef.create(getSystem(), Props.create(new DelegatingShardCreator(creator)), shardActorName); waitUntilLeader(shard); - writeToStore(shard, TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); - NormalizedNode expectedRoot = readStore(shard, YangInstanceIdentifier.builder().build()); + final NormalizedNode expectedRoot = readStore(shard, YangInstanceIdentifier.builder().build()); // Trigger creation of a snapshot by ensuring - RaftActorContext raftActorContext = ((TestShard) shard.underlyingActor()).getRaftActorContext(); + final RaftActorContext raftActorContext = ((TestShard) shard.underlyingActor()).getRaftActorContext(); raftActorContext.getSnapshotManager().capture(mock(ReplicatedLogEntry.class), -1); - - assertEquals("Snapshot saved", true, latch.get().await(5, TimeUnit.SECONDS)); - - assertTrue("Invalid saved snapshot " + savedSnapshot.get(), - savedSnapshot.get() instanceof Snapshot); - - verifySnapshot((Snapshot)savedSnapshot.get(), expectedRoot); - - latch.set(new CountDownLatch(1)); - savedSnapshot.set(null); + awaitAndValidateSnapshot(expectedRoot); raftActorContext.getSnapshotManager().capture(mock(ReplicatedLogEntry.class), -1); + awaitAndValidateSnapshot(expectedRoot); - assertEquals("Snapshot saved", true, latch.get().await(5, TimeUnit.SECONDS)); + shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); + } - assertTrue("Invalid saved snapshot " + savedSnapshot.get(), - savedSnapshot.get() instanceof Snapshot); + private void awaitAndValidateSnapshot(NormalizedNode expectedRoot + ) throws InterruptedException { + System.out.println("Inside awaitAndValidateSnapshot {}" + savedSnapshot.get()); + assertEquals("Snapshot saved", true, latch.get().await(5, TimeUnit.SECONDS)); - verifySnapshot((Snapshot)savedSnapshot.get(), expectedRoot); + assertTrue("Invalid saved snapshot " + savedSnapshot.get(), + savedSnapshot.get() instanceof Snapshot); - shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); - } + verifySnapshot((Snapshot)savedSnapshot.get(), expectedRoot); - private void verifySnapshot(Snapshot snapshot, NormalizedNode expectedRoot) { + latch.set(new CountDownLatch(1)); + savedSnapshot.set(null); + } - NormalizedNode actual = SerializationUtils.deserializeNormalizedNode(snapshot.getState()); - assertEquals("Root node", expectedRoot, actual); + private void verifySnapshot(final Snapshot snapshot, final NormalizedNode expectedRoot) { - }}; + final NormalizedNode actual = SerializationUtils.deserializeNormalizedNode(snapshot.getState()); + assertEquals("Root node", expectedRoot, actual); + + } + }; } /** @@ -2262,25 +2374,25 @@ public class ShardTest extends AbstractShardTest { */ @Test public void testInMemoryDataTreeRestore() throws ReadFailedException, DataValidationFailedException { - DataTree store = InMemoryDataTreeFactory.getInstance().create(); + final DataTree store = InMemoryDataTreeFactory.getInstance().create(TreeType.OPERATIONAL); store.setSchemaContext(SCHEMA_CONTEXT); - DataTreeModification putTransaction = store.takeSnapshot().newModification(); + final DataTreeModification putTransaction = store.takeSnapshot().newModification(); putTransaction.write(TestModel.TEST_PATH, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); commitTransaction(store, putTransaction); - NormalizedNode expected = readStore(store, YangInstanceIdentifier.builder().build()); + final NormalizedNode expected = readStore(store, YangInstanceIdentifier.builder().build()); - DataTreeModification writeTransaction = store.takeSnapshot().newModification(); + final DataTreeModification writeTransaction = store.takeSnapshot().newModification(); writeTransaction.delete(YangInstanceIdentifier.builder().build()); writeTransaction.write(YangInstanceIdentifier.builder().build(), expected); commitTransaction(store, writeTransaction); - NormalizedNode actual = readStore(store, YangInstanceIdentifier.builder().build()); + final NormalizedNode actual = readStore(store, YangInstanceIdentifier.builder().build()); assertEquals(expected, actual); } @@ -2291,24 +2403,24 @@ public class ShardTest extends AbstractShardTest { final DatastoreContext persistentContext = DatastoreContext.newBuilder(). shardJournalRecoveryLogBatchSize(3).shardSnapshotBatchCount(5000).persistent(true).build(); - final Props persistentProps = Shard.props(shardID, Collections.emptyMap(), - persistentContext, SCHEMA_CONTEXT); + final Props persistentProps = Shard.builder().id(shardID).datastoreContext(persistentContext). + schemaContext(SCHEMA_CONTEXT).props(); final DatastoreContext nonPersistentContext = DatastoreContext.newBuilder(). shardJournalRecoveryLogBatchSize(3).shardSnapshotBatchCount(5000).persistent(false).build(); - final Props nonPersistentProps = Shard.props(shardID, Collections.emptyMap(), - nonPersistentContext, SCHEMA_CONTEXT); + final Props nonPersistentProps = Shard.builder().id(shardID).datastoreContext(nonPersistentContext). + schemaContext(SCHEMA_CONTEXT).props(); new ShardTestKit(getSystem()) {{ - TestActorRef shard1 = TestActorRef.create(getSystem(), + final TestActorRef shard1 = TestActorRef.create(getSystem(), persistentProps, "testPersistence1"); assertTrue("Recovery Applicable", shard1.underlyingActor().persistence().isRecoveryApplicable()); shard1.tell(PoisonPill.getInstance(), ActorRef.noSender()); - TestActorRef shard2 = TestActorRef.create(getSystem(), + final TestActorRef shard2 = TestActorRef.create(getSystem(), nonPersistentProps, "testPersistence2"); assertFalse("Recovery Not Applicable", shard2.underlyingActor().persistence().isRecoveryApplicable()); @@ -2324,7 +2436,7 @@ public class ShardTest extends AbstractShardTest { new ShardTestKit(getSystem()) {{ dataStoreContextBuilder.persistent(true); - TestActorRef shard = TestActorRef.create(getSystem(), newShardProps(), "testOnDatastoreContext"); + final TestActorRef shard = TestActorRef.create(getSystem(), newShardProps(), "testOnDatastoreContext"); assertEquals("isRecoveryApplicable", true, shard.underlyingActor().persistence().isRecoveryApplicable()); @@ -2334,12 +2446,12 @@ public class ShardTest extends AbstractShardTest { shard.tell(dataStoreContextBuilder.persistent(false).build(), ActorRef.noSender()); assertEquals("isRecoveryApplicable", false, - shard.underlyingActor().persistence().isRecoveryApplicable()); + shard.underlyingActor().persistence().isRecoveryApplicable()); shard.tell(dataStoreContextBuilder.persistent(true).build(), ActorRef.noSender()); assertEquals("isRecoveryApplicable", true, - shard.underlyingActor().persistence().isRecoveryApplicable()); + shard.underlyingActor().persistence().isRecoveryApplicable()); shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); }}; @@ -2355,7 +2467,7 @@ public class ShardTest extends AbstractShardTest { waitUntilLeader(shard); - TestActorRef listener = + final TestActorRef listener = TestActorRef.create(getSystem(), Props.create(MessageCollectorActor.class)); shard.tell(new RegisterRoleChangeListener(), listener); @@ -2363,11 +2475,11 @@ public class ShardTest extends AbstractShardTest { MessageCollectorActor.expectFirstMatching(listener, RegisterRoleChangeListenerReply.class); ShardLeaderStateChanged leaderStateChanged = MessageCollectorActor.expectFirstMatching(listener, - ShardLeaderStateChanged.class); + ShardLeaderStateChanged.class); assertEquals("getLocalShardDataTree present", true, leaderStateChanged.getLocalShardDataTree().isPresent()); assertSame("getLocalShardDataTree", shard.underlyingActor().getDataStore().getDataTree(), - leaderStateChanged.getLocalShardDataTree().get()); + leaderStateChanged.getLocalShardDataTree().get()); MessageCollectorActor.clearMessages(listener); @@ -2402,9 +2514,172 @@ public class ShardTest extends AbstractShardTest { shard.tell(PoisonPill.getInstance(), ActorRef.noSender()); } - private static void commitTransaction(DataTree store, final DataTreeModification modification) throws DataValidationFailedException { + private static void commitTransaction(final DataTree store, final DataTreeModification modification) throws DataValidationFailedException { modification.ready(); store.validate(modification); store.commit(store.prepare(modification)); } + + @Test + public void testClusteredDataChangeListenerDelayedRegistration() throws Exception { + new ShardTestKit(getSystem()) {{ + String testName = "testClusteredDataChangeListenerDelayedRegistration"; + dataStoreContextBuilder.shardElectionTimeoutFactor(1000); + + final MockDataChangeListener listener = new MockDataChangeListener(1); + final ActorRef dclActor = actorFactory.createActor(DataChangeListener.props(listener), + actorFactory.generateActorId(testName + "-DataChangeListener")); + + final TestActorRef shard = actorFactory.createTestActor( + newShardBuilder().props().withDispatcher(Dispatchers.DefaultDispatcherId()), + actorFactory.generateActorId(testName + "-shard")); + + waitUntilNoLeader(shard); + + final YangInstanceIdentifier path = TestModel.TEST_PATH; + + shard.tell(new RegisterChangeListener(path, dclActor, AsyncDataBroker.DataChangeScope.BASE, true), getRef()); + final RegisterChangeListenerReply reply = expectMsgClass(duration("5 seconds"), + RegisterChangeListenerReply.class); + assertNotNull("getListenerRegistrationPath", reply.getListenerRegistrationPath()); + + writeToStore(shard, path, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); + + shard.tell(new ElectionTimeout(), ActorRef.noSender()); + + listener.waitForChangeEvents(); + }}; + } + + @Test + public void testClusteredDataChangeListenerRegistration() throws Exception { + new ShardTestKit(getSystem()) {{ + String testName = "testClusteredDataChangeListenerRegistration"; + final ShardIdentifier followerShardID = ShardIdentifier.builder().memberName( + actorFactory.generateActorId(testName + "-follower")).shardName("inventory").type("config").build(); + + final ShardIdentifier leaderShardID = ShardIdentifier.builder().memberName( + actorFactory.generateActorId(testName + "-leader")).shardName("inventory").type("config").build(); + + final TestActorRef followerShard = actorFactory.createTestActor( + Shard.builder().id(followerShardID). + datastoreContext(dataStoreContextBuilder.shardElectionTimeoutFactor(1000).build()). + peerAddresses(Collections.singletonMap(leaderShardID.toString(), + "akka://test/user/" + leaderShardID.toString())).schemaContext(SCHEMA_CONTEXT).props(). + withDispatcher(Dispatchers.DefaultDispatcherId()), followerShardID.toString()); + + final TestActorRef leaderShard = actorFactory.createTestActor( + Shard.builder().id(leaderShardID).datastoreContext(newDatastoreContext()). + peerAddresses(Collections.singletonMap(followerShardID.toString(), + "akka://test/user/" + followerShardID.toString())).schemaContext(SCHEMA_CONTEXT).props(). + withDispatcher(Dispatchers.DefaultDispatcherId()), leaderShardID.toString()); + + leaderShard.tell(new ElectionTimeout(), ActorRef.noSender()); + String leaderPath = waitUntilLeader(followerShard); + assertEquals("Shard leader path", leaderShard.path().toString(), leaderPath); + + final YangInstanceIdentifier path = TestModel.TEST_PATH; + final MockDataChangeListener listener = new MockDataChangeListener(1); + final ActorRef dclActor = actorFactory.createActor(DataChangeListener.props(listener), + actorFactory.generateActorId(testName + "-DataChangeListener")); + + followerShard.tell(new RegisterChangeListener(path, dclActor, AsyncDataBroker.DataChangeScope.BASE, true), getRef()); + final RegisterChangeListenerReply reply = expectMsgClass(duration("5 seconds"), + RegisterChangeListenerReply.class); + assertNotNull("getListenerRegistratioznPath", reply.getListenerRegistrationPath()); + + writeToStore(followerShard, path, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); + + listener.waitForChangeEvents(); + }}; + } + + @Test + public void testClusteredDataTreeChangeListenerDelayedRegistration() throws Exception { + new ShardTestKit(getSystem()) {{ + String testName = "testClusteredDataTreeChangeListenerDelayedRegistration"; + dataStoreContextBuilder.shardElectionTimeoutFactor(1000); + + final MockDataTreeChangeListener listener = new MockDataTreeChangeListener(1); + final ActorRef dclActor = actorFactory.createActor(DataTreeChangeListenerActor.props(listener), + actorFactory.generateActorId(testName + "-DataTreeChangeListener")); + + final TestActorRef shard = actorFactory.createTestActor( + newShardBuilder().props().withDispatcher(Dispatchers.DefaultDispatcherId()), + actorFactory.generateActorId(testName + "-shard")); + + waitUntilNoLeader(shard); + + final YangInstanceIdentifier path = TestModel.TEST_PATH; + + shard.tell(new RegisterDataTreeChangeListener(TestModel.TEST_PATH, dclActor, true), getRef()); + final RegisterDataTreeChangeListenerReply reply = expectMsgClass(duration("5 seconds"), + RegisterDataTreeChangeListenerReply.class); + assertNotNull("getListenerRegistrationPath", reply.getListenerRegistrationPath()); + + writeToStore(shard, path, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); + + shard.tell(new ElectionTimeout(), ActorRef.noSender()); + + listener.waitForChangeEvents(); + }}; + } + + @Test + public void testClusteredDataTreeChangeListenerRegistration() throws Exception { + new ShardTestKit(getSystem()) {{ + String testName = "testClusteredDataTreeChangeListenerRegistration"; + final ShardIdentifier followerShardID = ShardIdentifier.builder().memberName( + actorFactory.generateActorId(testName + "-follower")).shardName("inventory").type("config").build(); + + final ShardIdentifier leaderShardID = ShardIdentifier.builder().memberName( + actorFactory.generateActorId(testName + "-leader")).shardName("inventory").type("config").build(); + + final TestActorRef followerShard = actorFactory.createTestActor( + Shard.builder().id(followerShardID). + datastoreContext(dataStoreContextBuilder.shardElectionTimeoutFactor(1000).build()). + peerAddresses(Collections.singletonMap(leaderShardID.toString(), + "akka://test/user/" + leaderShardID.toString())).schemaContext(SCHEMA_CONTEXT).props(). + withDispatcher(Dispatchers.DefaultDispatcherId()), followerShardID.toString()); + + final TestActorRef leaderShard = actorFactory.createTestActor( + Shard.builder().id(leaderShardID).datastoreContext(newDatastoreContext()). + peerAddresses(Collections.singletonMap(followerShardID.toString(), + "akka://test/user/" + followerShardID.toString())).schemaContext(SCHEMA_CONTEXT).props(). + withDispatcher(Dispatchers.DefaultDispatcherId()), leaderShardID.toString()); + + leaderShard.tell(new ElectionTimeout(), ActorRef.noSender()); + String leaderPath = waitUntilLeader(followerShard); + assertEquals("Shard leader path", leaderShard.path().toString(), leaderPath); + + final YangInstanceIdentifier path = TestModel.TEST_PATH; + final MockDataTreeChangeListener listener = new MockDataTreeChangeListener(1); + final ActorRef dclActor = actorFactory.createActor(DataTreeChangeListenerActor.props(listener), + actorFactory.generateActorId(testName + "-DataTreeChangeListener")); + + followerShard.tell(new RegisterDataTreeChangeListener(TestModel.TEST_PATH, dclActor, true), getRef()); + final RegisterDataTreeChangeListenerReply reply = expectMsgClass(duration("5 seconds"), + RegisterDataTreeChangeListenerReply.class); + assertNotNull("getListenerRegistrationPath", reply.getListenerRegistrationPath()); + + writeToStore(followerShard, path, ImmutableNodes.containerNode(TestModel.TEST_QNAME)); + + listener.waitForChangeEvents(); + }}; + } + + @Test + public void testServerRemoved() throws Exception { + final TestActorRef parent = TestActorRef.create(getSystem(), MessageCollectorActor.props()); + + final ActorRef shard = parent.underlyingActor().context().actorOf( + newShardBuilder().props().withDispatcher(Dispatchers.DefaultDispatcherId()), + "testServerRemoved"); + + shard.tell(new ServerRemoved("test"), ActorRef.noSender()); + + MessageCollectorActor.expectFirstMatching(parent, ServerRemoved.class); + + } + }