import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
-import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
+import org.opendaylight.controller.cluster.access.concepts.TransactionIdentifier;
import org.opendaylight.controller.cluster.databroker.ConcurrentDOMDataBroker;
import org.opendaylight.controller.cluster.datastore.DatastoreContext.Builder;
-import org.opendaylight.controller.cluster.datastore.IntegrationTestKit.ShardStatsVerifier;
-import org.opendaylight.controller.cluster.datastore.MemberNode.RaftStateVerifier;
import org.opendaylight.controller.cluster.datastore.exceptions.NoShardLeaderException;
import org.opendaylight.controller.cluster.datastore.exceptions.ShardLeaderNotRespondingException;
-import org.opendaylight.controller.cluster.datastore.jmx.mbeans.shard.ShardStats;
import org.opendaylight.controller.cluster.datastore.messages.CommitTransactionReply;
import org.opendaylight.controller.cluster.datastore.messages.ForwardedReadyTransaction;
import org.opendaylight.controller.cluster.datastore.messages.GetShardDataTree;
import org.opendaylight.controller.cluster.datastore.messages.ReadyTransactionReply;
import org.opendaylight.controller.cluster.datastore.modification.MergeModification;
import org.opendaylight.controller.cluster.datastore.modification.WriteModification;
-import org.opendaylight.controller.cluster.raft.base.messages.ApplyJournalEntries;
-import org.opendaylight.controller.cluster.raft.client.messages.OnDemandRaftState;
import org.opendaylight.controller.cluster.raft.client.messages.Shutdown;
+import org.opendaylight.controller.cluster.raft.persisted.ApplyJournalEntries;
import org.opendaylight.controller.cluster.raft.policy.DisableElectionsRaftPolicy;
import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal;
import org.opendaylight.controller.md.cluster.datastore.model.CarsModel;
*
* @author Thomas Pantelis
*/
-public class DistributedDataStoreRemotingIntegrationTest {
+public class DistributedDataStoreRemotingIntegrationTest extends AbstractTest {
private static final String[] CARS_AND_PEOPLE = {"cars", "people"};
private static final String[] CARS = {"cars"};
private final DatastoreContext.Builder followerDatastoreContextBuilder =
DatastoreContext.newBuilder().shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(5).
customRaftPolicyImplementation(DisableElectionsRaftPolicy.class.getName());
+ private final TransactionIdentifier tx1 = nextTransactionId();
+ private final TransactionIdentifier tx2 = nextTransactionId();
private DistributedDataStore followerDistributedDataStore;
private DistributedDataStore leaderDistributedDataStore;
shardElectionTimeoutFactor(1).customRaftPolicyImplementation(null));
JavaTestKit.shutdownActorSystem(leaderSystem, null, true);
+ Cluster.get(followerSystem).leave(MEMBER_1_ADDRESS);
followerTestKit.waitUntilNoLeader(followerDistributedDataStore.getActorContext(), CARS);
new WriteModification(CarsModel.newCarPath("optima"), car1).apply(modification);
modification.ready();
- ReadyLocalTransaction readyLocal = new ReadyLocalTransaction("tx-1" , modification, true);
+ ReadyLocalTransaction readyLocal = new ReadyLocalTransaction(tx1 , modification, true);
carsFollowerShard.get().tell(readyLocal, followerTestKit.getRef());
Object resp = followerTestKit.expectMsgClass(Object.class);
new WriteModification(CarsModel.newCarPath("sportage"), car2).apply(modification);
modification.ready();
- readyLocal = new ReadyLocalTransaction("tx-2" , modification, false);
+ readyLocal = new ReadyLocalTransaction(tx2 , modification, false);
carsFollowerShard.get().tell(readyLocal, followerTestKit.getRef());
resp = followerTestKit.expectMsgClass(Object.class);
Mockito.doReturn(DataStoreVersions.CURRENT_VERSION).when(versionSupplier).get();
ThreePhaseCommitCohortProxy cohort = new ThreePhaseCommitCohortProxy(
leaderDistributedDataStore.getActorContext(), Arrays.asList(
- new ThreePhaseCommitCohortProxy.CohortInfo(Futures.successful(txActor), versionSupplier)), "tx-2");
+ new ThreePhaseCommitCohortProxy.CohortInfo(Futures.successful(txActor), versionSupplier)), tx2);
cohort.canCommit().get(5, TimeUnit.SECONDS);
cohort.preCommit().get(5, TimeUnit.SECONDS);
cohort.commit().get(5, TimeUnit.SECONDS);
MapEntryNode car1 = CarsModel.newCarEntry("optima", BigInteger.valueOf(20000));
new WriteModification(CarsModel.newCarPath("optima"), car1).apply(modification);
- ForwardedReadyTransaction forwardedReady = new ForwardedReadyTransaction("tx-1",
+ ForwardedReadyTransaction forwardedReady = new ForwardedReadyTransaction(tx1,
DataStoreVersions.CURRENT_VERSION, new ReadWriteShardDataTreeTransaction(
- Mockito.mock(ShardDataTreeTransactionParent.class), "tx-1", modification), true);
+ Mockito.mock(ShardDataTreeTransactionParent.class), tx1, modification), true);
carsFollowerShard.get().tell(forwardedReady, followerTestKit.getRef());
Object resp = followerTestKit.expectMsgClass(Object.class);
MapEntryNode car2 = CarsModel.newCarEntry("sportage", BigInteger.valueOf(30000));
new WriteModification(CarsModel.newCarPath("sportage"), car2).apply(modification);
- forwardedReady = new ForwardedReadyTransaction("tx-2",
+ forwardedReady = new ForwardedReadyTransaction(tx2,
DataStoreVersions.CURRENT_VERSION, new ReadWriteShardDataTreeTransaction(
- Mockito.mock(ShardDataTreeTransactionParent.class), "tx-2", modification), false);
+ Mockito.mock(ShardDataTreeTransactionParent.class), tx2, modification), false);
carsFollowerShard.get().tell(forwardedReady, followerTestKit.getRef());
resp = followerTestKit.expectMsgClass(Object.class);
Mockito.doReturn(DataStoreVersions.CURRENT_VERSION).when(versionSupplier).get();
ThreePhaseCommitCohortProxy cohort = new ThreePhaseCommitCohortProxy(
leaderDistributedDataStore.getActorContext(), Arrays.asList(
- new ThreePhaseCommitCohortProxy.CohortInfo(Futures.successful(txActor), versionSupplier)), "tx-2");
+ new ThreePhaseCommitCohortProxy.CohortInfo(Futures.successful(txActor), versionSupplier)), tx2);
cohort.canCommit().get(5, TimeUnit.SECONDS);
cohort.preCommit().get(5, TimeUnit.SECONDS);
cohort.commit().get(5, TimeUnit.SECONDS);
// Wait for the commit to be replicated to the follower.
- MemberNode.verifyRaftState(followerDistributedDataStore, "cars", new RaftStateVerifier() {
- @Override
- public void verify(OnDemandRaftState raftState) {
- assertEquals("getLastApplied", 0, raftState.getLastApplied());
- }
- });
+ MemberNode.verifyRaftState(followerDistributedDataStore, "cars",
+ raftState -> assertEquals("getLastApplied", 0, raftState.getLastApplied()));
+
+ MemberNode.verifyRaftState(followerDistributedDataStore, "people",
+ raftState -> assertEquals("getLastApplied", 0, raftState.getLastApplied()));
// Prepare, ready and canCommit a WO tx that writes to 2 shards. This will become the current tx in
// the leader shard.
cars.add(CarsModel.newCarEntry("car" + carIndex, BigInteger.valueOf(carIndex)));
readWriteTx.write(CarsModel.newCarPath("car" + carIndex), cars.getLast());
- IntegrationTestKit.verifyShardStats(leaderDistributedDataStore, "cars", new ShardStatsVerifier() {
- @Override
- public void verify(ShardStats stats) {
- assertEquals("getReadWriteTransactionCount", 1, stats.getReadWriteTransactionCount());
- }
- });
+ IntegrationTestKit.verifyShardStats(leaderDistributedDataStore, "cars",
+ stats -> assertEquals("getReadWriteTransactionCount", 1, stats.getReadWriteTransactionCount()));
// Disable elections on the leader so it switches to follower.
customRaftPolicyImplementation(DisableElectionsRaftPolicy.class.getName()).
shardElectionTimeoutFactor(10));
+ Cluster.get(followerSystem).leave(MEMBER_1_ADDRESS);
leaderTestKit.waitUntilNoLeader(leaderDistributedDataStore.getActorContext(), "cars");
// Submit all tx's - the messages should get queued for retry.
writeTx.write(PeopleModel.BASE_PATH, PeopleModel.emptyContainer());
DOMStoreThreePhaseCommitCohort cohort1 = writeTx.ready();
- IntegrationTestKit.verifyShardStats(leaderDistributedDataStore, "cars", new ShardStatsVerifier() {
- @Override
- public void verify(ShardStats stats) {
- assertEquals("getTxCohortCacheSize", 1, stats.getTxCohortCacheSize());
- }
- });
+ IntegrationTestKit.verifyShardStats(leaderDistributedDataStore, "cars",
+ stats -> assertEquals("getTxCohortCacheSize", 1, stats.getTxCohortCacheSize()));
writeTx = followerDistributedDataStore.newWriteOnlyTransaction();
MapEntryNode car = CarsModel.newCarEntry("optima", BigInteger.valueOf(20000));
writeTx.write(CarsModel.newCarPath("optima"), car);
DOMStoreThreePhaseCommitCohort cohort2 = writeTx.ready();
- IntegrationTestKit.verifyShardStats(leaderDistributedDataStore, "cars", new ShardStatsVerifier() {
- @Override
- public void verify(ShardStats stats) {
- assertEquals("getTxCohortCacheSize", 2, stats.getTxCohortCacheSize());
- }
- });
+ IntegrationTestKit.verifyShardStats(leaderDistributedDataStore, "cars",
+ stats -> assertEquals("getTxCohortCacheSize", 2, stats.getTxCohortCacheSize()));
// Gracefully stop the leader via a Shutdown message.
@Test
public void testTransactionWithIsolatedLeader() throws Throwable {
- leaderDatastoreContextBuilder.shardIsolatedLeaderCheckIntervalInMillis(200);
+ // Set the isolated leader check interval high so we can control the switch to IsolatedLeader.
+ leaderDatastoreContextBuilder.shardIsolatedLeaderCheckIntervalInMillis(10000000);
String testName = "testTransactionWithIsolatedLeader";
initDatastoresWithCars(testName);
- DOMStoreWriteTransaction failWriteTx = leaderDistributedDataStore.newWriteOnlyTransaction();
- failWriteTx.write(CarsModel.BASE_PATH, CarsModel.emptyContainer());
+ // Tx that is submitted after the follower is stopped but before the leader transitions to IsolatedLeader.
+ DOMStoreWriteTransaction preIsolatedLeaderWriteTx = leaderDistributedDataStore.newWriteOnlyTransaction();
+ preIsolatedLeaderWriteTx.write(CarsModel.BASE_PATH, CarsModel.emptyContainer());
+
+ // Tx that is submitted after the leader transitions to IsolatedLeader.
+ DOMStoreWriteTransaction noShardLeaderWriteTx = leaderDistributedDataStore.newWriteOnlyTransaction();
+ noShardLeaderWriteTx.write(CarsModel.BASE_PATH, CarsModel.emptyContainer());
+ // Tx that is submitted after the follower is reinstated.
DOMStoreWriteTransaction successWriteTx = leaderDistributedDataStore.newWriteOnlyTransaction();
successWriteTx.merge(CarsModel.BASE_PATH, CarsModel.emptyContainer());
+ // Stop the follower
followerTestKit.watch(followerDistributedDataStore.getActorContext().getShardManager());
followerDistributedDataStore.close();
followerTestKit.expectTerminated(followerDistributedDataStore.getActorContext().getShardManager());
- MemberNode.verifyRaftState(leaderDistributedDataStore, "cars", new RaftStateVerifier() {
- @Override
- public void verify(OnDemandRaftState raftState) {
- assertEquals("getRaftState", "IsolatedLeader", raftState.getRaftState());
- }
- });
+ // Submit the preIsolatedLeaderWriteTx so it's pending
+ DOMStoreThreePhaseCommitCohort preIsolatedLeaderTxCohort = preIsolatedLeaderWriteTx.ready();
+
+ // Change the isolated leader check interval low so it changes to IsolatedLeader.
+ sendDatastoreContextUpdate(leaderDistributedDataStore, leaderDatastoreContextBuilder.
+ shardIsolatedLeaderCheckIntervalInMillis(200));
+
+ MemberNode.verifyRaftState(leaderDistributedDataStore, "cars",
+ raftState -> assertEquals("getRaftState", "IsolatedLeader", raftState.getRaftState()));
try {
- leaderTestKit.doCommit(failWriteTx.ready());
+ leaderTestKit.doCommit(noShardLeaderWriteTx.ready());
fail("Expected NoShardLeaderException");
} catch (ExecutionException e) {
assertEquals("getCause", NoShardLeaderException.class, e.getCause().getClass());
sendDatastoreContextUpdate(leaderDistributedDataStore, leaderDatastoreContextBuilder.
shardElectionTimeoutFactor(100));
- DOMStoreThreePhaseCommitCohort writeTxCohort = successWriteTx.ready();
+ DOMStoreThreePhaseCommitCohort successTxCohort = successWriteTx.ready();
followerDistributedDataStore = followerTestKit.setupDistributedDataStore(testName,
MODULE_SHARDS_CARS_ONLY_1_2, false, CARS);
- leaderTestKit.doCommit(writeTxCohort);
+ leaderTestKit.doCommit(preIsolatedLeaderTxCohort);
+ leaderTestKit.doCommit(successTxCohort);
}
@Test(expected=AskTimeoutException.class)
JavaTestKit.shutdownActorSystem(leaderSystem, null, true);
+ Cluster.get(followerSystem).leave(MEMBER_1_ADDRESS);
+
Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
sendDatastoreContextUpdate(followerDistributedDataStore, followerDatastoreContextBuilder.
JavaTestKit.shutdownActorSystem(leaderSystem, null, true);
+ Cluster.get(followerSystem).leave(MEMBER_1_ADDRESS);
+
sendDatastoreContextUpdate(followerDistributedDataStore, followerDatastoreContextBuilder.
operationTimeoutInMillis(500).shardElectionTimeoutFactor(1).customRaftPolicyImplementation(null));
private static void sendDatastoreContextUpdate(DistributedDataStore dataStore, final Builder builder) {
final Builder newBuilder = DatastoreContext.newBuilderFrom(builder.build());
DatastoreContextFactory mockContextFactory = Mockito.mock(DatastoreContextFactory.class);
- Answer<DatastoreContext> answer = new Answer<DatastoreContext>() {
- @Override
- public DatastoreContext answer(InvocationOnMock invocation) {
- return newBuilder.build();
- }
- };
+ Answer<DatastoreContext> answer = invocation -> newBuilder.build();
Mockito.doAnswer(answer).when(mockContextFactory).getBaseDatastoreContext();
Mockito.doAnswer(answer).when(mockContextFactory).getShardDatastoreContext(Mockito.anyString());
dataStore.onDatastoreContextUpdated(mockContextFactory);
Code Review / controller.git / blobdiff