import org.junit.runners.Parameterized.Parameters;
import org.mockito.Mockito;
import org.mockito.stubbing.Answer;
+import org.opendaylight.controller.cluster.access.client.RequestTimeoutException;
import org.opendaylight.controller.cluster.access.concepts.TransactionIdentifier;
import org.opendaylight.controller.cluster.databroker.ClientBackedDataStore;
import org.opendaylight.controller.cluster.databroker.ConcurrentDOMDataBroker;
import org.opendaylight.controller.cluster.datastore.modification.WriteModification;
import org.opendaylight.controller.cluster.datastore.persisted.MetadataShardDataTreeSnapshot;
import org.opendaylight.controller.cluster.datastore.persisted.ShardSnapshotState;
+import org.opendaylight.controller.cluster.raft.base.messages.TimeoutNow;
import org.opendaylight.controller.cluster.raft.client.messages.Shutdown;
import org.opendaylight.controller.cluster.raft.persisted.ApplyJournalEntries;
import org.opendaylight.controller.cluster.raft.persisted.Snapshot;
@Parameters(name = "{0}")
public static Collection<Object[]> data() {
return Arrays.asList(new Object[][] {
- { DistributedDataStore.class, 7}, { ClientBackedDataStore.class, 60 }
+ { DistributedDataStore.class, 7}, { ClientBackedDataStore.class, 12 }
});
}
private static final String MODULE_SHARDS_CARS_ONLY_1_2 = "module-shards-cars-member-1-and-2.conf";
private static final String MODULE_SHARDS_CARS_PEOPLE_1_2 = "module-shards-member1-and-2.conf";
private static final String MODULE_SHARDS_CARS_PEOPLE_1_2_3 = "module-shards-member1-and-2-and-3.conf";
+ private static final String MODULE_SHARDS_CARS_1_2_3 = "module-shards-cars-member-1-and-2-and-3.conf";
private ActorSystem leaderSystem;
private ActorSystem followerSystem;
testParameter, type, moduleShardsConfig, false, shards);
leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(), shards);
+
+ leaderTestKit.waitForMembersUp("member-2");
+ followerTestKit.waitForMembersUp("member-1");
}
private static void verifyCars(final DOMStoreReadTransaction readTx, final MapEntryNode... entries)
@Test
public void testSingleShardTransactionsWithLeaderChanges() throws Exception {
+ followerDatastoreContextBuilder.backendAlivenessTimerIntervalInSeconds(2);
final String testName = "testSingleShardTransactionsWithLeaderChanges";
initDatastoresWithCars(testName);
.customRaftPolicyImplementation(DisableElectionsRaftPolicy.class.getName())
.shardElectionTimeoutFactor(10));
- Cluster.get(followerSystem).leave(MEMBER_1_ADDRESS);
leaderTestKit.waitUntilNoLeader(leaderDistributedDataStore.getActorContext(), "cars");
// Submit all tx's - the messages should get queued for retry.
sendDatastoreContextUpdate(followerDistributedDataStore, followerDatastoreContextBuilder
.customRaftPolicyImplementation(null).shardElectionTimeoutFactor(1));
+ IntegrationTestKit.findLocalShard(followerDistributedDataStore.getActorContext(), "cars")
+ .tell(TimeoutNow.INSTANCE, ActorRef.noSender());
+ IntegrationTestKit.findLocalShard(followerDistributedDataStore.getActorContext(), "people")
+ .tell(TimeoutNow.INSTANCE, ActorRef.noSender());
followerTestKit.doCommit(writeTx1CanCommit, writeTx1Cohort);
followerTestKit.doCommit(writeTx2CanCommit, writeTx2Cohort);
try (AbstractDataStore follower2DistributedDataStore = follower2TestKit.setupAbstractDataStore(
testParameter, testName, MODULE_SHARDS_CARS_PEOPLE_1_2_3, false)) {
+ followerTestKit.waitForMembersUp("member-3");
+ follower2TestKit.waitForMembersUp("member-1", "member-2");
+
// Create and submit a couple tx's so they're pending.
DOMStoreWriteTransaction writeTx = followerDistributedDataStore.newWriteOnlyTransaction();
@Test
public void testTransactionWithShardLeaderNotResponding() throws Exception {
+ followerDatastoreContextBuilder.frontendRequestTimeoutInSeconds(2);
followerDatastoreContextBuilder.shardElectionTimeoutFactor(50);
initDatastoresWithCars("testTransactionWithShardLeaderNotResponding");
fail("Exception expected");
} catch (final ExecutionException e) {
final String msg = "Unexpected exception: " + Throwables.getStackTraceAsString(e.getCause());
- assertTrue(msg, Throwables.getRootCause(e) instanceof NoShardLeaderException
- || e.getCause() instanceof ShardLeaderNotRespondingException);
+ if (DistributedDataStore.class.equals(testParameter)) {
+ assertTrue(msg, Throwables.getRootCause(e) instanceof NoShardLeaderException
+ || e.getCause() instanceof ShardLeaderNotRespondingException);
+ } else {
+ assertTrue(msg, Throwables.getRootCause(e) instanceof RequestTimeoutException);
+ }
}
}
@Test
public void testTransactionWithCreateTxFailureDueToNoLeader() throws Exception {
+ followerDatastoreContextBuilder.frontendRequestTimeoutInSeconds(2);
initDatastoresWithCars("testTransactionWithCreateTxFailureDueToNoLeader");
// Do an initial read to get the primary shard info cached.
followerTestKit.doCommit(rwTx.ready());
fail("Exception expected");
} catch (final ExecutionException e) {
- final String msg = "Expected instance of NoShardLeaderException, actual: \n"
- + Throwables.getStackTraceAsString(e.getCause());
- assertTrue(msg, Throwables.getRootCause(e) instanceof NoShardLeaderException);
+ final String msg = "Unexpected exception: " + Throwables.getStackTraceAsString(e.getCause());
+ if (DistributedDataStore.class.equals(testParameter)) {
+ assertTrue(msg, Throwables.getRootCause(e) instanceof NoShardLeaderException);
+ } else {
+ assertTrue(msg, Throwables.getRootCause(e) instanceof RequestTimeoutException);
+ }
}
}
@Test
public void testTransactionRetryWithInitialAskTimeoutExOnCreateTx() throws Exception {
+ followerDatastoreContextBuilder.backendAlivenessTimerIntervalInSeconds(2);
String testName = "testTransactionRetryWithInitialAskTimeoutExOnCreateTx";
- initDatastores(testName, MODULE_SHARDS_CARS_PEOPLE_1_2_3, CARS);
+ initDatastores(testName, MODULE_SHARDS_CARS_1_2_3, CARS);
final DatastoreContext.Builder follower2DatastoreContextBuilder = DatastoreContext.newBuilder()
- .shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(5);
+ .shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(10);
final IntegrationTestKit follower2TestKit = new IntegrationTestKit(
follower2System, follower2DatastoreContextBuilder, commitTimeout);
try (AbstractDataStore ds =
follower2TestKit.setupAbstractDataStore(
- testParameter, testName, MODULE_SHARDS_CARS_PEOPLE_1_2_3, false, CARS)) {
+ testParameter, testName, MODULE_SHARDS_CARS_1_2_3, false, CARS)) {
followerTestKit.waitForMembersUp("member-1", "member-3");
follower2TestKit.waitForMembersUp("member-1", "member-2");
Cluster.get(followerSystem).leave(MEMBER_1_ADDRESS);
sendDatastoreContextUpdate(followerDistributedDataStore, followerDatastoreContextBuilder
- .operationTimeoutInMillis(500).shardElectionTimeoutFactor(1).customRaftPolicyImplementation(null));
+ .operationTimeoutInMillis(500).shardElectionTimeoutFactor(5).customRaftPolicyImplementation(null));
final DOMStoreReadWriteTransaction rwTx = followerDistributedDataStore.newReadWriteTransaction();
initialSnapshot, snapshotRoot);
}
+ @Test
+ public void testLargeReadReplySlicing() throws Exception {
+ // The slicing is only implemented for tell-based protocol
+ Assume.assumeTrue(testParameter.equals(ClientBackedDataStore.class));
+
+ leaderDatastoreContextBuilder.maximumMessageSliceSize(50);
+ initDatastoresWithCars("testLargeReadReplySlicing");
+
+ final DOMStoreReadWriteTransaction rwTx = followerDistributedDataStore.newReadWriteTransaction();
+
+ final NormalizedNode<?, ?> carsNode = CarsModel.create();
+ rwTx.write(CarsModel.BASE_PATH, carsNode);
+
+ verifyNode(rwTx, CarsModel.BASE_PATH, carsNode);
+ }
+
private static void verifySnapshot(final Snapshot actual, final Snapshot expected,
final NormalizedNode<?, ?> expRoot) {
assertEquals("Snapshot getLastAppliedTerm", expected.getLastAppliedTerm(), actual.getLastAppliedTerm());