From: Tom Pantelis Date: Fri, 7 Jul 2017 21:13:59 +0000 (-0400) Subject: Speed up slow tell-based Distributed*IntegrationTest cases X-Git-Tag: release/nitrogen~57 X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?p=controller.git;a=commitdiff_plain;h=1402ac0fbcb87318db32b44f605621b23c94fbb3;hp=94362614be25e34e8427c02799daffb8cae29d94 Speed up slow tell-based Distributed*IntegrationTest cases Some test cases for tell-based take up to 2 minutes due to the default 2 minute request timeout and 30 sec backend aliveness timer. Speed up the tests by setting much lower values. Change-Id: If8dba80d625ded8753178e937f14b435675ef0e4 Signed-off-by: Tom Pantelis --- diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DistributedDataStoreIntegrationTest.java b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DistributedDataStoreIntegrationTest.java index 4a7a8134ad..bd9d683970 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DistributedDataStoreIntegrationTest.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DistributedDataStoreIntegrationTest.java @@ -40,7 +40,6 @@ import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import org.junit.After; import org.junit.Before; @@ -50,6 +49,7 @@ import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameter; import org.junit.runners.Parameterized.Parameters; import org.mockito.Mockito; +import org.opendaylight.controller.cluster.access.client.RequestTimeoutException; import org.opendaylight.controller.cluster.databroker.ClientBackedDataStore; import org.opendaylight.controller.cluster.databroker.ConcurrentDOMDataBroker; import org.opendaylight.controller.cluster.datastore.exceptions.NoShardLeaderException; @@ -649,7 +649,7 @@ public class DistributedDataStoreIntegrationTest { // The ShardManager uses the election timeout for FindPrimary so // reset it low so it will timeout quickly. datastoreContextBuilder.shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(1) - .shardInitializationTimeout(200, TimeUnit.MILLISECONDS); + .shardInitializationTimeout(200, TimeUnit.MILLISECONDS).frontendRequestTimeoutInSeconds(2); try (AbstractDataStore dataStore = setupAbstractDataStore( testParameter, testName, false, shardName)) { @@ -699,14 +699,16 @@ public class DistributedDataStoreIntegrationTest { // should have timed out and throw an appropriate // exception cause. try { - txCohort.get().canCommit().get(5, TimeUnit.SECONDS); + txCohort.get().canCommit().get(10, TimeUnit.SECONDS); fail("Expected NoShardLeaderException"); } catch (final ExecutionException e) { - assertTrue(Throwables.getRootCause(e) instanceof NoShardLeaderException); - assertEquals(DistributedDataStore.class, testParameter); - } catch (TimeoutException e) { - // ClientBackedDataStore doesn't set cause to ExecutionException, future just time outs - assertEquals(ClientBackedDataStore.class, testParameter); + final String msg = "Unexpected exception: " + + Throwables.getStackTraceAsString(e.getCause()); + if (DistributedDataStore.class.equals(testParameter)) { + assertTrue(Throwables.getRootCause(e) instanceof NoShardLeaderException); + } else { + assertTrue(msg, Throwables.getRootCause(e) instanceof RequestTimeoutException); + } } } finally { try { diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DistributedDataStoreRemotingIntegrationTest.java b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DistributedDataStoreRemotingIntegrationTest.java index dad3f1b223..a2a5e63a2b 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DistributedDataStoreRemotingIntegrationTest.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DistributedDataStoreRemotingIntegrationTest.java @@ -42,7 +42,6 @@ import java.util.LinkedList; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; import org.junit.After; import org.junit.Assume; @@ -54,6 +53,7 @@ import org.junit.runners.Parameterized.Parameter; import org.junit.runners.Parameterized.Parameters; import org.mockito.Mockito; import org.mockito.stubbing.Answer; +import org.opendaylight.controller.cluster.access.client.RequestTimeoutException; import org.opendaylight.controller.cluster.access.concepts.TransactionIdentifier; import org.opendaylight.controller.cluster.databroker.ClientBackedDataStore; import org.opendaylight.controller.cluster.databroker.ConcurrentDOMDataBroker; @@ -119,7 +119,7 @@ public class DistributedDataStoreRemotingIntegrationTest extends AbstractTest { @Parameters(name = "{0}") public static Collection data() { return Arrays.asList(new Object[][] { - { DistributedDataStore.class, 7}, { ClientBackedDataStore.class, 120 } + { DistributedDataStore.class, 7}, { ClientBackedDataStore.class, 12 } }); } @@ -139,6 +139,7 @@ public class DistributedDataStoreRemotingIntegrationTest extends AbstractTest { private static final String MODULE_SHARDS_CARS_ONLY_1_2 = "module-shards-cars-member-1-and-2.conf"; private static final String MODULE_SHARDS_CARS_PEOPLE_1_2 = "module-shards-member1-and-2.conf"; private static final String MODULE_SHARDS_CARS_PEOPLE_1_2_3 = "module-shards-member1-and-2-and-3.conf"; + private static final String MODULE_SHARDS_CARS_1_2_3 = "module-shards-cars-member-1-and-2-and-3.conf"; private ActorSystem leaderSystem; private ActorSystem followerSystem; @@ -576,6 +577,7 @@ public class DistributedDataStoreRemotingIntegrationTest extends AbstractTest { @Test public void testSingleShardTransactionsWithLeaderChanges() throws Exception { + followerDatastoreContextBuilder.backendAlivenessTimerIntervalInSeconds(2); final String testName = "testSingleShardTransactionsWithLeaderChanges"; initDatastoresWithCars(testName); @@ -999,6 +1001,7 @@ public class DistributedDataStoreRemotingIntegrationTest extends AbstractTest { @Test public void testTransactionWithShardLeaderNotResponding() throws Exception { + followerDatastoreContextBuilder.frontendRequestTimeoutInSeconds(2); followerDatastoreContextBuilder.shardElectionTimeoutFactor(50); initDatastoresWithCars("testTransactionWithShardLeaderNotResponding"); @@ -1023,17 +1026,18 @@ public class DistributedDataStoreRemotingIntegrationTest extends AbstractTest { fail("Exception expected"); } catch (final ExecutionException e) { final String msg = "Unexpected exception: " + Throwables.getStackTraceAsString(e.getCause()); - assertTrue(msg, Throwables.getRootCause(e) instanceof NoShardLeaderException - || e.getCause() instanceof ShardLeaderNotRespondingException); - assertEquals(DistributedDataStore.class, testParameter); - } catch (final TimeoutException e) { - // ClientBackedDataStore doesn't set cause to ExecutionException, future just time outs - assertEquals(ClientBackedDataStore.class, testParameter); + if (DistributedDataStore.class.equals(testParameter)) { + assertTrue(msg, Throwables.getRootCause(e) instanceof NoShardLeaderException + || e.getCause() instanceof ShardLeaderNotRespondingException); + } else { + assertTrue(msg, Throwables.getRootCause(e) instanceof RequestTimeoutException); + } } } @Test public void testTransactionWithCreateTxFailureDueToNoLeader() throws Exception { + followerDatastoreContextBuilder.frontendRequestTimeoutInSeconds(2); initDatastoresWithCars("testTransactionWithCreateTxFailureDueToNoLeader"); // Do an initial read to get the primary shard info cached. @@ -1060,29 +1064,29 @@ public class DistributedDataStoreRemotingIntegrationTest extends AbstractTest { followerTestKit.doCommit(rwTx.ready()); fail("Exception expected"); } catch (final ExecutionException e) { - final String msg = "Expected instance of NoShardLeaderException, actual: \n" - + Throwables.getStackTraceAsString(e.getCause()); - assertTrue(msg, Throwables.getRootCause(e) instanceof NoShardLeaderException); - assertEquals(DistributedDataStore.class, testParameter); - } catch (TimeoutException e) { - // ClientBackedDataStore doesn't set cause to ExecutionException, future just time outs - assertEquals(ClientBackedDataStore.class, testParameter); + final String msg = "Unexpected exception: " + Throwables.getStackTraceAsString(e.getCause()); + if (DistributedDataStore.class.equals(testParameter)) { + assertTrue(msg, Throwables.getRootCause(e) instanceof NoShardLeaderException); + } else { + assertTrue(msg, Throwables.getRootCause(e) instanceof RequestTimeoutException); + } } } @Test public void testTransactionRetryWithInitialAskTimeoutExOnCreateTx() throws Exception { + followerDatastoreContextBuilder.backendAlivenessTimerIntervalInSeconds(2); String testName = "testTransactionRetryWithInitialAskTimeoutExOnCreateTx"; - initDatastores(testName, MODULE_SHARDS_CARS_PEOPLE_1_2_3, CARS); + initDatastores(testName, MODULE_SHARDS_CARS_1_2_3, CARS); final DatastoreContext.Builder follower2DatastoreContextBuilder = DatastoreContext.newBuilder() - .shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(5); + .shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(10); final IntegrationTestKit follower2TestKit = new IntegrationTestKit( follower2System, follower2DatastoreContextBuilder, commitTimeout); try (AbstractDataStore ds = follower2TestKit.setupAbstractDataStore( - testParameter, testName, MODULE_SHARDS_CARS_PEOPLE_1_2_3, false, CARS)) { + testParameter, testName, MODULE_SHARDS_CARS_1_2_3, false, CARS)) { followerTestKit.waitForMembersUp("member-1", "member-3"); follower2TestKit.waitForMembersUp("member-1", "member-2"); @@ -1099,7 +1103,7 @@ public class DistributedDataStoreRemotingIntegrationTest extends AbstractTest { Cluster.get(followerSystem).leave(MEMBER_1_ADDRESS); sendDatastoreContextUpdate(followerDistributedDataStore, followerDatastoreContextBuilder - .operationTimeoutInMillis(500).shardElectionTimeoutFactor(1).customRaftPolicyImplementation(null)); + .operationTimeoutInMillis(500).shardElectionTimeoutFactor(5).customRaftPolicyImplementation(null)); final DOMStoreReadWriteTransaction rwTx = followerDistributedDataStore.newReadWriteTransaction(); diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/resources/module-shards-cars-member-1-and-2-and-3.conf b/opendaylight/md-sal/sal-distributed-datastore/src/test/resources/module-shards-cars-member-1-and-2-and-3.conf new file mode 100644 index 0000000000..9292e15981 --- /dev/null +++ b/opendaylight/md-sal/sal-distributed-datastore/src/test/resources/module-shards-cars-member-1-and-2-and-3.conf @@ -0,0 +1,15 @@ +module-shards = [ + { + name = "cars" + shards = [ + { + name="cars" + replicas = [ + "member-1", + "member-2", + "member-3" + ] + } + ] + } +]