Fix intermittent failures in DistributedDataStoreRemotingIntegrationTest

testTransactionRetryWithInitialAskTimeoutExOnCreateTx has failed a couple
times on jenkins. After leader shutdown it times out trying to elect a
new leader. This is b/c the time outs set are small for the test and it
sometimes runs quick enough before member-3 has actually joined the
cluster. So I added calls to wait for members to join the cluster before
running the test.

Change-Id: I4dabaffd3ace9082d46b27d78608df6d2f29734c
Signed-off-by: Tom Pantelis <tpanteli@brocade.com>

diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DistributedDataStoreRemotingIntegrationTest.java b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DistributedDataStoreRemotingIntegrationTest.java
index 903ffe78ea786fef7daf3c1625998fa1cfa5030e..132ac47e7da77de739c6fb1b50fb7d201ce81259 100644 (file)
--- a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DistributedDataStoreRemotingIntegrationTest.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DistributedDataStoreRemotingIntegrationTest.java
@@ -962,6 +962,9 @@ public class DistributedDataStoreRemotingIntegrationTest {
         IntegrationTestKit follower2TestKit = new IntegrationTestKit(follower2System, follower2DatastoreContextBuilder);
         follower2TestKit.setupDistributedDataStore(testName, MODULE_SHARDS_CARS_PEOPLE_1_2_3, false, CARS);
 
         IntegrationTestKit follower2TestKit = new IntegrationTestKit(follower2System, follower2DatastoreContextBuilder);
         follower2TestKit.setupDistributedDataStore(testName, MODULE_SHARDS_CARS_PEOPLE_1_2_3, false, CARS);
 

+        followerTestKit.waitForMembersUp("member-1", "member-3");
+        follower2TestKit.waitForMembersUp("member-1", "member-2");
+

         // Do an initial read to get the primary shard info cached.
 
         DOMStoreReadTransaction readTx = followerDistributedDataStore.newReadOnlyTransaction();
         // Do an initial read to get the primary shard info cached.
 
         DOMStoreReadTransaction readTx = followerDistributedDataStore.newReadOnlyTransaction();

diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/IntegrationTestKit.java b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/IntegrationTestKit.java
index c3d216be9ca9d144c51741194c06bb0550919549..feb8f7eaef6734c43c88e66cf0408160624a8d09 100644 (file)
--- a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/IntegrationTestKit.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/IntegrationTestKit.java
@@ -13,10 +13,16 @@ import static org.junit.Assert.fail;
 import akka.actor.ActorRef;
 import akka.actor.ActorSystem;
 import akka.actor.PoisonPill;
 import akka.actor.ActorRef;
 import akka.actor.ActorSystem;
 import akka.actor.PoisonPill;

+import akka.cluster.Cluster;
+import akka.cluster.ClusterEvent.CurrentClusterState;
+import akka.cluster.Member;
+import akka.cluster.MemberStatus;

 import com.google.common.base.Optional;
 import com.google.common.base.Stopwatch;
 import com.google.common.base.Optional;
 import com.google.common.base.Stopwatch;

+import com.google.common.collect.Sets;

 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.Uninterruptibles;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.Uninterruptibles;

+import java.util.Set;

 import java.util.concurrent.Callable;
 import java.util.concurrent.TimeUnit;
 import org.mockito.Mockito;
 import java.util.concurrent.Callable;
 import java.util.concurrent.TimeUnit;
 import org.mockito.Mockito;


@@ -112,6 +118,24 @@ public class IntegrationTestKit extends ShardTestKit {
         }
     }
 
         }
     }
 

+    public void waitForMembersUp(String... otherMembers) {
+        Set<String> otherMembersSet = Sets.newHashSet(otherMembers);
+        Stopwatch sw = Stopwatch.createStarted();
+        while(sw.elapsed(TimeUnit.SECONDS) <= 10) {
+            CurrentClusterState state = Cluster.get(getSystem()).state();
+            for(Member m: state.getMembers()) {
+                if(m.status() == MemberStatus.up() && otherMembersSet.remove(m.getRoles().iterator().next()) &&
+                        otherMembersSet.isEmpty()) {
+                    return;
+                }
+            }
+
+            Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
+        }
+
+        fail("Member(s) " + otherMembersSet + " are not Up");
+    }
+

     public static ActorRef findLocalShard(ActorContext actorContext, String shardName) {
         ActorRef shard = null;
         for(int i = 0; i < 20 * 5 && shard == null; i++) {
     public static ActorRef findLocalShard(ActorContext actorContext, String shardName) {
         ActorRef shard = null;
         for(int i = 0; i < 20 * 5 && shard == null; i++) {

diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/MemberNode.java b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/MemberNode.java
index 5ae53317f084f8b0f05e0023d23c581bf0e2c03e..01683130feaadb7c7ba102ad9c2799489d4af396 100644 (file)
--- a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/MemberNode.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/MemberNode.java
@@ -83,21 +83,7 @@ public class MemberNode {
     }
 
     public void waitForMembersUp(String... otherMembers) {
     }
 
     public void waitForMembersUp(String... otherMembers) {

-        Set<String> otherMembersSet = Sets.newHashSet(otherMembers);
-        Stopwatch sw = Stopwatch.createStarted();
-        while(sw.elapsed(TimeUnit.SECONDS) <= 10) {
-            CurrentClusterState state = Cluster.get(kit.getSystem()).state();
-            for(Member m: state.getMembers()) {
-                if(m.status() == MemberStatus.up() && otherMembersSet.remove(m.getRoles().iterator().next()) &&
-                        otherMembersSet.isEmpty()) {
-                    return;
-                }
-            }
-
-            Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
-        }
-
-        fail("Member(s) " + otherMembersSet + " are not Up");
+        kit.waitForMembersUp(otherMembers);

     }
 
     public void waitForMemberDown(String member) {
     }
 
     public void waitForMemberDown(String member) {