Fix intermittent failures in DistributedDataStoreRemotingIntegrationTest

[controller.git] / opendaylight / md-sal / sal-distributed-datastore / src / test / java / org / opendaylight / controller / cluster / datastore / IntegrationTestKit.java

diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/IntegrationTestKit.java b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/IntegrationTestKit.java
index 9cc6d838c8c547a59384b97a63ea36fbb26e1909..feb8f7eaef6734c43c88e66cf0408160624a8d09 100644 (file)
--- a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/IntegrationTestKit.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/IntegrationTestKit.java
@@ -13,15 +13,23 @@ import static org.junit.Assert.fail;
 import akka.actor.ActorRef;
 import akka.actor.ActorSystem;
 import akka.actor.PoisonPill;
+import akka.cluster.Cluster;
+import akka.cluster.ClusterEvent.CurrentClusterState;
+import akka.cluster.Member;
+import akka.cluster.MemberStatus;
 import com.google.common.base.Optional;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.Sets;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.Uninterruptibles;
+import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.TimeUnit;
 import org.mockito.Mockito;
 import org.opendaylight.controller.cluster.datastore.DatastoreContext.Builder;
 import org.opendaylight.controller.cluster.datastore.config.Configuration;
 import org.opendaylight.controller.cluster.datastore.config.ConfigurationImpl;
+import org.opendaylight.controller.cluster.datastore.jmx.mbeans.shard.ShardStats;
 import org.opendaylight.controller.cluster.datastore.messages.DatastoreSnapshot;
 import org.opendaylight.controller.cluster.datastore.utils.ActorContext;
 import org.opendaylight.controller.md.cluster.datastore.model.SchemaContextHelper;
@@ -32,6 +40,9 @@ import org.opendaylight.controller.sal.core.spi.data.DOMStoreWriteTransaction;
 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
+import scala.concurrent.Await;
+import scala.concurrent.Future;
+import scala.concurrent.duration.Duration;
 
 public class IntegrationTestKit extends ShardTestKit {
 
@@ -68,7 +79,7 @@ public class IntegrationTestKit extends ShardTestKit {
         ClusterWrapper cluster = new ClusterWrapperImpl(getSystem());
         Configuration config = new ConfigurationImpl(moduleShardsConfig, "modules.conf");
 
-        datastoreContextBuilder.dataStoreType(typeName);
+        datastoreContextBuilder.dataStoreName(typeName);
 
         DatastoreContext datastoreContext = datastoreContextBuilder.build();
         DatastoreContextFactory mockContextFactory = Mockito.mock(DatastoreContextFactory.class);
@@ -92,7 +103,7 @@ public class IntegrationTestKit extends ShardTestKit {
         for(String shardName: shardNames) {
             ActorRef shard = findLocalShard(actorContext, shardName);
 
-            assertNotNull("Shard was not created", shard);
+            assertNotNull("Shard was not created for " + shardName, shard);
 
             waitUntilLeader(shard);
         }
@@ -101,13 +112,31 @@ public class IntegrationTestKit extends ShardTestKit {
     public void waitUntilNoLeader(ActorContext actorContext, String... shardNames) {
         for(String shardName: shardNames) {
             ActorRef shard = findLocalShard(actorContext, shardName);
-            assertNotNull("No local shard found", shard);
+            assertNotNull("No local shard found for " + shardName, shard);
 
             waitUntilNoLeader(shard);
         }
     }
 
-    private static ActorRef findLocalShard(ActorContext actorContext, String shardName) {
+    public void waitForMembersUp(String... otherMembers) {
+        Set<String> otherMembersSet = Sets.newHashSet(otherMembers);
+        Stopwatch sw = Stopwatch.createStarted();
+        while(sw.elapsed(TimeUnit.SECONDS) <= 10) {
+            CurrentClusterState state = Cluster.get(getSystem()).state();
+            for(Member m: state.getMembers()) {
+                if(m.status() == MemberStatus.up() && otherMembersSet.remove(m.getRoles().iterator().next()) &&
+                        otherMembersSet.isEmpty()) {
+                    return;
+                }
+            }
+
+            Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
+        }
+
+        fail("Member(s) " + otherMembersSet + " are not Up");
+    }
+
+    public static ActorRef findLocalShard(ActorContext actorContext, String shardName) {
         ActorRef shard = null;
         for(int i = 0; i < 20 * 5 && shard == null; i++) {
             Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
@@ -119,6 +148,31 @@ public class IntegrationTestKit extends ShardTestKit {
         return shard;
     }
 
+    public static void verifyShardStats(DistributedDataStore datastore, String shardName, ShardStatsVerifier verifier)
+            throws Exception {
+        ActorContext actorContext = datastore.getActorContext();
+
+        Future<ActorRef> future = actorContext.findLocalShardAsync(shardName);
+        ActorRef shardActor = Await.result(future, Duration.create(10, TimeUnit.SECONDS));
+
+        AssertionError lastError = null;
+        Stopwatch sw = Stopwatch.createStarted();
+        while(sw.elapsed(TimeUnit.SECONDS) <= 5) {
+            ShardStats shardStats = (ShardStats)actorContext.
+                    executeOperation(shardActor, Shard.GET_SHARD_MBEAN_MESSAGE);
+
+            try {
+                verifier.verify(shardStats);
+                return;
+            } catch (AssertionError e) {
+                lastError = e;
+                Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
+            }
+        }
+
+        throw lastError;
+    }
+
     void testWriteTransaction(DistributedDataStore dataStore, YangInstanceIdentifier nodePath,
             NormalizedNode<?, ?> nodeToWrite) throws Exception {
 
@@ -204,4 +258,8 @@ public class IntegrationTestKit extends ShardTestKit {
             }
         }, expType);
     }
+
+    public interface ShardStatsVerifier {
+        void verify(ShardStats stats);
+    }
 }