Fix intermittent LeaderTest/CandidateTest failures

[controller.git] / opendaylight / md-sal / sal-akka-raft / src / test / java / org / opendaylight / controller / cluster / raft / TestActorFactory.java

diff --git a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/TestActorFactory.java b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/TestActorFactory.java
index b47df13fed8c97297abf66c68bc837a03988dfb9..0b709aa038d5e12ad4a897c95a7aa5b448feb451 100644 (file)
--- a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/TestActorFactory.java
+++ b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/TestActorFactory.java
@@ -17,16 +17,27 @@ package org.opendaylight.controller.cluster.raft;
  */
 
 import akka.actor.Actor;
+import akka.actor.ActorIdentity;
 import akka.actor.ActorRef;
+import akka.actor.ActorSelection;
 import akka.actor.ActorSystem;
+import akka.actor.Identify;
 import akka.actor.PoisonPill;
 import akka.actor.Props;
+import akka.pattern.Patterns;
 import akka.testkit.JavaTestKit;
 import akka.testkit.TestActorRef;
+import akka.util.Timeout;
+import com.google.common.base.Stopwatch;
+import com.google.common.util.concurrent.Uninterruptibles;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.junit.Assert;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import scala.concurrent.Await;
+import scala.concurrent.Future;
 
 /**
  * TestActorFactory provides methods to create both normal and test actors and to kill them when the factory is closed
@@ -58,8 +69,7 @@ public class TestActorFactory implements AutoCloseable {
      */
     public ActorRef createActor(Props props){
         ActorRef actorRef = system.actorOf(props);
-        createdActors.add(actorRef);
-        return actorRef;
+        return addActor(actorRef);
     }
 
     /**
@@ -70,8 +80,7 @@ public class TestActorFactory implements AutoCloseable {
      */
     public ActorRef createActor(Props props, String actorId){
         ActorRef actorRef = system.actorOf(props, actorId);
-        createdActors.add(actorRef);
-        return actorRef;
+        return addActor(actorRef);
     }
 
     /**
@@ -81,22 +90,53 @@ public class TestActorFactory implements AutoCloseable {
      * @param <T>
      * @return
      */
+    @SuppressWarnings("unchecked")
     public <T extends Actor> TestActorRef<T> createTestActor(Props props, String actorId){
         TestActorRef<T> actorRef = TestActorRef.create(system, props, actorId);
+        return (TestActorRef<T>) addActor(actorRef);
+    }
+
+    private <T extends ActorRef> ActorRef addActor(T actorRef) {
         createdActors.add(actorRef);
+        verifyActorReady(actorRef);
         return actorRef;
     }
 
+    private void verifyActorReady(ActorRef actorRef) {
+        // Sometimes we see messages go to dead letters soon after creation - it seems the actor isn't quite
+        // in a state yet to receive messages or isn't actually created yet. This seems to happen with
+        // actorSelection so, to alleviate it, we use an actorSelection and send an Identify message with
+        // retries to ensure it's ready.
+
+        Timeout timeout = new Timeout(100, TimeUnit.MILLISECONDS);
+        Throwable lastError = null;
+        Stopwatch sw = Stopwatch.createStarted();
+        while(sw.elapsed(TimeUnit.SECONDS) <= 10) {
+            try {
+                ActorSelection actorSelection = system.actorSelection(actorRef.path().toString());
+                Future<Object> future = Patterns.ask(actorSelection, new Identify(""), timeout);
+                ActorIdentity reply = (ActorIdentity)Await.result(future, timeout.duration());
+                Assert.assertNotNull("Identify returned null", reply.getRef());
+                return;
+            } catch (Exception | AssertionError e) {
+                Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
+                lastError = e;
+            }
+        }
+
+        throw new RuntimeException(lastError);
+    }
+
     /**
      * Create a test actor with an auto-generated name
      * @param props
      * @param <T>
      * @return
      */
+    @SuppressWarnings("unchecked")
     public <T extends Actor> TestActorRef<T> createTestActor(Props props){
         TestActorRef<T> actorRef = TestActorRef.create(system, props);
-        createdActors.add(actorRef);
-        return actorRef;
+        return (TestActorRef<T>) addActor(actorRef);
     }
 
     /**
@@ -108,15 +148,30 @@ public class TestActorFactory implements AutoCloseable {
         return prefix + actorCount++;
     }
 
+    public void killActor(ActorRef actor, JavaTestKit kit) {
+        killActor(actor, kit, true);
+    }
+
+    public String createTestActorPath(String actorId){
+        return "akka://test/user/" + actorId;
+    }
+
+    private void killActor(ActorRef actor, JavaTestKit kit, boolean remove) {
+        LOG.info("Killing actor {}", actor);
+        kit.watch(actor);
+        actor.tell(PoisonPill.getInstance(), ActorRef.noSender());
+        kit.expectTerminated(JavaTestKit.duration("5 seconds"), actor);
+
+        if(remove) {
+            createdActors.remove(actor);
+        }
+    }
+
     @Override
     public void close() {
-        new JavaTestKit(system) {{
-            for(ActorRef actor : createdActors) {
-                watch(actor);
-                LOG.info("Killing actor {}", actor);
-                actor.tell(PoisonPill.getInstance(), ActorRef.noSender());
-                expectTerminated(duration("5 seconds"), actor);
-            }
-        }};
+        JavaTestKit kit = new JavaTestKit(system);
+        for(ActorRef actor : createdActors) {
+            killActor(actor, kit, false);
+        }
     }
 }
\ No newline at end of file