Fix intermittent test failures in PartitionedLeadersElectionScenarioTest

Seeing intermittent failures on jenkins, eg

Failed tests:
  PartitionedLeadersElectionScenarioTest.runTest1:37->setupInitialMemberBehaviors:313->AbstractLeaderElectionScenarioTest.initializeLeaderBehavior:207
Missing messages of type class
org.opendaylight.controller.cluster.raft.messages.AppendEntriesReply

Sometimes the initial AppendEntries messages go to dead letters,
probably b/c the follower actors haven't been fully created/initialized by akka.
So added retries as a workaround.

Change-Id: I5c838950f8ed2af3d5bc8ee3bd29602d8a8e8a9f
Signed-off-by: Tom Pantelis <tpanteli@brocade.com>

diff --git a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/AbstractLeaderElectionScenarioTest.java b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/AbstractLeaderElectionScenarioTest.java
index 13fd698cd9d7d07a117a0ecf45804fe22208225d..1f354beba114e9b2bf7caa29d12ab869dad092e3 100644 (file)
--- a/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/AbstractLeaderElectionScenarioTest.java
+++ b/opendaylight/md-sal/sal-akka-raft/src/test/java/org/opendaylight/controller/cluster/raft/behaviors/AbstractLeaderElectionScenarioTest.java
@@ -199,12 +199,29 @@ public class AbstractLeaderElectionScenarioTest {
 
     void initializeLeaderBehavior(MemberActor actor, RaftActorContext context, int numActiveFollowers) throws Exception {
         // Leader sends immediate heartbeats - we don't care about it so ignore it.
+        // Sometimes the initial AppendEntries messages go to dead letters, probably b/c the follower actors
+        // haven't been fully created/initialized by akka. So we try up to 3 times to create the Leader as
+        // a workaround.
 
-        actor.expectMessageClass(AppendEntriesReply.class, numActiveFollowers);
+        Leader leader = null;
+        AssertionError lastAssertError = null;
+        for(int i = 1; i <= 3; i++) {
+            actor.expectMessageClass(AppendEntriesReply.class, numActiveFollowers);
+
+            leader = new Leader(context);
+            try {
+                actor.waitForExpectedMessages(AppendEntriesReply.class);
+                lastAssertError = null;
+                break;
+            } catch (AssertionError e) {
+                lastAssertError = e;
+            }
+        }
+
+        if(lastAssertError != null) {
+            throw lastAssertError;
+        }
 
-        @SuppressWarnings("resource")
-        Leader leader = new Leader(context);
-        actor.waitForExpectedMessages(AppendEntriesReply.class);
         // Delay assignment here so the AppendEntriesReply isn't forwarded to the behavior.
         actor.behavior = leader;