Fix testLeaderAndFollowerEntityOwnersReassignedAfterShutdown failure

testLeaderAndFollowerEntityOwnersReassignedAfterShutdown(org.opendaylight.controller.cluster.datastore.entityownership.DistributedEntityOwnershipIntegrationTest)  Time elapsed: 10.834 sec  <<< FAILURE!
00:35:56 java.lang.AssertionError: Last index expected:<18> but was:<19>
00:35:56 	at org.junit.Assert.fail(Assert.java:88)
00:35:56 	at org.junit.Assert.failNotEquals(Assert.java:743)
00:35:56 	at org.junit.Assert.assertEquals(Assert.java:118)
00:35:56 	at org.junit.Assert.assertEquals(Assert.java:555)
00:35:56 	at org.opendaylight.controller.cluster.datastore.entityownership.DistributedEntityOwnershipIntegrationTest.lambda$testLeaderAndFollowerEntityOwnersReassignedAfterShutdown$7(DistributedEntityOwnershipIntegrationTest.java:456)
00:35:56 	at org.opendaylight.controller.cluster.datastore.MemberNode.verifyRaftState(MemberNode.java:147)
00:35:56 	at org.opendaylight.controller.cluster.datastore.entityownership.DistributedEntityOwnershipIntegrationTest.testLeaderAndFollowerEntityOwnersReassignedAfterShutdown(DistributedEntityOwnershipIntegrationTest.java:455)

The problem in this run is that lastIndex on the leader updated after
it was captured. It's difficult to know when all PurgeTransaction payloads
have been committed. The test tries to force follower1 to become leader
after the current leader is shut down but it doesn't really matter which
of the remaining followers becomes leader - we only care that the expected
entities get re-assigned. So I changed the test to enable elections on all
remaining followers and not expect a particular one to grab leadership.

Change-Id: I56a457db9ca6ba07f3a2a6378b745291c54ac99e
Signed-off-by: Tom Pantelis <tompantelis@gmail.com>

diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/DistributedEntityOwnershipIntegrationTest.java b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/DistributedEntityOwnershipIntegrationTest.java
index aaf4bc2de99e22ea041d9964a84bcff973ce06d9..6273e13dff49fd5e3c42987f08848b6cb07f632e 100644 (file)
--- a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/DistributedEntityOwnershipIntegrationTest.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/DistributedEntityOwnershipIntegrationTest.java
@@ -38,7 +38,6 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -53,7 +52,6 @@ import org.opendaylight.controller.cluster.datastore.IntegrationTestKit;
 import org.opendaylight.controller.cluster.datastore.MemberNode;
 import org.opendaylight.controller.cluster.datastore.entityownership.selectionstrategy.EntityOwnerSelectionStrategyConfig;
 import org.opendaylight.controller.cluster.datastore.messages.AddShardReplica;
-import org.opendaylight.controller.cluster.raft.RaftState;
 import org.opendaylight.controller.cluster.raft.policy.DisableElectionsRaftPolicy;
 import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal;
 import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore;
@@ -325,32 +323,23 @@ public class DistributedEntityOwnershipIntegrationTest {
         verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-1", "member-3");
         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
 
-        // Get the leader's lastIndex and verify followers are fully synced before shutting down the leader
+        // Re-enable elections on all remaining followers so one becomes the new leader
 
-        AtomicLong leaderLastIndex = new AtomicLong();
-        MemberNode.verifyRaftState(leaderDistributedDataStore, ENTITY_OWNERSHIP_SHARD_NAME,
-            raftState -> leaderLastIndex.set(raftState.getLastIndex()));
-
-        MemberNode.verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
-            raftState -> assertEquals("Last index", leaderLastIndex.get(), raftState.getLastIndex()));
+        ActorRef follower1Shard = IntegrationTestKit.findLocalShard(follower1Node.configDataStore().getActorContext(),
+                ENTITY_OWNERSHIP_SHARD_NAME);
+        follower1Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build())
+                .customRaftPolicyImplementation(null).build(), ActorRef.noSender());
 
-        MemberNode.verifyRaftState(follower2Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
-            raftState -> assertEquals("Last index", leaderLastIndex.get(), raftState.getLastIndex()));
+        ActorRef follower2Shard = IntegrationTestKit.findLocalShard(follower2Node.configDataStore().getActorContext(),
+                ENTITY_OWNERSHIP_SHARD_NAME);
+        follower2Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build())
+                .customRaftPolicyImplementation(null).build(), ActorRef.noSender());
 
         // Shutdown the leader and verify its removed from the candidate list
 
         leaderNode.cleanup();
         follower1Node.waitForMemberDown("member-1");
-
-        // Re-enable elections on follower1 so it becomes the leader
-
-        ActorRef follower1Shard = IntegrationTestKit.findLocalShard(follower1Node.configDataStore().getActorContext(),
-                ENTITY_OWNERSHIP_SHARD_NAME);
-        follower1Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build())
-                .customRaftPolicyImplementation(null).build(), ActorRef.noSender());
-
-        MemberNode.verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
-            raftState -> assertEquals("Raft state", RaftState.Leader.toString(), raftState.getRaftState()));
+        follower2Node.waitForMemberDown("member-1");
 
         // Verify the prior leader's entity owners are re-assigned.
 
@@ -441,19 +430,22 @@ public class DistributedEntityOwnershipIntegrationTest {
         verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-1", "member-3", "member-4");
         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
 
-        // Get the leader's lastIndex and verify followers are fully synced before shutting down the leader
-        AtomicLong leaderLastIndex = new AtomicLong();
-        MemberNode.verifyRaftState(leaderDistributedDataStore, ENTITY_OWNERSHIP_SHARD_NAME,
-            raftState -> leaderLastIndex.set(raftState.getLastIndex()));
+        // Re-enable elections on all remaining followers so one becomes the new leader
 
-        MemberNode.verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
-            raftState -> assertEquals("Last index", leaderLastIndex.get(), raftState.getLastIndex()));
+        ActorRef follower1Shard = IntegrationTestKit.findLocalShard(follower1Node.configDataStore().getActorContext(),
+                ENTITY_OWNERSHIP_SHARD_NAME);
+        follower1Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build())
+                .customRaftPolicyImplementation(null).build(), ActorRef.noSender());
 
-        MemberNode.verifyRaftState(follower2Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
-            raftState -> assertEquals("Last index", leaderLastIndex.get(), raftState.getLastIndex()));
+        ActorRef follower2Shard = IntegrationTestKit.findLocalShard(follower2Node.configDataStore().getActorContext(),
+                ENTITY_OWNERSHIP_SHARD_NAME);
+        follower2Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build())
+                .customRaftPolicyImplementation(null).build(), ActorRef.noSender());
 
-        MemberNode.verifyRaftState(follower4Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
-            raftState -> assertEquals("Last index", leaderLastIndex.get(), raftState.getLastIndex()));
+        ActorRef follower4Shard = IntegrationTestKit.findLocalShard(follower4Node.configDataStore().getActorContext(),
+                ENTITY_OWNERSHIP_SHARD_NAME);
+        follower4Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build())
+                .customRaftPolicyImplementation(null).build(), ActorRef.noSender());
 
         // Shutdown the leader and follower3
 
@@ -462,16 +454,10 @@ public class DistributedEntityOwnershipIntegrationTest {
 
         follower1Node.waitForMemberDown("member-1");
         follower1Node.waitForMemberDown("member-4");
-
-        // Re-enable elections on follower1 so it becomes the leader
-
-        ActorRef follower1Shard = IntegrationTestKit.findLocalShard(follower1Node.configDataStore().getActorContext(),
-                ENTITY_OWNERSHIP_SHARD_NAME);
-        follower1Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build())
-                .customRaftPolicyImplementation(null).build(), ActorRef.noSender());
-
-        MemberNode.verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
-            raftState -> assertEquals("Raft state", RaftState.Leader.toString(), raftState.getRaftState()));
+        follower2Node.waitForMemberDown("member-1");
+        follower2Node.waitForMemberDown("member-4");
+        follower4Node.waitForMemberDown("member-1");
+        follower4Node.waitForMemberDown("member-4");
 
         // Verify the prior leader's and follower3 entity owners are re-assigned.