Fix testLeaderAndFollowerEntityOwnersReassignedAfterShutdown failure 55/63155/3
authorTom Pantelis <tompantelis@gmail.com>
Thu, 14 Sep 2017 18:50:00 +0000 (14:50 -0400)
committerRobert Varga <nite@hq.sk>
Mon, 18 Sep 2017 13:04:13 +0000 (13:04 +0000)
testLeaderAndFollowerEntityOwnersReassignedAfterShutdown(org.opendaylight.controller.cluster.datastore.entityownership.DistributedEntityOwnershipIntegrationTest)  Time elapsed: 10.834 sec  <<< FAILURE!
00:35:56 java.lang.AssertionError: Last index expected:<18> but was:<19>
00:35:56  at org.junit.Assert.fail(Assert.java:88)
00:35:56  at org.junit.Assert.failNotEquals(Assert.java:743)
00:35:56  at org.junit.Assert.assertEquals(Assert.java:118)
00:35:56  at org.junit.Assert.assertEquals(Assert.java:555)
00:35:56  at org.opendaylight.controller.cluster.datastore.entityownership.DistributedEntityOwnershipIntegrationTest.lambda$testLeaderAndFollowerEntityOwnersReassignedAfterShutdown$7(DistributedEntityOwnershipIntegrationTest.java:456)
00:35:56  at org.opendaylight.controller.cluster.datastore.MemberNode.verifyRaftState(MemberNode.java:147)
00:35:56  at org.opendaylight.controller.cluster.datastore.entityownership.DistributedEntityOwnershipIntegrationTest.testLeaderAndFollowerEntityOwnersReassignedAfterShutdown(DistributedEntityOwnershipIntegrationTest.java:455)

The problem in this run is that lastIndex on the leader updated after
it was captured. It's difficult to know when all PurgeTransaction payloads
have been committed. The test tries to force follower1 to become leader
after the current leader is shut down but it doesn't really matter which
of the remaining followers becomes leader - we only care that the expected
entities get re-assigned. So I changed the test to enable elections on all
remaining followers and not expect a particular one to grab leadership.

Change-Id: I56a457db9ca6ba07f3a2a6378b745291c54ac99e
Signed-off-by: Tom Pantelis <tompantelis@gmail.com>
opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/DistributedEntityOwnershipIntegrationTest.java

index aaf4bc2..6273e13 100644 (file)
@@ -38,7 +38,6 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -53,7 +52,6 @@ import org.opendaylight.controller.cluster.datastore.IntegrationTestKit;
 import org.opendaylight.controller.cluster.datastore.MemberNode;
 import org.opendaylight.controller.cluster.datastore.entityownership.selectionstrategy.EntityOwnerSelectionStrategyConfig;
 import org.opendaylight.controller.cluster.datastore.messages.AddShardReplica;
-import org.opendaylight.controller.cluster.raft.RaftState;
 import org.opendaylight.controller.cluster.raft.policy.DisableElectionsRaftPolicy;
 import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal;
 import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore;
@@ -325,32 +323,23 @@ public class DistributedEntityOwnershipIntegrationTest {
         verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-1", "member-3");
         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
 
-        // Get the leader's lastIndex and verify followers are fully synced before shutting down the leader
+        // Re-enable elections on all remaining followers so one becomes the new leader
 
-        AtomicLong leaderLastIndex = new AtomicLong();
-        MemberNode.verifyRaftState(leaderDistributedDataStore, ENTITY_OWNERSHIP_SHARD_NAME,
-            raftState -> leaderLastIndex.set(raftState.getLastIndex()));
-
-        MemberNode.verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
-            raftState -> assertEquals("Last index", leaderLastIndex.get(), raftState.getLastIndex()));
+        ActorRef follower1Shard = IntegrationTestKit.findLocalShard(follower1Node.configDataStore().getActorContext(),
+                ENTITY_OWNERSHIP_SHARD_NAME);
+        follower1Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build())
+                .customRaftPolicyImplementation(null).build(), ActorRef.noSender());
 
-        MemberNode.verifyRaftState(follower2Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
-            raftState -> assertEquals("Last index", leaderLastIndex.get(), raftState.getLastIndex()));
+        ActorRef follower2Shard = IntegrationTestKit.findLocalShard(follower2Node.configDataStore().getActorContext(),
+                ENTITY_OWNERSHIP_SHARD_NAME);
+        follower2Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build())
+                .customRaftPolicyImplementation(null).build(), ActorRef.noSender());
 
         // Shutdown the leader and verify its removed from the candidate list
 
         leaderNode.cleanup();
         follower1Node.waitForMemberDown("member-1");
-
-        // Re-enable elections on follower1 so it becomes the leader
-
-        ActorRef follower1Shard = IntegrationTestKit.findLocalShard(follower1Node.configDataStore().getActorContext(),
-                ENTITY_OWNERSHIP_SHARD_NAME);
-        follower1Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build())
-                .customRaftPolicyImplementation(null).build(), ActorRef.noSender());
-
-        MemberNode.verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
-            raftState -> assertEquals("Raft state", RaftState.Leader.toString(), raftState.getRaftState()));
+        follower2Node.waitForMemberDown("member-1");
 
         // Verify the prior leader's entity owners are re-assigned.
 
@@ -441,19 +430,22 @@ public class DistributedEntityOwnershipIntegrationTest {
         verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-1", "member-3", "member-4");
         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
 
-        // Get the leader's lastIndex and verify followers are fully synced before shutting down the leader
-        AtomicLong leaderLastIndex = new AtomicLong();
-        MemberNode.verifyRaftState(leaderDistributedDataStore, ENTITY_OWNERSHIP_SHARD_NAME,
-            raftState -> leaderLastIndex.set(raftState.getLastIndex()));
+        // Re-enable elections on all remaining followers so one becomes the new leader
 
-        MemberNode.verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
-            raftState -> assertEquals("Last index", leaderLastIndex.get(), raftState.getLastIndex()));
+        ActorRef follower1Shard = IntegrationTestKit.findLocalShard(follower1Node.configDataStore().getActorContext(),
+                ENTITY_OWNERSHIP_SHARD_NAME);
+        follower1Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build())
+                .customRaftPolicyImplementation(null).build(), ActorRef.noSender());
 
-        MemberNode.verifyRaftState(follower2Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
-            raftState -> assertEquals("Last index", leaderLastIndex.get(), raftState.getLastIndex()));
+        ActorRef follower2Shard = IntegrationTestKit.findLocalShard(follower2Node.configDataStore().getActorContext(),
+                ENTITY_OWNERSHIP_SHARD_NAME);
+        follower2Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build())
+                .customRaftPolicyImplementation(null).build(), ActorRef.noSender());
 
-        MemberNode.verifyRaftState(follower4Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
-            raftState -> assertEquals("Last index", leaderLastIndex.get(), raftState.getLastIndex()));
+        ActorRef follower4Shard = IntegrationTestKit.findLocalShard(follower4Node.configDataStore().getActorContext(),
+                ENTITY_OWNERSHIP_SHARD_NAME);
+        follower4Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build())
+                .customRaftPolicyImplementation(null).build(), ActorRef.noSender());
 
         // Shutdown the leader and follower3
 
@@ -462,16 +454,10 @@ public class DistributedEntityOwnershipIntegrationTest {
 
         follower1Node.waitForMemberDown("member-1");
         follower1Node.waitForMemberDown("member-4");
-
-        // Re-enable elections on follower1 so it becomes the leader
-
-        ActorRef follower1Shard = IntegrationTestKit.findLocalShard(follower1Node.configDataStore().getActorContext(),
-                ENTITY_OWNERSHIP_SHARD_NAME);
-        follower1Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build())
-                .customRaftPolicyImplementation(null).build(), ActorRef.noSender());
-
-        MemberNode.verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
-            raftState -> assertEquals("Raft state", RaftState.Leader.toString(), raftState.getRaftState()));
+        follower2Node.waitForMemberDown("member-1");
+        follower2Node.waitForMemberDown("member-4");
+        follower4Node.waitForMemberDown("member-1");
+        follower4Node.waitForMemberDown("member-4");
 
         // Verify the prior leader's and follower3 entity owners are re-assigned.
 

©2013 OpenDaylight, A Linux Foundation Collaborative Project. All Rights Reserved.
OpenDaylight is a registered trademark of The OpenDaylight Project, Inc.
Linux Foundation and OpenDaylight are registered trademarks of the Linux Foundation.
Linux is a registered trademark of Linus Torvalds.