Fix intermittent failure in testRemoveShardReplica

[controller.git] / opendaylight / md-sal / sal-cluster-admin / src / test / java / org / opendaylight / controller / cluster / datastore / admin / ClusterAdminRpcServiceTest.java

diff --git a/opendaylight/md-sal/sal-cluster-admin/src/test/java/org/opendaylight/controller/cluster/datastore/admin/ClusterAdminRpcServiceTest.java b/opendaylight/md-sal/sal-cluster-admin/src/test/java/org/opendaylight/controller/cluster/datastore/admin/ClusterAdminRpcServiceTest.java
index 8ec173d284bc88eeeab4be3c3d581370016f906c..62c1a2eb722dabf4caf0ec15b7ed2ad6005e5b05 100644 (file)
--- a/opendaylight/md-sal/sal-cluster-admin/src/test/java/org/opendaylight/controller/cluster/datastore/admin/ClusterAdminRpcServiceTest.java
+++ b/opendaylight/md-sal/sal-cluster-admin/src/test/java/org/opendaylight/controller/cluster/datastore/admin/ClusterAdminRpcServiceTest.java
@@ -26,6 +26,7 @@ import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import java.io.File;
 import java.io.FileInputStream;
@@ -56,10 +57,9 @@ import org.opendaylight.controller.cluster.datastore.messages.CreateShard;
 import org.opendaylight.controller.cluster.datastore.messages.DatastoreSnapshot;
 import org.opendaylight.controller.cluster.raft.RaftState;
 import org.opendaylight.controller.cluster.raft.ReplicatedLogImplEntry;
-import org.opendaylight.controller.cluster.raft.ServerConfigurationPayload;
-import org.opendaylight.controller.cluster.raft.ServerConfigurationPayload.ServerInfo;
+import org.opendaylight.controller.cluster.raft.persisted.ServerConfigurationPayload;
+import org.opendaylight.controller.cluster.raft.persisted.ServerInfo;
 import org.opendaylight.controller.cluster.raft.base.messages.UpdateElectionTerm;
-import org.opendaylight.controller.cluster.raft.client.messages.OnDemandRaftState;
 import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal;
 import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore;
 import org.opendaylight.controller.md.cluster.datastore.model.CarsModel;
@@ -102,9 +102,10 @@ public class ClusterAdminRpcServiceTest {
 
     @After
     public void tearDown() {
-        for(MemberNode m: memberNodes) {
+        for (MemberNode m : Lists.reverse(memberNodes)) {
             m.cleanup();
         }
+        memberNodes.clear();
     }
 
     @Test
@@ -195,12 +196,9 @@ public class ClusterAdminRpcServiceTest {
         // Verify all data has been replicated. We expect 3 log entries and thus last applied index of 2 -
         // 2 ServerConfigurationPayload entries and the transaction payload entry.
 
-        RaftStateVerifier verifier = new RaftStateVerifier() {
-            @Override
-            public void verify(OnDemandRaftState raftState) {
-                assertEquals("Commit index", 2, raftState.getCommitIndex());
-                assertEquals("Last applied index", 2, raftState.getLastApplied());
-            }
+        RaftStateVerifier verifier = raftState -> {
+            assertEquals("Commit index", 2, raftState.getCommitIndex());
+            assertEquals("Last applied index", 2, raftState.getLastApplied());
         };
 
         verifyRaftState(leaderNode1.configDataStore(), "cars", verifier);
@@ -315,7 +313,7 @@ public class ClusterAdminRpcServiceTest {
 
     @Test
     public void testRemoveShardReplica() throws Exception {
-        String name = "testRemoveShardReplicaLocal";
+        String name = "testRemoveShardReplica";
         String moduleShardsConfig = "module-shards-member1-and-2-and-3.conf";
         MemberNode leaderNode1 = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name ).
                 moduleShardsConfig(moduleShardsConfig).
@@ -329,6 +327,7 @@ public class ClusterAdminRpcServiceTest {
                 moduleShardsConfig(moduleShardsConfig).build();
 
         leaderNode1.configDataStore().waitTillReady();
+        replicaNode3.configDataStore().waitTillReady();
         verifyRaftPeersPresent(leaderNode1.configDataStore(), "cars", "member-2", "member-3");
         verifyRaftPeersPresent(replicaNode2.configDataStore(), "cars", "member-1", "member-3");
         verifyRaftPeersPresent(replicaNode3.configDataStore(), "cars", "member-1", "member-2");
@@ -355,6 +354,7 @@ public class ClusterAdminRpcServiceTest {
         replicaNode2 = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name).
                 moduleShardsConfig(moduleShardsConfig).build();
 
+        replicaNode2.configDataStore().waitTillReady();
         verifyRaftPeersPresent(replicaNode2.configDataStore(), "cars", "member-1");
 
         // Invoke RPC service on member-1 to remove member-2
@@ -392,7 +392,7 @@ public class ClusterAdminRpcServiceTest {
         verifyRaftPeersPresent(replicaNode3.configDataStore(), "cars", "member-1", "member-2");
 
         replicaNode2.waitForMembersUp("member-1", "member-3");
-        replicaNode2.waitForMembersUp("member-1", "member-2");
+        replicaNode3.waitForMembersUp("member-1", "member-2");
 
         // Invoke RPC service on leader member-1 to remove it's local shard
 
@@ -404,13 +404,8 @@ public class ClusterAdminRpcServiceTest {
                         get(10, TimeUnit.SECONDS);
         verifySuccessfulRpcResult(rpcResult);
 
-        verifyRaftState(replicaNode2.configDataStore(), "cars", new RaftStateVerifier() {
-            @Override
-            public void verify(OnDemandRaftState raftState) {
-                assertThat("Leader Id", raftState.getLeader(), anyOf(containsString("member-2"),
-                        containsString("member-3")));
-            }
-        });
+        verifyRaftState(replicaNode2.configDataStore(), "cars", raftState ->
+                assertThat("Leader Id", raftState.getLeader(), anyOf(containsString("member-2"), containsString("member-3"))));
 
         verifyRaftPeersPresent(replicaNode2.configDataStore(), "cars", "member-3");
         verifyRaftPeersPresent(replicaNode3.configDataStore(), "cars", "member-2");
@@ -541,6 +536,7 @@ public class ClusterAdminRpcServiceTest {
                 moduleShardsConfig(moduleShardsConfig).build();
 
         leaderNode1.configDataStore().waitTillReady();
+        replicaNode3.configDataStore().waitTillReady();
         verifyRaftPeersPresent(leaderNode1.configDataStore(), "cars", "member-2", "member-3");
         verifyRaftPeersPresent(replicaNode2.configDataStore(), "cars", "member-1", "member-3");
         verifyRaftPeersPresent(replicaNode3.configDataStore(), "cars", "member-1", "member-2");
@@ -607,6 +603,8 @@ public class ClusterAdminRpcServiceTest {
 
         leaderNode1.configDataStore().waitTillReady();
         leaderNode1.operDataStore().waitTillReady();
+        replicaNode3.configDataStore().waitTillReady();
+        replicaNode3.operDataStore().waitTillReady();
         verifyRaftPeersPresent(leaderNode1.configDataStore(), "cars", "member-2", "member-3");
         verifyRaftPeersPresent(replicaNode2.configDataStore(), "cars", "member-1", "member-3");
         verifyRaftPeersPresent(replicaNode3.configDataStore(), "cars", "member-1", "member-2");
@@ -659,6 +657,8 @@ public class ClusterAdminRpcServiceTest {
 
         leaderNode1.configDataStore().waitTillReady();
         leaderNode1.operDataStore().waitTillReady();
+        replicaNode3.configDataStore().waitTillReady();
+        replicaNode3.operDataStore().waitTillReady();
         verifyVotingStates(leaderNode1.configDataStore(), "cars", new SimpleEntry<>("member-1", true),
                 new SimpleEntry<>("member-2", true), new SimpleEntry<>("member-3", false));