Fix intermittent ShardTest failures

Some tests fail intermittenly due to modifying Shard state directly
instead of thru messages.

Change-Id: I704d6d23c1b2a47e78b3d8823a3136e921e9113b
Signed-off-by: Tom Pantelis <tpanteli@brocade.com>

diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/AbstractShardTest.java b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/AbstractShardTest.java
index 17dec1911eb3a531ca69800cc924107c35518cac..3b6625f15a1f9db9d2211c871ed69874ba52c72b 100644 (file)
--- a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/AbstractShardTest.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/AbstractShardTest.java
@@ -22,7 +22,9 @@ import akka.actor.PoisonPill;
 import akka.actor.Props;
 import akka.dispatch.Dispatchers;
 import akka.japi.Creator;
+import akka.pattern.Patterns;
 import akka.testkit.TestActorRef;
+import akka.util.Timeout;
 import com.google.common.base.Function;
 import com.google.common.base.Optional;
 import com.google.common.util.concurrent.Futures;
@@ -33,6 +35,7 @@ import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.junit.After;
 import org.junit.Assert;
@@ -69,6 +72,9 @@ import org.opendaylight.yangtools.yang.data.api.schema.tree.TreeType;
 import org.opendaylight.yangtools.yang.data.impl.schema.ImmutableNodes;
 import org.opendaylight.yangtools.yang.data.impl.schema.tree.InMemoryDataTreeFactory;
 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
+import scala.concurrent.Await;
+import scala.concurrent.Future;
+import scala.concurrent.duration.Duration;
 
 /**
  * Abstract base for shard unit tests.
@@ -341,7 +347,13 @@ public abstract class AbstractShardTest extends AbstractActorTest{
 
     public static void writeToStore(final TestActorRef<Shard> shard, final YangInstanceIdentifier id,
             final NormalizedNode<?,?> node) throws InterruptedException, ExecutionException {
-        writeToStore(shard.underlyingActor().getDataStore(), id, node);
+        Future<Object> future = Patterns.ask(shard, newBatchedModifications("tx", id, node, true, true, 1),
+                new Timeout(5, TimeUnit.SECONDS));
+        try {
+            Await.ready(future, Duration.create(5, TimeUnit.SECONDS));
+        } catch(TimeoutException e) {
+            throw new ExecutionException(e);
+        }
     }
 
     public static void writeToStore(final ShardDataTree store, final YangInstanceIdentifier id,

diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/ShardTest.java b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/ShardTest.java
index 6ea6dae2231451fc7d549c4d9880aa8d4ef85bdc..51063c8d2cc5fe6b2f9e2eb8b4a16d7ed62e30b3 100644 (file)
--- a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/ShardTest.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/ShardTest.java
@@ -204,6 +204,8 @@ public class ShardTest extends AbstractShardTest {
                 }
             };
 
+            setupInMemorySnapshotStore();
+
             final MockDataChangeListener listener = new MockDataChangeListener(1);
             final ActorRef dclActor = actorFactory.createActor(DataChangeListener.props(listener),
                     "testRegisterChangeListenerWhenNotLeaderInitially-DataChangeListener");
@@ -212,9 +214,7 @@ public class ShardTest extends AbstractShardTest {
                     Props.create(new DelegatingShardCreator(creator)).withDispatcher(Dispatchers.DefaultDispatcherId()),
                     "testRegisterChangeListenerWhenNotLeaderInitially");
 
-            // Write initial data into the in-memory store.
             final YangInstanceIdentifier path = TestModel.TEST_PATH;
-            writeToStore(shard, path, ImmutableNodes.containerNode(TestModel.TEST_QNAME));
 
             // Wait until the shard receives the first ElectionTimeout message.
             assertEquals("Got first ElectionTimeout", true,
@@ -284,8 +284,7 @@ public class ShardTest extends AbstractShardTest {
 
                 @Override
                 public Shard create() throws Exception {
-                    return new Shard(Shard.builder().id(shardID).datastoreContext(
-                            dataStoreContextBuilder.persistent(false).build()).schemaContext(SCHEMA_CONTEXT)) {
+                    return new Shard(newShardBuilder()) {
                         @Override
                         public void onReceiveCommand(final Object message) throws Exception {
                             if(message instanceof ElectionTimeout && firstElectionTimeout) {
@@ -309,6 +308,8 @@ public class ShardTest extends AbstractShardTest {
                 }
             };
 
+            setupInMemorySnapshotStore();
+
             final MockDataTreeChangeListener listener = new MockDataTreeChangeListener(1);
             final ActorRef dclActor = actorFactory.createActor(DataTreeChangeListenerActor.props(listener),
                     "testDataTreeChangeListenerNotifiedWhenNotTheLeaderOnRegistration-DataChangeListener");
@@ -318,7 +319,6 @@ public class ShardTest extends AbstractShardTest {
                     "testDataTreeChangeListenerNotifiedWhenNotTheLeaderOnRegistration");
 
             final YangInstanceIdentifier path = TestModel.TEST_PATH;
-            writeToStore(shard, path, ImmutableNodes.containerNode(TestModel.TEST_QNAME));
 
             assertEquals("Got first ElectionTimeout", true,
                 onFirstElectionTimeout.await(5, TimeUnit.SECONDS));
@@ -333,7 +333,6 @@ public class ShardTest extends AbstractShardTest {
                     expectMsgClass(duration("5 seconds"), FindLeaderReply.class);
             assertFalse("Expected the shard not to be the leader", findLeadeReply.getLeaderActor().isPresent());
 
-            writeToStore(shard, path, ImmutableNodes.containerNode(TestModel.TEST_QNAME));
 
             onChangeListenerRegistered.countDown();
 
@@ -2287,12 +2286,15 @@ public class ShardTest extends AbstractShardTest {
     public void testClusteredDataChangeListenerDelayedRegistration() throws Exception {
         new ShardTestKit(getSystem()) {{
             String testName = "testClusteredDataChangeListenerDelayedRegistration";
-            dataStoreContextBuilder.shardElectionTimeoutFactor(1000);
+            dataStoreContextBuilder.shardElectionTimeoutFactor(1000).
+                    customRaftPolicyImplementation(DisableElectionsRaftPolicy.class.getName());
 
             final MockDataChangeListener listener = new MockDataChangeListener(1);
             final ActorRef dclActor = actorFactory.createActor(DataChangeListener.props(listener),
                     actorFactory.generateActorId(testName + "-DataChangeListener"));
 
+            setupInMemorySnapshotStore();
+
             final TestActorRef<Shard> shard = actorFactory.createTestActor(
                     newShardBuilder().props().withDispatcher(Dispatchers.DefaultDispatcherId()),
                     actorFactory.generateActorId(testName + "-shard"));
@@ -2306,9 +2308,8 @@ public class ShardTest extends AbstractShardTest {
                 RegisterChangeListenerReply.class);
             assertNotNull("getListenerRegistrationPath", reply.getListenerRegistrationPath());
 
-            writeToStore(shard, path, ImmutableNodes.containerNode(TestModel.TEST_QNAME));
-
-            shard.tell(new ElectionTimeout(), ActorRef.noSender());
+            shard.tell(DatastoreContext.newBuilderFrom(dataStoreContextBuilder.build()).
+                    customRaftPolicyImplementation(null).build(), ActorRef.noSender());
 
             listener.waitForChangeEvents();
         }};
@@ -2361,12 +2362,15 @@ public class ShardTest extends AbstractShardTest {
     public void testClusteredDataTreeChangeListenerDelayedRegistration() throws Exception {
         new ShardTestKit(getSystem()) {{
             String testName = "testClusteredDataTreeChangeListenerDelayedRegistration";
-            dataStoreContextBuilder.shardElectionTimeoutFactor(1000);
+            dataStoreContextBuilder.shardElectionTimeoutFactor(1000).
+                    customRaftPolicyImplementation(DisableElectionsRaftPolicy.class.getName());
 
             final MockDataTreeChangeListener listener = new MockDataTreeChangeListener(1);
             final ActorRef dclActor = actorFactory.createActor(DataTreeChangeListenerActor.props(listener),
                     actorFactory.generateActorId(testName + "-DataTreeChangeListener"));
 
+            setupInMemorySnapshotStore();
+
             final TestActorRef<Shard> shard = actorFactory.createTestActor(
                     newShardBuilder().props().withDispatcher(Dispatchers.DefaultDispatcherId()),
                     actorFactory.generateActorId(testName + "-shard"));
@@ -2380,9 +2384,8 @@ public class ShardTest extends AbstractShardTest {
                     RegisterDataTreeChangeListenerReply.class);
             assertNotNull("getListenerRegistrationPath", reply.getListenerRegistrationPath());
 
-            writeToStore(shard, path, ImmutableNodes.containerNode(TestModel.TEST_QNAME));
-
-            shard.tell(new ElectionTimeout(), ActorRef.noSender());
+            shard.tell(DatastoreContext.newBuilderFrom(dataStoreContextBuilder.build()).
+                    customRaftPolicyImplementation(null).build(), ActorRef.noSender());
 
             listener.waitForChangeEvents();
         }};

diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/utils/MockDataTreeChangeListener.java b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/utils/MockDataTreeChangeListener.java
index cf188b339e2d5b3c420634919b6e46b8b2ba2d05..0696fcf832c9a3ff81fe7027cfb46b0618df2feb 100644 (file)
--- a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/utils/MockDataTreeChangeListener.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/utils/MockDataTreeChangeListener.java
@@ -14,7 +14,6 @@ import com.google.common.collect.Lists;
 import com.google.common.util.concurrent.Uninterruptibles;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -28,7 +27,7 @@ import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidate;
 public class MockDataTreeChangeListener implements DOMDataTreeChangeListener {
 
     private final List<Collection<DataTreeCandidate>> changeList =
-            Collections.synchronizedList(Lists.<Collection<DataTreeCandidate>>newArrayList());
+            Lists.<Collection<DataTreeCandidate>>newArrayList();
 
     private volatile CountDownLatch changeLatch;
     private int expChangeEventCount;
@@ -40,12 +39,16 @@ public class MockDataTreeChangeListener implements DOMDataTreeChangeListener {
     public void reset(int expChangeEventCount) {
         changeLatch = new CountDownLatch(expChangeEventCount);
         this.expChangeEventCount = expChangeEventCount;
-        changeList.clear();
+        synchronized(changeList) {
+            changeList.clear();
+        }
     }
 
     @Override
     public void onDataTreeChanged(@Nonnull final Collection<DataTreeCandidate> changes) {
-        changeList.add(changes);
+        synchronized(changeList) {
+            changeList.add(changes);
+        }
         changeLatch.countDown();
     }
 
@@ -59,9 +62,11 @@ public class MockDataTreeChangeListener implements DOMDataTreeChangeListener {
 
     public void verifyNotifiedData(YangInstanceIdentifier... paths) {
         Set<YangInstanceIdentifier> pathSet = new HashSet<>(Arrays.asList(paths));
-        for(Collection<DataTreeCandidate> list: changeList) {
-            for(DataTreeCandidate c: list) {
-                pathSet.remove(c.getRootPath());
+        synchronized(changeList) {
+            for(Collection<DataTreeCandidate> list: changeList) {
+                for(DataTreeCandidate c: list) {
+                    pathSet.remove(c.getRootPath());
+                }
             }
         }
 
@@ -72,15 +77,19 @@ public class MockDataTreeChangeListener implements DOMDataTreeChangeListener {
 
     public void expectNoMoreChanges(String assertMsg) {
         Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
-        assertEquals(assertMsg, expChangeEventCount, changeList.size());
+        synchronized(changeList) {
+            assertEquals(assertMsg, expChangeEventCount, changeList.size());
+        }
     }
 
     public void verifyNoNotifiedData(YangInstanceIdentifier... paths) {
         Set<YangInstanceIdentifier> pathSet = new HashSet<>(Arrays.asList(paths));
-        for(Collection<DataTreeCandidate> list: changeList) {
-            for(DataTreeCandidate c: list) {
-                assertFalse("Unexpected " + c.getRootPath() + " present in DataTreeCandidate",
-                        pathSet.contains(c.getRootPath()));
+        synchronized(changeList) {
+            for(Collection<DataTreeCandidate> list: changeList) {
+                for(DataTreeCandidate c: list) {
+                    assertFalse("Unexpected " + c.getRootPath() + " present in DataTreeCandidate",
+                            pathSet.contains(c.getRootPath()));
+                }
             }
         }
     }