Bug 2038: Ensure only one concurrent 3-phase commit in Shard
[controller.git] / opendaylight / md-sal / sal-distributed-datastore / src / main / java / org / opendaylight / controller / cluster / datastore / DistributedDataStore.java
index 202ced9a26512047f57ee8ad2b0ced7e61366771..f6c31aab04c76a376e18dad96dae95aa9cffb637 100644 (file)
@@ -8,13 +8,13 @@
 
 package org.opendaylight.controller.cluster.datastore;
 
-import java.util.concurrent.TimeUnit;
-
 import akka.actor.ActorRef;
 import akka.actor.ActorSystem;
-
+import akka.dispatch.OnComplete;
+import akka.util.Timeout;
+import com.google.common.base.Optional;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
-
 import org.opendaylight.controller.cluster.datastore.identifiers.ShardManagerIdentifier;
 import org.opendaylight.controller.cluster.datastore.messages.RegisterChangeListener;
 import org.opendaylight.controller.cluster.datastore.messages.RegisterChangeListenerReply;
@@ -22,7 +22,6 @@ import org.opendaylight.controller.cluster.datastore.shardstrategy.ShardStrategy
 import org.opendaylight.controller.cluster.datastore.utils.ActorContext;
 import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker;
 import org.opendaylight.controller.md.sal.common.api.data.AsyncDataChangeListener;
-import org.opendaylight.controller.md.sal.dom.store.impl.InMemoryDOMDataStoreConfigProperties;
 import org.opendaylight.controller.sal.core.spi.data.DOMStore;
 import org.opendaylight.controller.sal.core.spi.data.DOMStoreReadTransaction;
 import org.opendaylight.controller.sal.core.spi.data.DOMStoreReadWriteTransaction;
@@ -35,8 +34,7 @@ import org.opendaylight.yangtools.yang.model.api.SchemaContext;
 import org.opendaylight.yangtools.yang.model.api.SchemaContextListener;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-
-import scala.concurrent.duration.Duration;
+import scala.concurrent.Future;
 
 /**
  *
@@ -44,47 +42,38 @@ import scala.concurrent.duration.Duration;
 public class DistributedDataStore implements DOMStore, SchemaContextListener, AutoCloseable {
 
     private static final Logger LOG = LoggerFactory.getLogger(DistributedDataStore.class);
+    public static final int REGISTER_DATA_CHANGE_LISTENER_TIMEOUT_FACTOR = 24; // 24 times the usual operation timeout
 
     private final ActorContext actorContext;
-    private final DatastoreContext datastoreContext;
 
     public DistributedDataStore(ActorSystem actorSystem, String type, ClusterWrapper cluster,
-            Configuration configuration, DistributedDataStoreProperties dataStoreProperties) {
+            Configuration configuration, DatastoreContext datastoreContext) {
         Preconditions.checkNotNull(actorSystem, "actorSystem should not be null");
         Preconditions.checkNotNull(type, "type should not be null");
         Preconditions.checkNotNull(cluster, "cluster should not be null");
         Preconditions.checkNotNull(configuration, "configuration should not be null");
-
+        Preconditions.checkNotNull(datastoreContext, "datastoreContext should not be null");
 
         String shardManagerId = ShardManagerIdentifier.builder().type(type).build().toString();
 
         LOG.info("Creating ShardManager : {}", shardManagerId);
 
-        datastoreContext = new DatastoreContext(InMemoryDOMDataStoreConfigProperties.create(
-                dataStoreProperties.getMaxShardDataChangeExecutorPoolSize(),
-                dataStoreProperties.getMaxShardDataChangeExecutorQueueSize(),
-                dataStoreProperties.getMaxShardDataChangeListenerQueueSize()),
-                Duration.create(dataStoreProperties.getShardTransactionIdleTimeoutInMinutes(),
-                        TimeUnit.MINUTES));
-
-        actorContext
-                = new ActorContext(
-                    actorSystem, actorSystem.actorOf(
-                        ShardManager.props(type, cluster, configuration, datastoreContext).
-                            withMailbox(ActorContext.MAILBOX), shardManagerId ), cluster, configuration);
+        actorContext = new ActorContext(actorSystem, actorSystem.actorOf(
+                ShardManager.props(type, cluster, configuration, datastoreContext)
+                    .withMailbox(ActorContext.MAILBOX), shardManagerId ), cluster, configuration);
+
+        actorContext.setOperationTimeout(datastoreContext.getOperationTimeoutInSeconds());
     }
 
     public DistributedDataStore(ActorContext actorContext) {
         this.actorContext = Preconditions.checkNotNull(actorContext, "actorContext should not be null");
-        this.datastoreContext = new DatastoreContext();
     }
 
-
     @SuppressWarnings("unchecked")
     @Override
     public <L extends AsyncDataChangeListener<YangInstanceIdentifier, NormalizedNode<?, ?>>>
                                               ListenerRegistration<L> registerChangeListener(
-        YangInstanceIdentifier path, L listener,
+        final YangInstanceIdentifier path, L listener,
         AsyncDataBroker.DataChangeScope scope) {
 
         Preconditions.checkNotNull(path, "path should not be null");
@@ -92,27 +81,41 @@ public class DistributedDataStore implements DOMStore, SchemaContextListener, Au
 
         LOG.debug("Registering listener: {} for path: {} scope: {}", listener, path, scope);
 
-        ActorRef dataChangeListenerActor = actorContext.getActorSystem().actorOf(
-            DataChangeListener.props(listener ));
-
         String shardName = ShardStrategyFactory.getStrategy(path).findShard(path);
 
-        Object result = actorContext.executeLocalShardOperation(shardName,
-            new RegisterChangeListener(path, dataChangeListenerActor.path(), scope),
-            ActorContext.ASK_DURATION);
+        Optional<ActorRef> shard = actorContext.findLocalShard(shardName);
 
-        if (result != null) {
-            RegisterChangeListenerReply reply = (RegisterChangeListenerReply) result;
-            return new DataChangeListenerRegistrationProxy(actorContext
-                .actorSelection(reply.getListenerRegistrationPath()), listener,
-                dataChangeListenerActor);
+        //if shard is NOT local
+        if (!shard.isPresent()) {
+            LOG.debug("No local shard for shardName {} was found so returning a noop registration", shardName);
+            return new NoOpDataChangeListenerRegistration(listener);
         }
+        //if shard is local
+        ActorRef dataChangeListenerActor = actorContext.getActorSystem().actorOf(DataChangeListener.props(listener));
+        Future future = actorContext.executeOperationAsync(shard.get(),
+                new RegisterChangeListener(path, dataChangeListenerActor.path(), scope),
+                new Timeout(actorContext.getOperationDuration().$times(REGISTER_DATA_CHANGE_LISTENER_TIMEOUT_FACTOR)));
+
+        final DataChangeListenerRegistrationProxy listenerRegistrationProxy =
+                new DataChangeListenerRegistrationProxy(listener, dataChangeListenerActor);
+
+        future.onComplete(new OnComplete() {
+
+            @Override
+            public void onComplete(Throwable failure, Object result)
+                    throws Throwable {
+                if (failure != null) {
+                    LOG.error("Failed to register listener at path " + path.toString(), failure);
+                    return;
+                }
+                RegisterChangeListenerReply reply = (RegisterChangeListenerReply) result;
+                listenerRegistrationProxy.setListenerRegistrationActor(actorContext
+                        .actorSelection(reply.getListenerRegistrationPath()));
+            }
+        }, actorContext.getActorSystem().dispatcher());
+
+        return listenerRegistrationProxy;
 
-        LOG.debug(
-            "No local shard for shardName {} was found so returning a noop registration",
-            shardName);
-
-        return new NoOpDataChangeListenerRegistration(listener);
     }
 
     @Override
@@ -144,4 +147,9 @@ public class DistributedDataStore implements DOMStore, SchemaContextListener, Au
     public void close() throws Exception {
         actorContext.shutdown();
     }
+
+    @VisibleForTesting
+    ActorContext getActorContext() {
+        return actorContext;
+    }
 }