From: Tom Pantelis <tompantelis@gmail.com>
Date: Thu, 27 Apr 2017 11:23:34 +0000 (-0400)
Subject: Add more debug logging for DTCL registration/notification code paths
X-Git-Tag: release/nitrogen~305
X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?p=controller.git;a=commitdiff_plain;h=4d15b2022b97238fa9da4df4239f88c619b2e264

Add more debug logging for DTCL registration/notification code paths

Added logging so the listener instance and actor can be traced
end-to-end from FE registration to the BE publisher actor.

Also added log context to some classes to identify which shard it
belongs to.

Change-Id: I3e6dd92e7632139372407abf94a160096aa7750e
Signed-off-by: Tom Pantelis <tompantelis@gmail.com>
---

diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/AbstractDataListenerSupport.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/AbstractDataListenerSupport.java
index f9995a28b1..15de81e290 100644
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/AbstractDataListenerSupport.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/AbstractDataListenerSupport.java
@@ -42,7 +42,7 @@ abstract class AbstractDataListenerSupport<L extends EventListener, M extends Li
     void onLeadershipChange(boolean isLeader, boolean hasLeader) {
         log.debug("{}: onLeadershipChange, isLeader: {}, hasLeader : {}", persistenceId(), isLeader, hasLeader);
 
-        final EnableNotification msg = new EnableNotification(isLeader);
+        final EnableNotification msg = new EnableNotification(isLeader, persistenceId());
         for (ActorSelection dataChangeListener : leaderOnlyListenerActors) {
             dataChangeListener.tell(msg, getSelf());
         }
@@ -66,7 +66,8 @@ abstract class AbstractDataListenerSupport<L extends EventListener, M extends Li
 
     @Override
     void onMessage(M message, boolean isLeader, boolean hasLeader) {
-        log.debug("{}: {} for {}, leader: {}", persistenceId(), logName(), message.getPath(), isLeader);
+        log.debug("{}: {} for {}, isLeader: {}, hasLeader: {}", persistenceId(), logName(), message,
+                isLeader, hasLeader);
 
         ActorRef registrationActor = createActor(DataTreeNotificationListenerRegistrationActor.props());
 
@@ -99,7 +100,7 @@ abstract class AbstractDataListenerSupport<L extends EventListener, M extends Li
         final ActorSelection listenerActor = selectActor(message.getListenerActorPath());
 
         // We have a leader so enable the listener.
-        listenerActor.tell(new EnableNotification(true), getSelf());
+        listenerActor.tell(new EnableNotification(true, persistenceId()), getSelf());
 
         if (!message.isRegisterOnAllInstances()) {
             // This is a leader-only registration so store a reference to the listener actor so it can be notified
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/AbstractShardDataTreeNotificationPublisherActorProxy.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/AbstractShardDataTreeNotificationPublisherActorProxy.java
index 6b348fb76f..1780bb78b3 100644
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/AbstractShardDataTreeNotificationPublisherActorProxy.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/AbstractShardDataTreeNotificationPublisherActorProxy.java
@@ -24,13 +24,12 @@ import org.slf4j.LoggerFactory;
  */
 @NotThreadSafe
 abstract class AbstractShardDataTreeNotificationPublisherActorProxy implements ShardDataTreeNotificationPublisher {
-    private static final Logger LOG = LoggerFactory.getLogger(
-            AbstractShardDataTreeNotificationPublisherActorProxy.class);
+    protected final Logger log = LoggerFactory.getLogger(getClass());
 
     private final ActorContext actorContext;
     private final String actorName;
     private final String logContext;
-    private ActorRef notifierActor;
+    private ActorRef publisherActor;
 
     protected AbstractShardDataTreeNotificationPublisherActorProxy(ActorContext actorContext, String actorName,
             String logContext) {
@@ -50,21 +49,21 @@ abstract class AbstractShardDataTreeNotificationPublisherActorProxy implements S
     }
 
     @Override
-    public void publishChanges(DataTreeCandidate candidate, String logContext) {
-        notifierActor().tell(new ShardDataTreeNotificationPublisherActor.PublishNotifications(candidate),
+    public void publishChanges(DataTreeCandidate candidate) {
+        publisherActor().tell(new ShardDataTreeNotificationPublisherActor.PublishNotifications(candidate),
                 ActorRef.noSender());
     }
 
-    protected final ActorRef notifierActor() {
-        if (notifierActor == null) {
-            LOG.debug("Creating actor {}", actorName);
-
+    protected final ActorRef publisherActor() {
+        if (publisherActor == null) {
             String dispatcher = new Dispatchers(actorContext.system().dispatchers()).getDispatcherPath(
                     Dispatchers.DispatcherType.Notification);
-            notifierActor = actorContext.actorOf(props().withDispatcher(dispatcher).withMailbox(
+            publisherActor = actorContext.actorOf(props().withDispatcher(dispatcher).withMailbox(
                     org.opendaylight.controller.cluster.datastore.utils.ActorContext.BOUNDED_MAILBOX), actorName);
+
+            log.debug("{}: Created publisher actor {} with name {}", logContext, publisherActor, actorName);
         }
 
-        return notifierActor;
+        return publisherActor;
     }
 }
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DataTreeChangeListenerActor.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DataTreeChangeListenerActor.java
index 2936a28b90..902156f346 100644
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DataTreeChangeListenerActor.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DataTreeChangeListenerActor.java
@@ -24,6 +24,7 @@ final class DataTreeChangeListenerActor extends AbstractUntypedActor {
     private final DOMDataTreeChangeListener listener;
     private final YangInstanceIdentifier registeredPath;
     private boolean notificationsEnabled = false;
+    private String logContext = "";
 
     private DataTreeChangeListenerActor(final DOMDataTreeChangeListener listener,
             final YangInstanceIdentifier registeredPath) {
@@ -46,16 +47,18 @@ final class DataTreeChangeListenerActor extends AbstractUntypedActor {
     private void dataChanged(final DataTreeChanged message) {
         // Do nothing if notifications are not enabled
         if (!notificationsEnabled) {
-            LOG.debug("Notifications not enabled for listener {} - dropping change notification", listener);
+            LOG.debug("{}: Notifications not enabled for listener {} - dropping change notification",
+                    logContext, listener);
             return;
         }
 
-        LOG.debug("Sending change notification {} to listener {}", message.getChanges(), listener);
+        LOG.debug("{}: Sending {} change notification(s) {} to listener {}", logContext, message.getChanges().size(),
+                message.getChanges(), listener);
 
         try {
             this.listener.onDataTreeChanged(message.getChanges());
         } catch (Exception e) {
-            LOG.error("Error notifying listener {}", this.listener, e);
+            LOG.error("{}: Error notifying listener {}", logContext, this.listener, e);
         }
 
         // TODO: do we really need this?
@@ -67,8 +70,9 @@ final class DataTreeChangeListenerActor extends AbstractUntypedActor {
     }
 
     private void enableNotification(final EnableNotification message) {
+        logContext = message.getLogContext();
         notificationsEnabled = message.isEnabled();
-        LOG.debug("{} notifications for listener {}", notificationsEnabled ? "Enabled" : "Disabled",
+        LOG.debug("{}: {} notifications for listener {}", logContext, notificationsEnabled ? "Enabled" : "Disabled",
                 listener);
     }
 
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DataTreeChangeListenerProxy.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DataTreeChangeListenerProxy.java
index dd280a6c08..5f5114d399 100644
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DataTreeChangeListenerProxy.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DataTreeChangeListenerProxy.java
@@ -50,6 +50,9 @@ final class DataTreeChangeListenerProxy<T extends DOMDataTreeChangeListener> ext
         this.dataChangeListenerActor = actorContext.getActorSystem().actorOf(
                 DataTreeChangeListenerActor.props(getInstance(), registeredPath)
                     .withDispatcher(actorContext.getNotificationDispatcherPath()));
+
+        LOG.debug("{}: Created actor {} for DTCL {}", actorContext.getDatastoreContext().getLogicalStoreType(),
+                dataChangeListenerActor, listener);
     }
 
     @Override
@@ -69,11 +72,12 @@ final class DataTreeChangeListenerProxy<T extends DOMDataTreeChangeListener> ext
             @Override
             public void onComplete(final Throwable failure, final ActorRef shard) {
                 if (failure instanceof LocalShardNotFoundException) {
-                    LOG.debug("No local shard found for {} - DataTreeChangeListener {} at path {} "
-                            + "cannot be registered", shardName, getInstance(), registeredPath);
+                    LOG.debug("{}: No local shard found for {} - DataTreeChangeListener {} at path {} "
+                            + "cannot be registered", logContext(), shardName, getInstance(), registeredPath);
                 } else if (failure != null) {
-                    LOG.error("Failed to find local shard {} - DataTreeChangeListener {} at path {} "
-                            + "cannot be registered: {}", shardName, getInstance(), registeredPath, failure);
+                    LOG.error("{}: Failed to find local shard {} - DataTreeChangeListener {} at path {} "
+                            + "cannot be registered: {}", logContext(), shardName, getInstance(), registeredPath,
+                            failure);
                 } else {
                     doRegistration(shard);
                 }
@@ -83,7 +87,7 @@ final class DataTreeChangeListenerProxy<T extends DOMDataTreeChangeListener> ext
 
     private void setListenerRegistrationActor(final ActorSelection actor) {
         if (actor == null) {
-            LOG.debug("Ignoring null actor on {}", this);
+            LOG.debug("{}: Ignoring null actor on {}", logContext(), this);
             return;
         }
 
@@ -109,7 +113,7 @@ final class DataTreeChangeListenerProxy<T extends DOMDataTreeChangeListener> ext
             @Override
             public void onComplete(final Throwable failure, final Object result) {
                 if (failure != null) {
-                    LOG.error("Failed to register DataTreeChangeListener {} at path {}",
+                    LOG.error("{}: Failed to register DataTreeChangeListener {} at path {}", logContext(),
                             getInstance(), registeredPath, failure);
                 } else {
                     RegisterDataTreeChangeListenerReply reply = (RegisterDataTreeChangeListenerReply) result;
@@ -129,4 +133,8 @@ final class DataTreeChangeListenerProxy<T extends DOMDataTreeChangeListener> ext
     ActorRef getDataChangeListenerActor() {
         return dataChangeListenerActor;
     }
+
+    private String logContext() {
+        return actorContext.getDatastoreContext().getLogicalStoreType().toString();
+    }
 }
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DataTreeChangeListenerSupport.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DataTreeChangeListenerSupport.java
index f250fb2904..a3ed4a19c9 100644
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DataTreeChangeListenerSupport.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DataTreeChangeListenerSupport.java
@@ -27,7 +27,7 @@ final class DataTreeChangeListenerSupport extends AbstractDataListenerSupport<DO
 
         DOMDataTreeChangeListener listener = new ForwardingDataTreeChangeListener(listenerActor);
 
-        log().debug("{}: Registering for path {}", persistenceId(), message.getPath());
+        log().debug("{}: Registering listenerActor {} for path {}", persistenceId(), listenerActor, message.getPath());
 
         final ShardDataTree shardDataTree = getShard().getDataStore();
         shardDataTree.registerTreeChangeListener(message.getPath(),
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DefaultShardDataChangeListenerPublisher.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DefaultShardDataChangeListenerPublisher.java
index 98083a0b1c..89da5b2621 100644
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DefaultShardDataChangeListenerPublisher.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DefaultShardDataChangeListenerPublisher.java
@@ -36,11 +36,16 @@ final class DefaultShardDataChangeListenerPublisher implements ShardDataChangeLi
     private static final Logger LOG = LoggerFactory.getLogger(DefaultShardDataChangeListenerPublisher.class);
 
     private final ListenerTree dataChangeListenerTree = ListenerTree.create();
+    private final String logContext;
+
+    DefaultShardDataChangeListenerPublisher(String logContext) {
+        this.logContext = logContext;
+    }
 
     @Override
     public void submitNotification(final DataChangeListenerRegistration<?> listener,
             final DOMImmutableDataChangeEvent notification) {
-        LOG.debug("Notifying listener {} about {}", listener.getInstance(), notification);
+        LOG.debug("{}: Notifying listener {} about {}", logContext, listener.getInstance(), notification);
 
         listener.getInstance().onDataChanged(notification);
     }
@@ -49,7 +54,7 @@ final class DefaultShardDataChangeListenerPublisher implements ShardDataChangeLi
     public void submitNotifications(final DataChangeListenerRegistration<?> listener,
             final Iterable<DOMImmutableDataChangeEvent> notifications) {
         final AsyncDataChangeListener<YangInstanceIdentifier, NormalizedNode<?, ?>> instance = listener.getInstance();
-        LOG.debug("Notifying listener {} about {}", instance, notifications);
+        LOG.debug("{}: Notifying listener {} about {}", logContext, instance, notifications);
 
         for (DOMImmutableDataChangeEvent n : notifications) {
             instance.onDataChanged(n);
@@ -57,7 +62,7 @@ final class DefaultShardDataChangeListenerPublisher implements ShardDataChangeLi
     }
 
     @Override
-    public void publishChanges(DataTreeCandidate candidate, String logContext) {
+    public void publishChanges(DataTreeCandidate candidate) {
         ResolveDataChangeEventsTask.create(candidate, dataChangeListenerTree).resolve(this);
     }
 
@@ -73,15 +78,15 @@ final class DefaultShardDataChangeListenerPublisher implements ShardDataChangeLi
         onRegistration.accept(registration);
 
         if (initialState.isPresent()) {
-            notifySingleListener(path, listener, scope, initialState.get());
+            notifySingleListener(path, listener, scope, initialState.get(), logContext);
         }
     }
 
     static void notifySingleListener(final YangInstanceIdentifier path,
             final AsyncDataChangeListener<YangInstanceIdentifier,NormalizedNode<?, ?>> listener,
-            final DataChangeScope scope, final DataTreeCandidate initialState) {
-        DefaultShardDataChangeListenerPublisher publisher = new DefaultShardDataChangeListenerPublisher();
+            final DataChangeScope scope, final DataTreeCandidate initialState, String logContext) {
+        DefaultShardDataChangeListenerPublisher publisher = new DefaultShardDataChangeListenerPublisher(logContext);
         publisher.registerDataChangeListener(path, listener, scope, Optional.absent(), noop -> { });
-        publisher.publishChanges(initialState, "");
+        publisher.publishChanges(initialState);
     }
 }
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DefaultShardDataTreeChangeListenerPublisher.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DefaultShardDataTreeChangeListenerPublisher.java
index 02326e0f84..895ce30e8b 100644
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DefaultShardDataTreeChangeListenerPublisher.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/DefaultShardDataTreeChangeListenerPublisher.java
@@ -30,15 +30,22 @@ import org.slf4j.LoggerFactory;
 final class DefaultShardDataTreeChangeListenerPublisher extends AbstractDOMStoreTreeChangePublisher
         implements ShardDataTreeChangeListenerPublisher {
     private static final Logger LOG = LoggerFactory.getLogger(DefaultShardDataTreeChangeListenerPublisher.class);
+    private String logContext;
+
+    DefaultShardDataTreeChangeListenerPublisher(String logContext) {
+        this.logContext = logContext;
+    }
 
     @Override
-    public void publishChanges(final DataTreeCandidate candidate, String logContext) {
+    public void publishChanges(final DataTreeCandidate candidate) {
+        LOG.debug("{}: publishChanges: {}", logContext, candidate);
         processCandidateTree(candidate);
     }
 
     @Override
     protected void notifyListener(AbstractDOMDataTreeChangeListenerRegistration<?> registration,
             Collection<DataTreeCandidate> changes) {
+        LOG.debug("{}: notifyListener: listener: {}", logContext, registration.getInstance());
         registration.getInstance().onDataTreeChanged(changes);
     }
 
@@ -51,10 +58,10 @@ final class DefaultShardDataTreeChangeListenerPublisher extends AbstractDOMStore
     public void registerTreeChangeListener(YangInstanceIdentifier treeId, DOMDataTreeChangeListener listener,
             Optional<DataTreeCandidate> initialState,
             Consumer<ListenerRegistration<DOMDataTreeChangeListener>> onRegistration) {
+        LOG.debug("{}: registerTreeChangeListener: path: {}, listener: {}", logContext, treeId, listener);
+
         AbstractDOMDataTreeChangeListenerRegistration<org.opendaylight.mdsal.dom.api.DOMDataTreeChangeListener>
-            registration = super.registerTreeChangeListener(treeId,
-                (org.opendaylight.mdsal.dom.api.DOMDataTreeChangeListener)changes ->
-                    listener.onDataTreeChanged(changes));
+            registration = super.registerTreeChangeListener(treeId, new ForwardingDOMDataTreeChangeListener(listener));
 
         onRegistration.accept(
             new org.opendaylight.controller.md.sal.dom.spi.AbstractDOMDataTreeChangeListenerRegistration<
@@ -66,14 +73,36 @@ final class DefaultShardDataTreeChangeListenerPublisher extends AbstractDOMStore
             });
 
         if (initialState.isPresent()) {
-            notifySingleListener(treeId, listener, initialState.get());
+            notifySingleListener(treeId, listener, initialState.get(), logContext);
         }
     }
 
     static void notifySingleListener(YangInstanceIdentifier treeId, DOMDataTreeChangeListener listener,
-            DataTreeCandidate state) {
-        DefaultShardDataTreeChangeListenerPublisher publisher = new DefaultShardDataTreeChangeListenerPublisher();
+            DataTreeCandidate state, String logContext) {
+        LOG.debug("{}: notifySingleListener: path: {}, listener: {}", logContext, treeId, listener);
+        DefaultShardDataTreeChangeListenerPublisher publisher =
+                new DefaultShardDataTreeChangeListenerPublisher(logContext);
+        publisher.logContext = logContext;
         publisher.registerTreeChangeListener(treeId, listener, Optional.absent(), noop -> { });
-        publisher.publishChanges(state, "");
+        publisher.publishChanges(state);
+    }
+
+    private static class ForwardingDOMDataTreeChangeListener
+            implements org.opendaylight.mdsal.dom.api.DOMDataTreeChangeListener {
+        final DOMDataTreeChangeListener delegate;
+
+        ForwardingDOMDataTreeChangeListener(DOMDataTreeChangeListener delegate) {
+            this.delegate = delegate;
+        }
+
+        @Override
+        public void onDataTreeChanged(Collection<DataTreeCandidate> changes) {
+            delegate.onDataTreeChanged(changes);
+        }
+
+        @Override
+        public String toString() {
+            return delegate.toString();
+        }
     }
 }
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ForwardingDataTreeChangeListener.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ForwardingDataTreeChangeListener.java
index 12e4294916..6cda0de152 100644
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ForwardingDataTreeChangeListener.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ForwardingDataTreeChangeListener.java
@@ -14,6 +14,8 @@ import java.util.Collection;
 import org.opendaylight.controller.cluster.datastore.messages.DataTreeChanged;
 import org.opendaylight.controller.md.sal.dom.api.DOMDataTreeChangeListener;
 import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidate;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Internal implementation of a {@link DOMDataTreeChangeListener} which
@@ -21,6 +23,8 @@ import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidate;
  * message and forwards them towards the client's {@link DataTreeChangeListenerActor}.
  */
 final class ForwardingDataTreeChangeListener implements DOMDataTreeChangeListener {
+    private static final Logger LOG = LoggerFactory.getLogger(ForwardingDataTreeChangeListener.class);
+
     private final ActorSelection actor;
 
     ForwardingDataTreeChangeListener(final ActorSelection actor) {
@@ -29,6 +33,12 @@ final class ForwardingDataTreeChangeListener implements DOMDataTreeChangeListene
 
     @Override
     public void onDataTreeChanged(Collection<DataTreeCandidate> changes) {
+        LOG.debug("Sending DataTreeChanged to {}", actor);
         actor.tell(new DataTreeChanged(changes), ActorRef.noSender());
     }
+
+    @Override
+    public String toString() {
+        return "ForwardingDataTreeChangeListener [actor=" + actor + "]";
+    }
 }
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataChangeListenerPublisherActorProxy.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataChangeListenerPublisherActorProxy.java
index a17c603596..28b58f7262 100644
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataChangeListenerPublisherActorProxy.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataChangeListenerPublisherActorProxy.java
@@ -40,7 +40,7 @@ class ShardDataChangeListenerPublisherActorProxy extends AbstractShardDataTreeNo
             Optional<DataTreeCandidate> initialState,
             Consumer<ListenerRegistration<AsyncDataChangeListener<YangInstanceIdentifier, NormalizedNode<?, ?>>>>
                 onRegistration) {
-        notifierActor().tell(new ShardDataChangePublisherActor.RegisterListener(path, listener, scope, initialState,
+        publisherActor().tell(new ShardDataChangePublisherActor.RegisterListener(path, listener, scope, initialState,
                 onRegistration), ActorRef.noSender());
     }
 
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataChangePublisherActor.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataChangePublisherActor.java
index cb7d80dbdc..ef40480e4d 100644
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataChangePublisherActor.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataChangePublisherActor.java
@@ -27,7 +27,7 @@ public class ShardDataChangePublisherActor
         extends ShardDataTreeNotificationPublisherActor<ShardDataChangeListenerPublisher> {
 
     private ShardDataChangePublisherActor(final String name, final String logContext) {
-        super(new DefaultShardDataChangeListenerPublisher(), name, logContext);
+        super(new DefaultShardDataChangeListenerPublisher(logContext), name, logContext);
     }
 
     @Override
@@ -36,7 +36,7 @@ public class ShardDataChangePublisherActor
             RegisterListener reg = (RegisterListener)message;
             if (reg.initialState.isPresent()) {
                 DefaultShardDataChangeListenerPublisher.notifySingleListener(reg.path, reg.listener, reg.scope,
-                        reg.initialState.get());
+                        reg.initialState.get(), logContext());
             }
 
             publisher().registerDataChangeListener(reg.path, reg.listener, reg.scope, Optional.absent(),
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataTree.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataTree.java
index 1460982bf3..4eeed882fa 100644
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataTree.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataTree.java
@@ -163,7 +163,8 @@ public class ShardDataTree extends ShardDataTreeTransactionParent {
     @VisibleForTesting
     public ShardDataTree(final Shard shard, final SchemaContext schemaContext, final TreeType treeType) {
         this(shard, schemaContext, treeType, YangInstanceIdentifier.EMPTY,
-                new DefaultShardDataTreeChangeListenerPublisher(), new DefaultShardDataChangeListenerPublisher(), "");
+                new DefaultShardDataTreeChangeListenerPublisher(""),
+                new DefaultShardDataChangeListenerPublisher(""), "");
     }
 
     final String logContext() {
@@ -538,8 +539,8 @@ public class ShardDataTree extends ShardDataTreeTransactionParent {
 
     @VisibleForTesting
     public void notifyListeners(final DataTreeCandidate candidate) {
-        treeChangeListenerPublisher.publishChanges(candidate, logContext);
-        dataChangeListenerPublisher.publishChanges(candidate, logContext);
+        treeChangeListenerPublisher.publishChanges(candidate);
+        dataChangeListenerPublisher.publishChanges(candidate);
     }
 
     /**
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataTreeChangeListenerPublisherActorProxy.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataTreeChangeListenerPublisherActorProxy.java
index ceaeeccaad..020319203b 100644
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataTreeChangeListenerPublisherActorProxy.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataTreeChangeListenerPublisherActorProxy.java
@@ -36,8 +36,10 @@ class ShardDataTreeChangeListenerPublisherActorProxy extends AbstractShardDataTr
     public void registerTreeChangeListener(YangInstanceIdentifier treeId,
             DOMDataTreeChangeListener listener, Optional<DataTreeCandidate> currentState,
             Consumer<ListenerRegistration<DOMDataTreeChangeListener>> onRegistration) {
-        notifierActor().tell(new ShardDataTreeChangePublisherActor.RegisterListener(treeId, listener, currentState,
-                onRegistration), ActorRef.noSender());
+        final ShardDataTreeChangePublisherActor.RegisterListener regMessage =
+                new ShardDataTreeChangePublisherActor.RegisterListener(treeId, listener, currentState, onRegistration);
+        log.debug("{}: Sending {} to publisher actor {}", logContext(), regMessage, publisherActor());
+        publisherActor().tell(regMessage, ActorRef.noSender());
     }
 
     @Override
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataTreeChangePublisherActor.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataTreeChangePublisherActor.java
index f054ccdae8..61a3fa13be 100644
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataTreeChangePublisherActor.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataTreeChangePublisherActor.java
@@ -25,16 +25,17 @@ public class ShardDataTreeChangePublisherActor
         extends ShardDataTreeNotificationPublisherActor<ShardDataTreeChangeListenerPublisher> {
 
     private ShardDataTreeChangePublisherActor(final String name, final String logContext) {
-        super(new DefaultShardDataTreeChangeListenerPublisher(), name, logContext);
+        super(new DefaultShardDataTreeChangeListenerPublisher(logContext), name, logContext);
     }
 
     @Override
     protected void handleReceive(Object message) {
         if (message instanceof RegisterListener) {
             RegisterListener reg = (RegisterListener)message;
+            LOG.debug("{}: Received {}", logContext(), reg);
             if (reg.initialState.isPresent()) {
                 DefaultShardDataTreeChangeListenerPublisher.notifySingleListener(reg.path, reg.listener,
-                        reg.initialState.get());
+                        reg.initialState.get(), logContext());
             }
 
             publisher().registerTreeChangeListener(reg.path, reg.listener, Optional.absent(), reg.onRegistration);
@@ -61,5 +62,11 @@ public class ShardDataTreeChangePublisherActor
             this.initialState = Preconditions.checkNotNull(initialState);
             this.onRegistration = Preconditions.checkNotNull(onRegistration);
         }
+
+        @Override
+        public String toString() {
+            return "RegisterListener [path=" + path + ", listener=" + listener + ", initialState present="
+                    + initialState.isPresent() + "]";
+        }
     }
 }
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataTreeNotificationPublisher.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataTreeNotificationPublisher.java
index 30b7eaf2ae..4dbd818488 100644
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataTreeNotificationPublisher.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataTreeNotificationPublisher.java
@@ -18,5 +18,5 @@ import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidate;
 interface ShardDataTreeNotificationPublisher {
     long PUBLISH_DELAY_THRESHOLD_IN_MS = TimeUnit.MILLISECONDS.convert(1, TimeUnit.SECONDS);
 
-    void publishChanges(DataTreeCandidate candidate, String logContext);
+    void publishChanges(DataTreeCandidate candidate);
 }
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataTreeNotificationPublisherActor.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataTreeNotificationPublisherActor.java
index ea08830829..c22bc3bd98 100644
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataTreeNotificationPublisherActor.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardDataTreeNotificationPublisherActor.java
@@ -46,7 +46,7 @@ public class ShardDataTreeNotificationPublisherActor<T extends ShardDataTreeNoti
             timer.start();
 
             try {
-                publisher.publishChanges(toPublish.candidate, logContext);
+                publisher.publishChanges(toPublish.candidate);
             } finally {
                 long elapsedTime = timer.elapsed(TimeUnit.MILLISECONDS);
 
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/messages/EnableNotification.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/messages/EnableNotification.java
index 67dab7e663..3cd96177f5 100644
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/messages/EnableNotification.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/messages/EnableNotification.java
@@ -10,12 +10,18 @@ package org.opendaylight.controller.cluster.datastore.messages;
 
 public class EnableNotification {
     private final boolean enabled;
+    private final String logContext;
 
-    public EnableNotification(boolean enabled) {
+    public EnableNotification(boolean enabled, String logContext) {
         this.enabled = enabled;
+        this.logContext = logContext;
     }
 
     public boolean isEnabled() {
         return enabled;
     }
+
+    public String getLogContext() {
+        return logContext;
+    }
 }
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/messages/RegisterChangeListener.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/messages/RegisterChangeListener.java
index 163525a8ca..d83ff2483a 100644
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/messages/RegisterChangeListener.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/messages/RegisterChangeListener.java
@@ -45,4 +45,10 @@ public class RegisterChangeListener implements ListenerRegistrationMessage {
     public boolean isRegisterOnAllInstances() {
         return registerOnAllInstances;
     }
+
+    @Override
+    public String toString() {
+        return "RegisterChangeListener [path=" + path + ", scope=" + scope + ", registerOnAllInstances="
+                + registerOnAllInstances + ", dataChangeListenerActor=" + dataChangeListenerActor + "]";
+    }
 }
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/messages/RegisterDataTreeChangeListener.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/messages/RegisterDataTreeChangeListener.java
index abf072b640..e091b5bd0f 100644
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/messages/RegisterDataTreeChangeListener.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/messages/RegisterDataTreeChangeListener.java
@@ -66,4 +66,10 @@ public final class RegisterDataTreeChangeListener implements Externalizable, Lis
         path = SerializationUtils.deserializePath(in);
         registerOnAllInstances = in.readBoolean();
     }
+
+    @Override
+    public String toString() {
+        return "RegisterDataTreeChangeListener [path=" + path + ", registerOnAllInstances=" + registerOnAllInstances
+                + ", dataTreeChangeListenerPath=" + dataTreeChangeListenerPath + "]";
+    }
 }
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/shardmanager/ShardManager.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/shardmanager/ShardManager.java
index 4ae1540776..a8eebf83a6 100644
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/shardmanager/ShardManager.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/shardmanager/ShardManager.java
@@ -298,7 +298,7 @@ class ShardManager extends AbstractUntypedPersistentActorWithMetering {
     }
 
     private void onInitConfigListener() {
-        LOG.debug("{}: Initializing config listener.", persistenceId());
+        LOG.debug("{}: Initializing config listener on {}", persistenceId(), cluster.getCurrentMemberName());
 
         final org.opendaylight.mdsal.common.api.LogicalDatastoreType type =
                 org.opendaylight.mdsal.common.api.LogicalDatastoreType
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/sharding/PrefixedShardConfigUpdateHandler.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/sharding/PrefixedShardConfigUpdateHandler.java
index f8ecdb804c..ac209e3c6d 100644
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/sharding/PrefixedShardConfigUpdateHandler.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/sharding/PrefixedShardConfigUpdateHandler.java
@@ -77,6 +77,7 @@ public class PrefixedShardConfigUpdateHandler {
         private final MemberName memberName;
         private final LogicalDatastoreType type;
         private final ActorRef handlingActor;
+        private final String logName;
 
         public ShardConfigHandler(final MemberName memberName,
                            final LogicalDatastoreType type,
@@ -84,6 +85,7 @@ public class PrefixedShardConfigUpdateHandler {
             this.memberName = memberName;
             this.type = type;
             this.handlingActor = handlingActor;
+            logName = memberName.getName() + "-" + type;
         }
 
         @Override
@@ -111,8 +113,8 @@ public class PrefixedShardConfigUpdateHandler {
 
         private void resolveWrite(final DataTreeCandidateNode rootNode) {
 
-            LOG.debug("{}: New config received {}", memberName, rootNode);
-            LOG.debug("{}: Data after: {}", memberName, rootNode.getDataAfter());
+            LOG.debug("{}: New config received {}", logName, rootNode);
+            LOG.debug("{}: Data after: {}", logName, rootNode.getDataAfter());
 
             // were in the shards list, iter children and resolve
             for (final DataTreeCandidateNode childNode : rootNode.getChildNodes()) {
@@ -134,6 +136,7 @@ public class PrefixedShardConfigUpdateHandler {
             }
         }
 
+        @SuppressWarnings("unchecked")
         private void resolveWrittenShard(final DataTreeCandidateNode childNode) {
             final MapEntryNode entryNode = (MapEntryNode) childNode.getDataAfter().get();
             final LeafNode<YangInstanceIdentifier> prefix =
@@ -141,7 +144,7 @@ public class PrefixedShardConfigUpdateHandler {
 
             final YangInstanceIdentifier identifier = prefix.getValue();
 
-            LOG.debug("{}: Deserialized {} from datastore", memberName, identifier);
+            LOG.debug("{}: Deserialized {} from datastore", logName, identifier);
 
             final ContainerNode replicas =
                     (ContainerNode) entryNode.getChild(new NodeIdentifier(SHARD_REPLICAS_QNAME)).get();
@@ -153,13 +156,13 @@ public class PrefixedShardConfigUpdateHandler {
                     .map(child -> MemberName.forName(child.getValue()))
                     .collect(Collectors.toList());
 
-            LOG.debug("{}: Replicas read from ds {}", memberName, retReplicas.toString());
+            LOG.debug("{}: Replicas read from ds {}", logName, retReplicas.toString());
 
             final PrefixShardConfiguration newConfig =
                     new PrefixShardConfiguration(new DOMDataTreeIdentifier(type, identifier),
                             PrefixShardStrategy.NAME, retReplicas);
 
-            LOG.debug("{}: Resulting config {}", memberName, newConfig);
+            LOG.debug("{}: Resulting config {} - sending PrefixShardCreated to {}", logName, newConfig, handlingActor);
 
             handlingActor.tell(new PrefixShardCreated(newConfig), noSender());
         }
@@ -183,5 +186,10 @@ public class PrefixedShardConfigUpdateHandler {
         private void resolveDelete(final DataTreeCandidateNode rootNode) {
 
         }
+
+        @Override
+        public String toString() {
+            return "ShardConfigHandler [logName=" + logName + ", handlingActor=" + handlingActor + "]";
+        }
     }
 }
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DataChangeListenerTest.java b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DataChangeListenerTest.java
index f3dcaa2b70..a2e6b117eb 100644
--- a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DataChangeListenerTest.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DataChangeListenerTest.java
@@ -37,7 +37,7 @@ public class DataChangeListenerTest extends AbstractActorTest {
 
                 // Let the DataChangeListener know that notifications should be
                 // enabled
-                subject.tell(new EnableNotification(true), getRef());
+                subject.tell(new EnableNotification(true, "test"), getRef());
 
                 subject.tell(new DataChanged(mockChangeEvent), getRef());
 
@@ -122,7 +122,7 @@ public class DataChangeListenerTest extends AbstractActorTest {
 
                 // Let the DataChangeListener know that notifications should be
                 // enabled
-                subject.tell(new EnableNotification(true), getRef());
+                subject.tell(new EnableNotification(true, "test"), getRef());
 
                 subject.tell(new DataChanged(mockChangeEvent1), getRef());
                 expectMsgClass(DataChangedReply.class);
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DataTreeChangeListenerActorTest.java b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DataTreeChangeListenerActorTest.java
index 62b95c214b..a98d55a99f 100644
--- a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DataTreeChangeListenerActorTest.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DataTreeChangeListenerActorTest.java
@@ -38,7 +38,7 @@ public class DataTreeChangeListenerActorTest extends AbstractActorTest {
 
                 // Let the DataChangeListener know that notifications should be
                 // enabled
-                subject.tell(new EnableNotification(true), getRef());
+                subject.tell(new EnableNotification(true, "test"), getRef());
 
                 subject.tell(new DataTreeChanged(mockCandidates), getRef());
 
@@ -126,7 +126,7 @@ public class DataTreeChangeListenerActorTest extends AbstractActorTest {
 
                 // Let the DataChangeListener know that notifications should be
                 // enabled
-                subject.tell(new EnableNotification(true), getRef());
+                subject.tell(new EnableNotification(true, "test"), getRef());
 
                 subject.tell(new DataTreeChanged(mockCandidates1), getRef());
                 expectMsgClass(DataTreeChangedReply.class);
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DataTreeChangeListenerProxyTest.java b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DataTreeChangeListenerProxyTest.java
index 6f2f99b24e..820fe5614f 100644
--- a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DataTreeChangeListenerProxyTest.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/DataTreeChangeListenerProxyTest.java
@@ -226,6 +226,7 @@ public class DataTreeChangeListenerProxyTest extends AbstractActorTest {
                 final YangInstanceIdentifier path = YangInstanceIdentifier.of(TestModel.TEST_QNAME);
 
                 doReturn(executor).when(actorContext).getClientDispatcher();
+                doReturn(DatastoreContext.newBuilder().build()).when(actorContext).getDatastoreContext();
                 doReturn(mockActorSystem).when(actorContext).getActorSystem();
 
                 String shardName = "shard-1";