BUG-8618: record LeaderFrontendState time

In order to deal with IsolatedLeader state and transaction timeouts,
we need to maintain an accurate view of when we have seen the frontend
even if we are not accepting messages from it.

Add correspoding field and maintain it whenever we interact with
LeaderFrontend state. Also record last connect ticks for the same use.

Change-Id: I8e49037507fcd01470a03be8c0d611efca55dabf
Signed-off-by: Robert Varga <robert.varga@pantheon.tech>

diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/LeaderFrontendState.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/LeaderFrontendState.java
index 061e035c6647c883c84916005e0c4b81b512820f..ebc384b8e1899bbf753d93ba3c22c27e85093a76 100644 (file)
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/LeaderFrontendState.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/LeaderFrontendState.java
@@ -60,6 +60,8 @@ final class LeaderFrontendState implements Identifiable<ClientIdentifier> {
     private final ClientIdentifier clientId;
     private final String persistenceId;
 
+    private long lastConnectTicks;
+    private long lastSeenTicks;
     private long expectedTxSequence;
     private Long lastSeenHistory = null;
 
@@ -86,6 +88,7 @@ final class LeaderFrontendState implements Identifiable<ClientIdentifier> {
         this.purgedHistories = Preconditions.checkNotNull(purgedHistories);
         this.standaloneHistory = Preconditions.checkNotNull(standaloneHistory);
         this.localHistories = Preconditions.checkNotNull(localHistories);
+        this.lastSeenTicks = tree.readTime();
     }
 
     @Override
@@ -218,6 +221,7 @@ final class LeaderFrontendState implements Identifiable<ClientIdentifier> {
 
     void reconnect() {
         expectedTxSequence = 0;
+        lastConnectTicks = tree.readTime();
     }
 
     void retire() {
@@ -242,9 +246,24 @@ final class LeaderFrontendState implements Identifiable<ClientIdentifier> {
         standaloneHistory.retire();
     }
 
+    long getLastConnectTicks() {
+        return lastConnectTicks;
+    }
+
+    long getLastSeenTicks() {
+        return lastSeenTicks;
+    }
+
+    void touch() {
+        this.lastSeenTicks = tree.readTime();
+    }
+
     @Override
     public String toString() {
-        return MoreObjects.toStringHelper(LeaderFrontendState.class).add("clientId", clientId)
-                .add("purgedHistories", purgedHistories).toString();
+        return MoreObjects.toStringHelper(LeaderFrontendState.class)
+                .add("clientId", clientId)
+                .add("nanosAgo", tree.readTime() - lastSeenTicks)
+                .add("purgedHistories", purgedHistories)
+                .toString();
     }
 }

diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/Shard.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/Shard.java
index 318a4e68e793e01c01cc0316099f8134d2aebec1..8bf4257a1df8cb8a39688ab88e0fb5515ec1ffb0 100644 (file)
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/Shard.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/Shard.java
@@ -36,6 +36,7 @@ import org.opendaylight.controller.cluster.access.commands.ConnectClientRequest;
 import org.opendaylight.controller.cluster.access.commands.ConnectClientSuccess;
 import org.opendaylight.controller.cluster.access.commands.LocalHistoryRequest;
 import org.opendaylight.controller.cluster.access.commands.NotLeaderException;
+import org.opendaylight.controller.cluster.access.commands.OutOfSequenceEnvelopeException;
 import org.opendaylight.controller.cluster.access.commands.TransactionRequest;
 import org.opendaylight.controller.cluster.access.concepts.ClientIdentifier;
 import org.opendaylight.controller.cluster.access.concepts.FrontendIdentifier;
@@ -391,11 +392,13 @@ public class Shard extends RaftActor {
     }
 
     // Acquire our frontend tracking handle and verify generation matches
-    private LeaderFrontendState getFrontend(final ClientIdentifier clientId) throws RequestException {
+    @Nullable
+    private LeaderFrontendState findFrontend(final ClientIdentifier clientId) throws RequestException {
         final LeaderFrontendState existing = knownFrontends.get(clientId.getFrontendId());
         if (existing != null) {
             final int cmp = Long.compareUnsigned(existing.getIdentifier().getGeneration(), clientId.getGeneration());
             if (cmp == 0) {
+                existing.touch();
                 return existing;
             }
             if (cmp > 0) {
@@ -410,10 +413,17 @@ public class Shard extends RaftActor {
             LOG.debug("{}: client {} is not yet known", persistenceId(), clientId);
         }
 
-        final LeaderFrontendState ret = new LeaderFrontendState(persistenceId(), clientId, store);
-        knownFrontends.put(clientId.getFrontendId(), ret);
-        LOG.debug("{}: created state {} for client {}", persistenceId(), ret, clientId);
-        return ret;
+        return null;
+    }
+
+    private LeaderFrontendState getFrontend(final ClientIdentifier clientId) throws RequestException {
+        final LeaderFrontendState ret = findFrontend(clientId);
+        if (ret != null) {
+            return ret;
+        }
+
+        // TODO: a dedicated exception would be better, but this is technically true, too
+        throw new OutOfSequenceEnvelopeException(0);
     }
 
     private static @Nonnull ABIVersion selectVersion(final ConnectClientRequest message) {
@@ -432,6 +442,12 @@ public class Shard extends RaftActor {
     @SuppressWarnings("checkstyle:IllegalCatch")
     private void handleConnectClient(final ConnectClientRequest message) {
         try {
+            final ClientIdentifier clientId = message.getTarget();
+            final LeaderFrontendState existing = findFrontend(clientId);
+            if (existing != null) {
+                existing.touch();
+            }
+
             if (!isLeader() || !isLeaderActive()) {
                 LOG.info("{}: not currently leader, rejecting request {}. isLeader: {}, isLeaderActive: {},"
                                 + "isLeadershipTransferInProgress: {}.",
@@ -440,7 +456,15 @@ public class Shard extends RaftActor {
             }
 
             final ABIVersion selectedVersion = selectVersion(message);
-            final LeaderFrontendState frontend = getFrontend(message.getTarget());
+            final LeaderFrontendState frontend;
+            if (existing == null) {
+                frontend = new LeaderFrontendState(persistenceId(), clientId, store);
+                knownFrontends.put(clientId.getFrontendId(), frontend);
+                LOG.debug("{}: created state {} for client {}", persistenceId(), frontend, clientId);
+            } else {
+                frontend = existing;
+            }
+
             frontend.reconnect();
             message.getReplyTo().tell(new ConnectClientSuccess(message.getTarget(), message.getSequence(), getSelf(),
                 ImmutableList.of(), store.getDataTree(), CLIENT_MAX_MESSAGES).toVersion(selectedVersion),