From: Robert Varga <robert.varga@pantheon.tech>
Date: Sat, 15 Jul 2017 21:33:25 +0000 (+0200)
Subject: BUG-8618: record LeaderFrontendState time
X-Git-Tag: release/nitrogen~32
X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?p=controller.git;a=commitdiff_plain;h=730ff1686220fca0176a8757b4444d63ab2afc44

BUG-8618: record LeaderFrontendState time

In order to deal with IsolatedLeader state and transaction timeouts,
we need to maintain an accurate view of when we have seen the frontend
even if we are not accepting messages from it.

Add correspoding field and maintain it whenever we interact with
LeaderFrontend state. Also record last connect ticks for the same use.

Change-Id: I8e49037507fcd01470a03be8c0d611efca55dabf
Signed-off-by: Robert Varga <robert.varga@pantheon.tech>
(cherry picked from commit 7633a2a50144dad7cf987b29959dc06509575c05)
---

diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/LeaderFrontendState.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/LeaderFrontendState.java
index 5a5e42637e..b37e2d8bf1 100644
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/LeaderFrontendState.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/LeaderFrontendState.java
@@ -57,6 +57,8 @@ final class LeaderFrontendState implements Identifiable<ClientIdentifier> {
     private final ClientIdentifier clientId;
     private final String persistenceId;
 
+    private long lastConnectTicks;
+    private long lastSeenTicks;
     private long expectedTxSequence;
     private Long lastSeenHistory = null;
 
@@ -83,6 +85,7 @@ final class LeaderFrontendState implements Identifiable<ClientIdentifier> {
         this.purgedHistories = Preconditions.checkNotNull(purgedHistories);
         this.standaloneHistory = Preconditions.checkNotNull(standaloneHistory);
         this.localHistories = Preconditions.checkNotNull(localHistories);
+        this.lastSeenTicks = tree.readTime();
     }
 
     @Override
@@ -214,6 +217,7 @@ final class LeaderFrontendState implements Identifiable<ClientIdentifier> {
 
     void reconnect() {
         expectedTxSequence = 0;
+        lastConnectTicks = tree.readTime();
     }
 
     void retire() {
@@ -238,9 +242,24 @@ final class LeaderFrontendState implements Identifiable<ClientIdentifier> {
         standaloneHistory.retire();
     }
 
+    long getLastConnectTicks() {
+        return lastConnectTicks;
+    }
+
+    long getLastSeenTicks() {
+        return lastSeenTicks;
+    }
+
+    void touch() {
+        this.lastSeenTicks = tree.readTime();
+    }
+
     @Override
     public String toString() {
-        return MoreObjects.toStringHelper(LeaderFrontendState.class).add("clientId", clientId)
-                .add("purgedHistories", purgedHistories).toString();
+        return MoreObjects.toStringHelper(LeaderFrontendState.class)
+                .add("clientId", clientId)
+                .add("nanosAgo", tree.readTime() - lastSeenTicks)
+                .add("purgedHistories", purgedHistories)
+                .toString();
     }
 }
diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/Shard.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/Shard.java
index d9483d7b2b..768b19fa3e 100644
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/Shard.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/Shard.java
@@ -36,6 +36,7 @@ import org.opendaylight.controller.cluster.access.commands.ConnectClientRequest;
 import org.opendaylight.controller.cluster.access.commands.ConnectClientSuccess;
 import org.opendaylight.controller.cluster.access.commands.LocalHistoryRequest;
 import org.opendaylight.controller.cluster.access.commands.NotLeaderException;
+import org.opendaylight.controller.cluster.access.commands.OutOfSequenceEnvelopeException;
 import org.opendaylight.controller.cluster.access.commands.TransactionRequest;
 import org.opendaylight.controller.cluster.access.concepts.ClientIdentifier;
 import org.opendaylight.controller.cluster.access.concepts.FrontendIdentifier;
@@ -417,11 +418,13 @@ public class Shard extends RaftActor {
     }
 
     // Acquire our frontend tracking handle and verify generation matches
-    private LeaderFrontendState getFrontend(final ClientIdentifier clientId) throws RequestException {
+    @Nullable
+    private LeaderFrontendState findFrontend(final ClientIdentifier clientId) throws RequestException {
         final LeaderFrontendState existing = knownFrontends.get(clientId.getFrontendId());
         if (existing != null) {
             final int cmp = Long.compareUnsigned(existing.getIdentifier().getGeneration(), clientId.getGeneration());
             if (cmp == 0) {
+                existing.touch();
                 return existing;
             }
             if (cmp > 0) {
@@ -436,10 +439,17 @@ public class Shard extends RaftActor {
             LOG.debug("{}: client {} is not yet known", persistenceId(), clientId);
         }
 
-        final LeaderFrontendState ret = new LeaderFrontendState(persistenceId(), clientId, store);
-        knownFrontends.put(clientId.getFrontendId(), ret);
-        LOG.debug("{}: created state {} for client {}", persistenceId(), ret, clientId);
-        return ret;
+        return null;
+    }
+
+    private LeaderFrontendState getFrontend(final ClientIdentifier clientId) throws RequestException {
+        final LeaderFrontendState ret = findFrontend(clientId);
+        if (ret != null) {
+            return ret;
+        }
+
+        // TODO: a dedicated exception would be better, but this is technically true, too
+        throw new OutOfSequenceEnvelopeException(0);
     }
 
     private static @Nonnull ABIVersion selectVersion(final ConnectClientRequest message) {
@@ -458,6 +468,12 @@ public class Shard extends RaftActor {
     @SuppressWarnings("checkstyle:IllegalCatch")
     private void handleConnectClient(final ConnectClientRequest message) {
         try {
+            final ClientIdentifier clientId = message.getTarget();
+            final LeaderFrontendState existing = findFrontend(clientId);
+            if (existing != null) {
+                existing.touch();
+            }
+
             if (!isLeader() || !isLeaderActive()) {
                 LOG.info("{}: not currently leader, rejecting request {}. isLeader: {}, isLeaderActive: {},"
                                 + "isLeadershipTransferInProgress: {}.",
@@ -466,7 +482,15 @@ public class Shard extends RaftActor {
             }
 
             final ABIVersion selectedVersion = selectVersion(message);
-            final LeaderFrontendState frontend = getFrontend(message.getTarget());
+            final LeaderFrontendState frontend;
+            if (existing == null) {
+                frontend = new LeaderFrontendState(persistenceId(), clientId, store);
+                knownFrontends.put(clientId.getFrontendId(), frontend);
+                LOG.debug("{}: created state {} for client {}", persistenceId(), frontend, clientId);
+            } else {
+                frontend = existing;
+            }
+
             frontend.reconnect();
             message.getReplyTo().tell(new ConnectClientSuccess(message.getTarget(), message.getSequence(), getSelf(),
                 ImmutableList.of(), store.getDataTree(), CLIENT_MAX_MESSAGES).toVersion(selectedVersion),