BUG-8402: Separate out OutOfOrderRequestException

OutOfOrderRequestException is used for two distinct cases, which is
a mixup during refactor.

The first case is when an envelope's sequence does not match the
sequence we are expecting on a connection. This is a retriable
exception and happens due to mailbox queueing during leadership
changes:
- a FE sees us as a leader, sends requests
- we become a follower, we reject a few requests
- we become a leader, at which point we must not process requests
  until the FE reconnects, as we would not be processing them in
  the correct order.

The second case is when we receive a Request with an unexpected
sequence. This is a hard error, as it indicates that the client
has made a mistake and lost a request (like the case fixed in
fe69101801085580f2fe72762abea5c5fa83d978).

Separate these two cases out by introducing
OutOfSequenceEnvelopeException and handle it by initiating a session
reconnect.

Change-Id: Ifb0bac41ff2efd6385455fd9c77b8b39054dd4a0
Signed-off-by: Robert Varga <robert.varga@pantheon.tech>
(cherry picked from commit d02d60083ee163cf465c265364c21c0df9cdc3c7)

diff --git a/opendaylight/md-sal/cds-access-api/src/main/java/org/opendaylight/controller/cluster/access/commands/OutOfOrderRequestException.java b/opendaylight/md-sal/cds-access-api/src/main/java/org/opendaylight/controller/cluster/access/commands/OutOfOrderRequestException.java
index 07a3dba6200f80fdd1ee64dc9e94a38e25cce179..cd110d66b6aa708623fcca696a9daaec17872d44 100644 (file)
--- a/opendaylight/md-sal/cds-access-api/src/main/java/org/opendaylight/controller/cluster/access/commands/OutOfOrderRequestException.java
+++ b/opendaylight/md-sal/cds-access-api/src/main/java/org/opendaylight/controller/cluster/access/commands/OutOfOrderRequestException.java
@@ -11,8 +11,8 @@ import com.google.common.annotations.Beta;
 import org.opendaylight.controller.cluster.access.concepts.RequestException;
 
 /**
- * A {@link RequestException} indicating that the backend has received an unexpected request. This would typically
- * mean that messages are not arriving in the sequence they were generated by the frontend.
+ * A {@link RequestException} indicating that the backend has received a Request whose sequence does not match the
+ * next expected sequence for the target. This is a hard error, as it indicates a Request is missing in the stream.
  *
  * @author Robert Varga
  */
@@ -26,6 +26,6 @@ public final class OutOfOrderRequestException extends RequestException {
 
     @Override
     public boolean isRetriable() {
-        return true;
+        return false;
     }
 }

diff --git a/opendaylight/md-sal/cds-access-api/src/main/java/org/opendaylight/controller/cluster/access/commands/OutOfSequenceEnvelopeException.java b/opendaylight/md-sal/cds-access-api/src/main/java/org/opendaylight/controller/cluster/access/commands/OutOfSequenceEnvelopeException.java
new file mode 100644 (file)
index 0000000..ad3dd8d
--- /dev/null
+++ b/opendaylight/md-sal/cds-access-api/src/main/java/org/opendaylight/controller/cluster/access/commands/OutOfSequenceEnvelopeException.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2017 Pantheon Technologies, s.r.o. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+package org.opendaylight.controller.cluster.access.commands;
+
+import com.google.common.annotations.Beta;
+import org.opendaylight.controller.cluster.access.concepts.RequestException;
+
+/**
+ * A {@link RequestException} indicating that the backend has received a RequestEnvelope whose sequence does not match
+ * the next expected sequence. This can happen during leader transitions, when a part of the stream is rejected because
+ * the backend is not the leader and it transitions to being a leader with old stream messages still being present.
+ *
+ * @author Robert Varga
+ */
+@Beta
+public final class OutOfSequenceEnvelopeException extends RequestException {
+    private static final long serialVersionUID = 1L;
+
+    public OutOfSequenceEnvelopeException(final long expectedEnvelope) {
+        super("Expecting envelope " + Long.toUnsignedString(expectedEnvelope));
+    }
+
+    @Override
+    public boolean isRetriable() {
+        return true;
+    }
+}

diff --git a/opendaylight/md-sal/cds-access-api/src/test/java/org/opendaylight/controller/cluster/access/commands/OutOfOrderRequestExceptionTest.java b/opendaylight/md-sal/cds-access-api/src/test/java/org/opendaylight/controller/cluster/access/commands/OutOfOrderRequestExceptionTest.java
index 4f045adbd831f837ccbc590ba75d4a495c050e7a..135f31c30280dd59d1d0748fae01c48ac08c1c83 100644 (file)
--- a/opendaylight/md-sal/cds-access-api/src/test/java/org/opendaylight/controller/cluster/access/commands/OutOfOrderRequestExceptionTest.java
+++ b/opendaylight/md-sal/cds-access-api/src/test/java/org/opendaylight/controller/cluster/access/commands/OutOfOrderRequestExceptionTest.java
@@ -7,6 +7,7 @@
  */
 package org.opendaylight.controller.cluster.access.commands;
 
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
@@ -19,7 +20,7 @@ public class OutOfOrderRequestExceptionTest extends RequestExceptionTest<OutOfOr
 
     @Override
     protected void isRetriable() {
-        assertTrue(OBJECT.isRetriable());
+        assertFalse(OBJECT.isRetriable());
     }
 
     @Override

diff --git a/opendaylight/md-sal/cds-access-client/src/main/java/org/opendaylight/controller/cluster/access/client/ClientActorBehavior.java b/opendaylight/md-sal/cds-access-client/src/main/java/org/opendaylight/controller/cluster/access/client/ClientActorBehavior.java
index e70fdc9662c6232dcc81bda50d95795cca02dbde..ca78d0cb66f85a52b003757ec2c23d75c4acdc95 100644 (file)
--- a/opendaylight/md-sal/cds-access-client/src/main/java/org/opendaylight/controller/cluster/access/client/ClientActorBehavior.java
+++ b/opendaylight/md-sal/cds-access-client/src/main/java/org/opendaylight/controller/cluster/access/client/ClientActorBehavior.java
@@ -18,6 +18,7 @@ import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import javax.annotation.concurrent.GuardedBy;
 import org.opendaylight.controller.cluster.access.commands.NotLeaderException;
+import org.opendaylight.controller.cluster.access.commands.OutOfSequenceEnvelopeException;
 import org.opendaylight.controller.cluster.access.concepts.ClientIdentifier;
 import org.opendaylight.controller.cluster.access.concepts.FailureEnvelope;
 import org.opendaylight.controller.cluster.access.concepts.LocalHistoryIdentifier;
@@ -175,6 +176,17 @@ public abstract class ClientActorBehavior<T extends BackendInfo> extends
                 return conn.reconnect(this);
             }
         }
+        if (cause instanceof OutOfSequenceEnvelopeException) {
+            final AbstractClientConnection<T> conn = getConnection(command);
+            if (conn instanceof ReconnectingClientConnection) {
+                // Already reconnecting, do not churn the logs
+                return this;
+            } else if (conn != null) {
+                LOG.info("{}: connection {} indicated no sequencing mismatch on {} sequence {}, reconnecting it",
+                    persistenceId(), conn, failure.getTarget(), failure.getSequence(), cause);
+                return conn.reconnect(this);
+            }
+        }
 
         return onRequestFailure(command);
     }

diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/AbstractFrontendHistory.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/AbstractFrontendHistory.java
index 5690f1b0d5f4417214d9106099dfd3b046eab0b0..8262afa099154c32ffea89a9ad3896d709556f01 100644 (file)
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/AbstractFrontendHistory.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/AbstractFrontendHistory.java
@@ -44,7 +44,6 @@ import org.slf4j.LoggerFactory;
  */
 abstract class AbstractFrontendHistory implements Identifiable<LocalHistoryIdentifier> {
     private static final Logger LOG = LoggerFactory.getLogger(AbstractFrontendHistory.class);
-    private static final OutOfOrderRequestException UNSEQUENCED_START = new OutOfOrderRequestException(0);
 
     private final Map<TransactionIdentifier, FrontendTransaction> transactions = new HashMap<>();
     private final RangeSet<UnsignedLong> purgedTransactions;
@@ -137,7 +136,7 @@ abstract class AbstractFrontendHistory implements Identifiable<LocalHistoryIdent
             // The transaction does not exist and we are about to create it, check sequence number
             if (request.getSequence() != 0) {
                 LOG.debug("{}: no transaction state present, unexpected request {}", persistenceId(), request);
-                throw UNSEQUENCED_START;
+                throw new OutOfOrderRequestException(0);
             }
 
             tx = createTransaction(request, id);

diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/LeaderFrontendState.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/LeaderFrontendState.java
index 92b696ac4ca96c50250e71a2d63d9be3975aaa2b..24de062227878bb1151e2e82247c3fb2f3390516 100644 (file)
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/LeaderFrontendState.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/LeaderFrontendState.java
@@ -22,7 +22,7 @@ import org.opendaylight.controller.cluster.access.commands.DeadHistoryException;
 import org.opendaylight.controller.cluster.access.commands.DestroyLocalHistoryRequest;
 import org.opendaylight.controller.cluster.access.commands.LocalHistoryRequest;
 import org.opendaylight.controller.cluster.access.commands.LocalHistorySuccess;
-import org.opendaylight.controller.cluster.access.commands.OutOfOrderRequestException;
+import org.opendaylight.controller.cluster.access.commands.OutOfSequenceEnvelopeException;
 import org.opendaylight.controller.cluster.access.commands.PurgeLocalHistoryRequest;
 import org.opendaylight.controller.cluster.access.commands.TransactionRequest;
 import org.opendaylight.controller.cluster.access.commands.TransactionSuccess;
@@ -91,9 +91,9 @@ final class LeaderFrontendState implements Identifiable<ClientIdentifier> {
         return clientId;
     }
 
-    private void checkRequestSequence(final RequestEnvelope envelope) throws OutOfOrderRequestException {
+    private void checkRequestSequence(final RequestEnvelope envelope) throws OutOfSequenceEnvelopeException {
         if (expectedTxSequence != envelope.getTxSequence()) {
-            throw new OutOfOrderRequestException(expectedTxSequence);
+            throw new OutOfSequenceEnvelopeException(expectedTxSequence);
         }
     }