From d02d60083ee163cf465c265364c21c0df9cdc3c7 Mon Sep 17 00:00:00 2001 From: Robert Varga Date: Wed, 17 May 2017 16:19:04 +0200 Subject: [PATCH] BUG-8402: Separate out OutOfOrderRequestException OutOfOrderRequestException is used for two distinct cases, which is a mixup during refactor. The first case is when an envelope's sequence does not match the sequence we are expecting on a connection. This is a retriable exception and happens due to mailbox queueing during leadership changes: - a FE sees us as a leader, sends requests - we become a follower, we reject a few requests - we become a leader, at which point we must not process requests until the FE reconnects, as we would not be processing them in the correct order. The second case is when we receive a Request with an unexpected sequence. This is a hard error, as it indicates that the client has made a mistake and lost a request (like the case fixed in fe69101801085580f2fe72762abea5c5fa83d978). Separate these two cases out by introducing OutOfSequenceEnvelopeException and handle it by initiating a session reconnect. Change-Id: Ifb0bac41ff2efd6385455fd9c77b8b39054dd4a0 Signed-off-by: Robert Varga --- .../commands/OutOfOrderRequestException.java | 6 ++-- .../OutOfSequenceEnvelopeException.java | 32 +++++++++++++++++++ .../OutOfOrderRequestExceptionTest.java | 3 +- .../access/client/ClientActorBehavior.java | 12 +++++++ .../datastore/AbstractFrontendHistory.java | 3 +- .../datastore/LeaderFrontendState.java | 6 ++-- 6 files changed, 53 insertions(+), 9 deletions(-) create mode 100644 opendaylight/md-sal/cds-access-api/src/main/java/org/opendaylight/controller/cluster/access/commands/OutOfSequenceEnvelopeException.java diff --git a/opendaylight/md-sal/cds-access-api/src/main/java/org/opendaylight/controller/cluster/access/commands/OutOfOrderRequestException.java b/opendaylight/md-sal/cds-access-api/src/main/java/org/opendaylight/controller/cluster/access/commands/OutOfOrderRequestException.java index 07a3dba620..cd110d66b6 100644 --- a/opendaylight/md-sal/cds-access-api/src/main/java/org/opendaylight/controller/cluster/access/commands/OutOfOrderRequestException.java +++ b/opendaylight/md-sal/cds-access-api/src/main/java/org/opendaylight/controller/cluster/access/commands/OutOfOrderRequestException.java @@ -11,8 +11,8 @@ import com.google.common.annotations.Beta; import org.opendaylight.controller.cluster.access.concepts.RequestException; /** - * A {@link RequestException} indicating that the backend has received an unexpected request. This would typically - * mean that messages are not arriving in the sequence they were generated by the frontend. + * A {@link RequestException} indicating that the backend has received a Request whose sequence does not match the + * next expected sequence for the target. This is a hard error, as it indicates a Request is missing in the stream. * * @author Robert Varga */ @@ -26,6 +26,6 @@ public final class OutOfOrderRequestException extends RequestException { @Override public boolean isRetriable() { - return true; + return false; } } diff --git a/opendaylight/md-sal/cds-access-api/src/main/java/org/opendaylight/controller/cluster/access/commands/OutOfSequenceEnvelopeException.java b/opendaylight/md-sal/cds-access-api/src/main/java/org/opendaylight/controller/cluster/access/commands/OutOfSequenceEnvelopeException.java new file mode 100644 index 0000000000..ad3dd8d700 --- /dev/null +++ b/opendaylight/md-sal/cds-access-api/src/main/java/org/opendaylight/controller/cluster/access/commands/OutOfSequenceEnvelopeException.java @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2017 Pantheon Technologies, s.r.o. and others. All rights reserved. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v1.0 which accompanies this distribution, + * and is available at http://www.eclipse.org/legal/epl-v10.html + */ +package org.opendaylight.controller.cluster.access.commands; + +import com.google.common.annotations.Beta; +import org.opendaylight.controller.cluster.access.concepts.RequestException; + +/** + * A {@link RequestException} indicating that the backend has received a RequestEnvelope whose sequence does not match + * the next expected sequence. This can happen during leader transitions, when a part of the stream is rejected because + * the backend is not the leader and it transitions to being a leader with old stream messages still being present. + * + * @author Robert Varga + */ +@Beta +public final class OutOfSequenceEnvelopeException extends RequestException { + private static final long serialVersionUID = 1L; + + public OutOfSequenceEnvelopeException(final long expectedEnvelope) { + super("Expecting envelope " + Long.toUnsignedString(expectedEnvelope)); + } + + @Override + public boolean isRetriable() { + return true; + } +} diff --git a/opendaylight/md-sal/cds-access-api/src/test/java/org/opendaylight/controller/cluster/access/commands/OutOfOrderRequestExceptionTest.java b/opendaylight/md-sal/cds-access-api/src/test/java/org/opendaylight/controller/cluster/access/commands/OutOfOrderRequestExceptionTest.java index 4f045adbd8..135f31c302 100644 --- a/opendaylight/md-sal/cds-access-api/src/test/java/org/opendaylight/controller/cluster/access/commands/OutOfOrderRequestExceptionTest.java +++ b/opendaylight/md-sal/cds-access-api/src/test/java/org/opendaylight/controller/cluster/access/commands/OutOfOrderRequestExceptionTest.java @@ -7,6 +7,7 @@ */ package org.opendaylight.controller.cluster.access.commands; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -19,7 +20,7 @@ public class OutOfOrderRequestExceptionTest extends RequestExceptionTest extends return conn.reconnect(this); } } + if (cause instanceof OutOfSequenceEnvelopeException) { + final AbstractClientConnection conn = getConnection(command); + if (conn instanceof ReconnectingClientConnection) { + // Already reconnecting, do not churn the logs + return this; + } else if (conn != null) { + LOG.info("{}: connection {} indicated no sequencing mismatch on {} sequence {}, reconnecting it", + persistenceId(), conn, failure.getTarget(), failure.getSequence(), cause); + return conn.reconnect(this); + } + } return onRequestFailure(command); } diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/AbstractFrontendHistory.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/AbstractFrontendHistory.java index 8b437863e8..f2612949e0 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/AbstractFrontendHistory.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/AbstractFrontendHistory.java @@ -44,7 +44,6 @@ import org.slf4j.LoggerFactory; */ abstract class AbstractFrontendHistory implements Identifiable { private static final Logger LOG = LoggerFactory.getLogger(AbstractFrontendHistory.class); - private static final OutOfOrderRequestException UNSEQUENCED_START = new OutOfOrderRequestException(0); private final Map transactions = new HashMap<>(); private final RangeSet purgedTransactions; @@ -137,7 +136,7 @@ abstract class AbstractFrontendHistory implements Identifiable { return clientId; } - private void checkRequestSequence(final RequestEnvelope envelope) throws OutOfOrderRequestException { + private void checkRequestSequence(final RequestEnvelope envelope) throws OutOfSequenceEnvelopeException { if (expectedTxSequence != envelope.getTxSequence()) { - throw new OutOfOrderRequestException(expectedTxSequence); + throw new OutOfSequenceEnvelopeException(expectedTxSequence); } } -- 2.36.6