X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?p=controller.git;a=blobdiff_plain;f=opendaylight%2Fmd-sal%2Fsal-distributed-datastore%2Fsrc%2Fmain%2Fjava%2Forg%2Fopendaylight%2Fcontroller%2Fcluster%2Fdatabroker%2Factors%2Fdds%2FAbstractProxyTransaction.java;h=84546632f099bb3302938f6d0e8990c3f93e5e3b;hp=8ff8b8eff9cb5850d1c7a8bf5c39e408627355a4;hb=da42d2ffc8904b8dd24596cf6d918a0d30c8c521;hpb=1d34f75864ac09d31ef0f7b4ef59f7434167ae15 diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/databroker/actors/dds/AbstractProxyTransaction.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/databroker/actors/dds/AbstractProxyTransaction.java index 8ff8b8eff9..84546632f0 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/databroker/actors/dds/AbstractProxyTransaction.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/databroker/actors/dds/AbstractProxyTransaction.java @@ -8,13 +8,30 @@ package org.opendaylight.controller.cluster.databroker.actors.dds; import akka.actor.ActorRef; +import com.google.common.base.MoreObjects; import com.google.common.base.Optional; import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; import com.google.common.base.Verify; +import com.google.common.collect.Iterables; import com.google.common.util.concurrent.CheckedFuture; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Iterator; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import java.util.function.Consumer; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import javax.annotation.concurrent.GuardedBy; +import javax.annotation.concurrent.NotThreadSafe; +import org.opendaylight.controller.cluster.access.client.ConnectionEntry; +import org.opendaylight.controller.cluster.access.commands.AbstractLocalTransactionRequest; +import org.opendaylight.controller.cluster.access.commands.ClosedTransactionException; +import org.opendaylight.controller.cluster.access.commands.IncrementTransactionSequenceRequest; import org.opendaylight.controller.cluster.access.commands.TransactionAbortRequest; import org.opendaylight.controller.cluster.access.commands.TransactionAbortSuccess; import org.opendaylight.controller.cluster.access.commands.TransactionCanCommitSuccess; @@ -22,84 +39,357 @@ import org.opendaylight.controller.cluster.access.commands.TransactionCommitSucc import org.opendaylight.controller.cluster.access.commands.TransactionDoCommitRequest; import org.opendaylight.controller.cluster.access.commands.TransactionPreCommitRequest; import org.opendaylight.controller.cluster.access.commands.TransactionPreCommitSuccess; +import org.opendaylight.controller.cluster.access.commands.TransactionPurgeRequest; import org.opendaylight.controller.cluster.access.commands.TransactionRequest; +import org.opendaylight.controller.cluster.access.concepts.Request; import org.opendaylight.controller.cluster.access.concepts.RequestFailure; import org.opendaylight.controller.cluster.access.concepts.Response; import org.opendaylight.controller.cluster.access.concepts.TransactionIdentifier; -import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException; +import org.opendaylight.mdsal.common.api.ReadFailedException; import org.opendaylight.yangtools.concepts.Identifiable; import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier; import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Class translating transaction operations towards a particular backend shard. * + *

* This class is not safe to access from multiple application threads, as is usual for transactions. Internal state * transitions coming from interactions with backend are expected to be thread-safe. * + *

* This class interacts with the queueing mechanism in ClientActorBehavior, hence once we arrive at a decision * to use either a local or remote implementation, we are stuck with it. We can re-evaluate on the next transaction. * * @author Robert Varga */ abstract class AbstractProxyTransaction implements Identifiable { - private final DistributedDataStoreClientBehavior client; + /** + * Marker object used instead of read-type of requests, which are satisfied only once. This has a lower footprint + * and allows compressing multiple requests into a single entry. + */ + @NotThreadSafe + private static final class IncrementSequence { + private final long sequence; + private long delta = 0; + + IncrementSequence(final long sequence) { + this.sequence = sequence; + } + + long getDelta() { + return delta; + } + + long getSequence() { + return sequence; + } + + void incrementDelta() { + delta++; + } + } + + /** + * Base class for representing logical state of this proxy. See individual instantiations and {@link SuccessorState} + * for details. + */ + private static class State { + private final String string; + + State(final String string) { + this.string = Preconditions.checkNotNull(string); + } + + @Override + public final String toString() { + return string; + } + } + + /** + * State class used when a successor has interfered. Contains coordinator latch, the successor and previous state. + * This is a temporary state introduced during reconnection process and is necessary for correct state hand-off + * between the old connection (potentially being accessed by the user) and the new connection (being cleaned up + * by the actor. + * + *

+ * When a user operation encounters this state, it synchronizes on the it and wait until reconnection completes, + * at which point the request is routed to the successor transaction. This is a relatively heavy-weight solution + * to the problem of state transfer, but the user will observe it only if the race condition is hit. + */ + private static class SuccessorState extends State { + private final CountDownLatch latch = new CountDownLatch(1); + private AbstractProxyTransaction successor; + private State prevState; + + // SUCCESSOR + DONE + private boolean done; + + SuccessorState() { + super("SUCCESSOR"); + } + + // Synchronize with succession process and return the successor + AbstractProxyTransaction await() { + try { + latch.await(); + } catch (InterruptedException e) { + LOG.warn("Interrupted while waiting for latch of {}", successor); + throw Throwables.propagate(e); + } + return successor; + } + + void finish() { + latch.countDown(); + } + + State getPrevState() { + return Verify.verifyNotNull(prevState, "Attempted to access previous state, which was not set"); + } + + void setPrevState(final State prevState) { + Verify.verify(this.prevState == null, "Attempted to set previous state to %s when we already have %s", + prevState, this.prevState); + this.prevState = Preconditions.checkNotNull(prevState); + // We cannot have duplicate successor states, so this check is sufficient + this.done = DONE.equals(prevState); + } + + // To be called from safe contexts, where successor is known to be completed + AbstractProxyTransaction getSuccessor() { + return Verify.verifyNotNull(successor); + } + + void setSuccessor(final AbstractProxyTransaction successor) { + Verify.verify(this.successor == null, "Attempted to set successor to %s when we already have %s", + successor, this.successor); + this.successor = Preconditions.checkNotNull(successor); + } + + boolean isDone() { + return done; + } + + void setDone() { + done = true; + } + } + + private static final Logger LOG = LoggerFactory.getLogger(AbstractProxyTransaction.class); + private static final AtomicIntegerFieldUpdater SEALED_UPDATER = + AtomicIntegerFieldUpdater.newUpdater(AbstractProxyTransaction.class, "sealed"); + private static final AtomicReferenceFieldUpdater STATE_UPDATER = + AtomicReferenceFieldUpdater.newUpdater(AbstractProxyTransaction.class, State.class, "state"); + /** + * Transaction has been open and is being actively worked on. + */ + private static final State OPEN = new State("OPEN"); + + /** + * Transaction has been sealed by the user, but it has not completed flushing to the backed, yet. This is + * a transition state, as we are waiting for the user to initiate commit procedures. + * + *

+ * Since the reconnect mechanics relies on state replay for transactions, this state needs to be flushed into the + * queue to re-create state in successor transaction (which may be based on different messages as locality may have + * changed). Hence the transition to {@link #FLUSHED} state needs to be handled in a thread-safe manner. + */ + private static final State SEALED = new State("SEALED"); + + /** + * Transaction state has been flushed into the queue, i.e. it is visible by the successor and potentially + * the backend. At this point the transaction does not hold any state besides successful requests, all other state + * is held either in the connection's queue or the successor object. + * + *

+ * Transition to this state indicates we have all input from the user we need to initiate the correct commit + * protocol. + */ + private static final State FLUSHED = new State("FLUSHED"); + + /** + * Transaction state has been completely resolved, we have received confirmation of the transaction fate from + * the backend. The only remaining task left to do is finishing up the state cleanup, which is done via purge + * request. We need to hang on to the transaction until that is done, as we have to make sure backend completes + * purging its state -- otherwise we could have a leak on the backend. + */ + private static final State DONE = new State("DONE"); + + // Touched from client actor thread only + private final Deque successfulRequests = new ArrayDeque<>(); + private final ProxyHistory parent; + + // Accessed from user thread only, which may not access this object concurrently private long sequence; - private boolean sealed; - AbstractProxyTransaction(final DistributedDataStoreClientBehavior client) { - this.client = Preconditions.checkNotNull(client); + /* + * Atomic state-keeping is required to synchronize the process of propagating completed transaction state towards + * the backend -- which may include a successor. + * + * Successor, unlike {@link AbstractProxyTransaction#seal()} is triggered from the client actor thread, which means + * the successor placement needs to be atomic with regard to the application thread. + * + * In the common case, the application thread performs performs the seal operations and then "immediately" sends + * the corresponding message. The uncommon case is when the seal and send operations race with a connect completion + * or timeout, when a successor is injected. + * + * This leaves the problem of needing to completely transferring state just after all queued messages are replayed + * after a successor was injected, so that it can be properly sealed if we are racing. Further complication comes + * from lock ordering, where the successor injection works with a locked queue and locks proxy objects -- leading + * to a potential AB-BA deadlock in case of a naive implementation. + * + * For tracking user-visible state we use a single volatile int, which is flipped atomically from 0 to 1 exactly + * once in {@link AbstractProxyTransaction#seal()}. That keeps common operations fast, as they need to perform + * only a single volatile read to assert state correctness. + * + * For synchronizing client actor (successor-injecting) and user (commit-driving) thread, we keep a separate state + * variable. It uses pre-allocated objects for fast paths (i.e. no successor present) and a per-transition object + * for slow paths (when successor is injected/present). + */ + private volatile int sealed; + private volatile State state; + + AbstractProxyTransaction(final ProxyHistory parent, final boolean isDone) { + this.parent = Preconditions.checkNotNull(parent); + if (isDone) { + state = DONE; + // DONE implies previous seal operation completed + sealed = 1; + } else { + state = OPEN; + } + } + + final void executeInActor(final Runnable command) { + parent.context().executeInActor(behavior -> { + command.run(); + return behavior; + }); } final ActorRef localActor() { - return client.self(); + return parent.localActor(); + } + + final void incrementSequence(final long delta) { + sequence += delta; + LOG.debug("Transaction {} incremented sequence to {}", this, sequence); } final long nextSequence() { - return sequence++; + final long ret = sequence++; + LOG.debug("Transaction {} allocated sequence {}", this, ret); + return ret; } final void delete(final YangInstanceIdentifier path) { - checkSealed(); + checkReadWrite(); + checkNotSealed(); doDelete(path); } final void merge(final YangInstanceIdentifier path, final NormalizedNode data) { - checkSealed(); + checkReadWrite(); + checkNotSealed(); doMerge(path, data); } final void write(final YangInstanceIdentifier path, final NormalizedNode data) { - checkSealed(); + checkReadWrite(); + checkNotSealed(); doWrite(path, data); } final CheckedFuture exists(final YangInstanceIdentifier path) { - checkSealed(); + checkNotSealed(); return doExists(path); } final CheckedFuture>, ReadFailedException> read(final YangInstanceIdentifier path) { - checkSealed(); + checkNotSealed(); return doRead(path); } - final void sendRequest(final TransactionRequest request, final Consumer> completer) { - client.sendRequest(request, completer); + final void enqueueRequest(final TransactionRequest request, final Consumer> callback, + final long enqueuedTicks) { + LOG.debug("Transaction proxy {} enqueing request {} callback {}", this, request, callback); + parent.enqueueRequest(request, callback, enqueuedTicks); + } + + final void sendRequest(final TransactionRequest request, final Consumer> callback) { + LOG.debug("Transaction proxy {} sending request {} callback {}", this, request, callback); + parent.sendRequest(request, callback); } /** - * Seal this transaction before it is either + * Seal this transaction before it is either committed or aborted. */ final void seal() { - checkSealed(); + // Transition user-visible state first + final boolean success = SEALED_UPDATER.compareAndSet(this, 0, 1); + Preconditions.checkState(success, "Proxy %s was already sealed", getIdentifier()); + internalSeal(); + } + + final void ensureSealed() { + if (SEALED_UPDATER.compareAndSet(this, 0, 1)) { + internalSeal(); + } + } + + private void internalSeal() { doSeal(); - sealed = true; + parent.onTransactionSealed(this); + + // Now deal with state transfer, which can occur via successor or a follow-up canCommit() or directCommit(). + if (!STATE_UPDATER.compareAndSet(this, OPEN, SEALED)) { + // Slow path: wait for the successor to complete + final AbstractProxyTransaction successor = awaitSuccessor(); + + // At this point the successor has completed transition and is possibly visible by the user thread, which is + // still stuck here. The successor has not seen final part of our state, nor the fact it is sealed. + // Propagate state and seal the successor. + flushState(successor); + successor.ensureSealed(); + } + } + + private void checkNotSealed() { + Preconditions.checkState(sealed == 0, "Transaction %s has already been sealed", getIdentifier()); } private void checkSealed() { - Preconditions.checkState(sealed, "Transaction %s has not been sealed yet", getIdentifier()); + Preconditions.checkState(sealed != 0, "Transaction %s has not been sealed yet", getIdentifier()); + } + + private SuccessorState getSuccessorState() { + final State local = state; + Verify.verify(local instanceof SuccessorState, "State %s has unexpected class", local); + return (SuccessorState) local; + } + + private void checkReadWrite() { + if (isSnapshotOnly()) { + throw new UnsupportedOperationException("Transaction " + getIdentifier() + " is a read-only snapshot"); + } + } + + final void recordSuccessfulRequest(final @Nonnull TransactionRequest req) { + successfulRequests.add(Verify.verifyNotNull(req)); + } + + final void recordFinishedRequest(final Response response) { + final Object last = successfulRequests.peekLast(); + if (last instanceof IncrementSequence) { + ((IncrementSequence) last).incrementDelta(); + } else { + successfulRequests.addLast(new IncrementSequence(response.getSequence())); + } } /** @@ -107,8 +397,53 @@ abstract class AbstractProxyTransaction implements Identifiable { + LOG.debug("Transaction {} abort completed with {}", getIdentifier(), resp); + enqueuePurge(); + }); + } + + final void abort(final VotingFuture ret) { checkSealed(); - doAbort(); + + sendDoAbort(t -> { + if (t instanceof TransactionAbortSuccess) { + ret.voteYes(); + } else if (t instanceof RequestFailure) { + ret.voteNo(((RequestFailure) t).getCause().unwrap()); + } else { + ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass())); + } + + // This is a terminal request, hence we do not need to record it + LOG.debug("Transaction {} abort completed", this); + enqueuePurge(); + }); + } + + final void enqueueAbort(final Consumer> callback, final long enqueuedTicks) { + checkNotSealed(); + parent.abortTransaction(this); + + enqueueRequest(abortRequest(), resp -> { + LOG.debug("Transaction {} abort completed with {}", getIdentifier(), resp); + // Purge will be sent by the predecessor's callback + if (callback != null) { + callback.accept(resp); + } + }, enqueuedTicks); + } + + final void enqueueDoAbort(final Consumer> callback, final long enqueuedTicks) { + enqueueRequest(new TransactionAbortRequest(getIdentifier(), nextSequence(), localActor()), callback, + enqueuedTicks); + } + + final void sendDoAbort(final Consumer> callback) { + sendRequest(new TransactionAbortRequest(getIdentifier(), nextSequence(), localActor()), callback); } /** @@ -118,90 +453,363 @@ abstract class AbstractProxyTransaction implements Identifiable directCommit() { + checkReadWrite(); checkSealed(); - final SettableFuture ret = SettableFuture.create(); - sendRequest(Verify.verifyNotNull(doCommit(false)), t -> { - if (t instanceof TransactionCommitSuccess) { - ret.set(Boolean.TRUE); - } else if (t instanceof RequestFailure) { - ret.setException(((RequestFailure) t).getCause()); - } else { - ret.setException(new IllegalStateException("Unhandled response " + t.getClass())); + // Precludes startReconnect() from interfering with the fast path + synchronized (this) { + if (STATE_UPDATER.compareAndSet(this, SEALED, FLUSHED)) { + final SettableFuture ret = SettableFuture.create(); + sendRequest(Verify.verifyNotNull(commitRequest(false)), t -> { + if (t instanceof TransactionCommitSuccess) { + ret.set(Boolean.TRUE); + } else if (t instanceof RequestFailure) { + final Throwable cause = ((RequestFailure) t).getCause().unwrap(); + if (cause instanceof ClosedTransactionException) { + // This is okay, as it indicates the transaction has been completed. It can happen + // when we lose connectivity with the backend after it has received the request. + ret.set(Boolean.TRUE); + } else { + ret.setException(cause); + } + } else { + ret.setException(new IllegalStateException("Unhandled response " + t.getClass())); + } + + // This is a terminal request, hence we do not need to record it + LOG.debug("Transaction {} directCommit completed", this); + enqueuePurge(); + }); + + return ret; } - }); - return ret; + } + + // We have had some interference with successor injection, wait for it to complete and defer to the successor. + return awaitSuccessor().directCommit(); } - void abort(final VotingFuture ret) { + final void canCommit(final VotingFuture ret) { + checkReadWrite(); checkSealed(); - sendRequest(new TransactionAbortRequest(getIdentifier(), nextSequence(), localActor()), t -> { - if (t instanceof TransactionAbortSuccess) { - ret.voteYes(); - } else if (t instanceof RequestFailure) { - ret.voteNo(((RequestFailure) t).getCause()); - } else { - ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass())); + // Precludes startReconnect() from interfering with the fast path + synchronized (this) { + if (STATE_UPDATER.compareAndSet(this, SEALED, FLUSHED)) { + final TransactionRequest req = Verify.verifyNotNull(commitRequest(true)); + + sendRequest(req, t -> { + if (t instanceof TransactionCanCommitSuccess) { + ret.voteYes(); + } else if (t instanceof RequestFailure) { + ret.voteNo(((RequestFailure) t).getCause().unwrap()); + } else { + ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass())); + } + + recordSuccessfulRequest(req); + LOG.debug("Transaction {} canCommit completed", this); + }); + + return; } - }); - } + } - void canCommit(final VotingFuture ret) { - checkSealed(); + // We have had some interference with successor injection, wait for it to complete and defer to the successor. + awaitSuccessor().canCommit(ret); + } - sendRequest(Verify.verifyNotNull(doCommit(true)), t -> { - if (t instanceof TransactionCanCommitSuccess) { - ret.voteYes(); - } else if (t instanceof RequestFailure) { - ret.voteNo(((RequestFailure) t).getCause()); - } else { - ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass())); - } - }); + private AbstractProxyTransaction awaitSuccessor() { + return getSuccessorState().await(); } - void preCommit(final VotingFuture ret) { + final void preCommit(final VotingFuture ret) { + checkReadWrite(); checkSealed(); - sendRequest(new TransactionPreCommitRequest(getIdentifier(), nextSequence(), localActor()), t-> { + final TransactionRequest req = new TransactionPreCommitRequest(getIdentifier(), nextSequence(), + localActor()); + sendRequest(req, t -> { if (t instanceof TransactionPreCommitSuccess) { ret.voteYes(); } else if (t instanceof RequestFailure) { - ret.voteNo(((RequestFailure) t).getCause()); + ret.voteNo(((RequestFailure) t).getCause().unwrap()); } else { ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass())); } + + onPreCommitComplete(req); }); } - void doCommit(final VotingFuture ret) { + private void onPreCommitComplete(final TransactionRequest req) { + /* + * The backend has agreed that the transaction has entered PRE_COMMIT phase, meaning it will be committed + * to storage after the timeout completes. + * + * All state has been replicated to the backend, hence we do not need to keep it around. Retain only + * the precommit request, so we know which request to use for resync. + */ + LOG.debug("Transaction {} preCommit completed, clearing successfulRequests", this); + successfulRequests.clear(); + + // TODO: this works, but can contain some useless state (like batched operations). Create an empty + // equivalent of this request and store that. + recordSuccessfulRequest(req); + } + + final void doCommit(final VotingFuture ret) { + checkReadWrite(); checkSealed(); - sendRequest(new TransactionDoCommitRequest(getIdentifier(), nextSequence(), localActor()), t-> { + sendRequest(new TransactionDoCommitRequest(getIdentifier(), nextSequence(), localActor()), t -> { if (t instanceof TransactionCommitSuccess) { ret.voteYes(); } else if (t instanceof RequestFailure) { - ret.voteNo(((RequestFailure) t).getCause()); + ret.voteNo(((RequestFailure) t).getCause().unwrap()); } else { ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass())); } + + LOG.debug("Transaction {} doCommit completed", this); + + // Needed for ProxyHistory$Local data tree rebase points. + parent.completeTransaction(this); + + enqueuePurge(); }); } - abstract void doDelete(final YangInstanceIdentifier path); + private void enqueuePurge() { + enqueuePurge(null); + } + + final void enqueuePurge(final Consumer> callback) { + // Purge request are dispatched internally, hence should not wait + enqueuePurge(callback, parent.currentTime()); + } + + final void enqueuePurge(final Consumer> callback, final long enqueuedTicks) { + LOG.debug("{}: initiating purge", this); + + final State prev = state; + if (prev instanceof SuccessorState) { + ((SuccessorState) prev).setDone(); + } else { + final boolean success = STATE_UPDATER.compareAndSet(this, prev, DONE); + if (!success) { + LOG.warn("{}: moved from state {} while we were purging it", this, prev); + } + } + + successfulRequests.clear(); + + enqueueRequest(new TransactionPurgeRequest(getIdentifier(), nextSequence(), localActor()), resp -> { + LOG.debug("{}: purge completed", this); + parent.purgeTransaction(this); + + if (callback != null) { + callback.accept(resp); + } + }, enqueuedTicks); + } + + // Called with the connection unlocked + final synchronized void startReconnect() { + // At this point canCommit/directCommit are blocked, we assert a new successor state, retrieving the previous + // state. This method is called with the queue still unlocked. + final SuccessorState nextState = new SuccessorState(); + final State prevState = STATE_UPDATER.getAndSet(this, nextState); + + LOG.debug("Start reconnect of proxy {} previous state {}", this, prevState); + Verify.verify(!(prevState instanceof SuccessorState), "Proxy %s duplicate reconnect attempt after %s", this, + prevState); + + // We have asserted a slow-path state, seal(), canCommit(), directCommit() are forced to slow paths, which will + // wait until we unblock nextState's latch before accessing state. Now we record prevState for later use and we + // are done. + nextState.setPrevState(prevState); + } + + // Called with the connection locked + final void replayMessages(final ProxyHistory successorHistory, final Iterable enqueuedEntries) { + final SuccessorState local = getSuccessorState(); + final State prevState = local.getPrevState(); + + final AbstractProxyTransaction successor = successorHistory.createTransactionProxy(getIdentifier(), + isSnapshotOnly(), local.isDone()); + LOG.debug("{} created successor {}", this, successor); + local.setSuccessor(successor); + + // Replay successful requests first + if (!successfulRequests.isEmpty()) { + // We need to find a good timestamp to use for successful requests, as we do not want to time them out + // nor create timing inconsistencies in the queue -- requests are expected to be ordered by their enqueue + // time. We will pick the time of the first entry available. If there is none, we will just use current + // time, as all other requests will get enqueued afterwards. + final ConnectionEntry firstInQueue = Iterables.getFirst(enqueuedEntries, null); + final long now = firstInQueue != null ? firstInQueue.getEnqueuedTicks() : parent.currentTime(); + + for (Object obj : successfulRequests) { + if (obj instanceof TransactionRequest) { + LOG.debug("Forwarding successful request {} to successor {}", obj, successor); + successor.doReplayRequest((TransactionRequest) obj, resp -> { }, now); + } else { + Verify.verify(obj instanceof IncrementSequence); + final IncrementSequence increment = (IncrementSequence) obj; + successor.doReplayRequest(new IncrementTransactionSequenceRequest(getIdentifier(), + increment.getSequence(), localActor(), isSnapshotOnly(), increment.getDelta()), resp -> { }, + now); + LOG.debug("Incrementing sequence {} to successor {}", obj, successor); + } + } + LOG.debug("{} replayed {} successful requests", getIdentifier(), successfulRequests.size()); + successfulRequests.clear(); + } + + // Now replay whatever is in the connection + final Iterator it = enqueuedEntries.iterator(); + while (it.hasNext()) { + final ConnectionEntry e = it.next(); + final Request req = e.getRequest(); + + if (getIdentifier().equals(req.getTarget())) { + Verify.verify(req instanceof TransactionRequest, "Unhandled request %s", req); + LOG.debug("Replaying queued request {} to successor {}", req, successor); + successor.doReplayRequest((TransactionRequest) req, e.getCallback(), e.getEnqueuedTicks()); + it.remove(); + } + } + + /* + * Check the state at which we have started the reconnect attempt. State transitions triggered while we were + * reconnecting have been forced to slow paths, which will be unlocked once we unblock the state latch + * at the end of this method. + */ + if (SEALED.equals(prevState)) { + LOG.debug("Proxy {} reconnected while being sealed, propagating state to successor {}", this, successor); + flushState(successor); + successor.ensureSealed(); + } + } + + /** + * Invoked from {@link #replayMessages(AbstractProxyTransaction, Iterable)} to have successor adopt an in-flight + * request. + * + *

+ * Note: this method is invoked by the predecessor on the successor. + * + * @param request Request which needs to be forwarded + * @param callback Callback to be invoked once the request completes + * @param enqueuedTicks ticker-based time stamp when the request was enqueued + */ + private void doReplayRequest(final TransactionRequest request, final Consumer> callback, + final long enqueuedTicks) { + if (request instanceof AbstractLocalTransactionRequest) { + handleReplayedLocalRequest((AbstractLocalTransactionRequest) request, callback, enqueuedTicks); + } else { + handleReplayedRemoteRequest(request, callback, enqueuedTicks); + } + } + + // Called with the connection locked + final void finishReconnect() { + final SuccessorState local = getSuccessorState(); + LOG.debug("Finishing reconnect of proxy {}", this); + + // All done, release the latch, unblocking seal() and canCommit() slow paths + local.finish(); + } + + /** + * Invoked from a retired connection for requests which have been in-flight and need to be re-adjusted + * and forwarded to the successor connection. + * + * @param request Request to be forwarded + * @param callback Original callback + */ + final void forwardRequest(final TransactionRequest request, final Consumer> callback) { + forwardToSuccessor(getSuccessorState().getSuccessor(), request, callback); + } + + final void forwardToSuccessor(final AbstractProxyTransaction successor, final TransactionRequest request, + final Consumer> callback) { + if (successor instanceof LocalProxyTransaction) { + forwardToLocal((LocalProxyTransaction)successor, request, callback); + } else if (successor instanceof RemoteProxyTransaction) { + forwardToRemote((RemoteProxyTransaction)successor, request, callback); + } else { + throw new IllegalStateException("Unhandled successor " + successor); + } + } + + final void replayRequest(final TransactionRequest request, final Consumer> callback, + final long enqueuedTicks) { + getSuccessorState().getSuccessor().doReplayRequest(request, callback, enqueuedTicks); + } - abstract void doMerge(final YangInstanceIdentifier path, final NormalizedNode data); + abstract boolean isSnapshotOnly(); - abstract void doWrite(final YangInstanceIdentifier path, final NormalizedNode data); + abstract void doDelete(YangInstanceIdentifier path); - abstract CheckedFuture doExists(final YangInstanceIdentifier path); + abstract void doMerge(YangInstanceIdentifier path, NormalizedNode data); - abstract CheckedFuture>, ReadFailedException> doRead(final YangInstanceIdentifier path); + abstract void doWrite(YangInstanceIdentifier path, NormalizedNode data); + + abstract CheckedFuture doExists(YangInstanceIdentifier path); + + abstract CheckedFuture>, ReadFailedException> doRead(YangInstanceIdentifier path); abstract void doSeal(); - abstract void doAbort(); + @GuardedBy("this") + abstract void flushState(AbstractProxyTransaction successor); + + abstract TransactionRequest abortRequest(); - abstract TransactionRequest doCommit(boolean coordinated); + abstract TransactionRequest commitRequest(boolean coordinated); + + /** + * Replay a request originating in this proxy to a successor remote proxy. + */ + abstract void forwardToRemote(RemoteProxyTransaction successor, TransactionRequest request, + Consumer> callback); + + /** + * Replay a request originating in this proxy to a successor local proxy. + */ + abstract void forwardToLocal(LocalProxyTransaction successor, TransactionRequest request, + Consumer> callback); + + /** + * Invoked from {@link LocalProxyTransaction} when it replays its successful requests to its successor. + * + *

+ * Note: this method is invoked by the predecessor on the successor. + * + * @param request Request which needs to be forwarded + * @param callback Callback to be invoked once the request completes + * @param enqueuedTicks Time stamp to use for enqueue time + */ + abstract void handleReplayedLocalRequest(AbstractLocalTransactionRequest request, + @Nullable Consumer> callback, long enqueuedTicks); + + /** + * Invoked from {@link RemoteProxyTransaction} when it replays its successful requests to its successor. + * + *

+ * Note: this method is invoked by the predecessor on the successor. + * + * @param request Request which needs to be forwarded + * @param callback Callback to be invoked once the request completes + * @param enqueuedTicks Time stamp to use for enqueue time + */ + abstract void handleReplayedRemoteRequest(TransactionRequest request, + @Nullable Consumer> callback, long enqueuedTicks); + + @Override + public final String toString() { + return MoreObjects.toStringHelper(this).add("identifier", getIdentifier()).add("state", state).toString(); + } }