2 * Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.controller.cluster.databroker.actors.dds;
10 import akka.actor.ActorRef;
11 import com.google.common.base.MoreObjects;
12 import com.google.common.base.Optional;
13 import com.google.common.base.Preconditions;
14 import com.google.common.base.Throwables;
15 import com.google.common.base.Verify;
16 import com.google.common.collect.Iterables;
17 import com.google.common.util.concurrent.CheckedFuture;
18 import com.google.common.util.concurrent.ListenableFuture;
19 import com.google.common.util.concurrent.SettableFuture;
20 import java.util.ArrayDeque;
21 import java.util.Deque;
22 import java.util.Iterator;
23 import java.util.concurrent.CountDownLatch;
24 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
25 import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
26 import java.util.function.Consumer;
27 import javax.annotation.Nonnull;
28 import javax.annotation.Nullable;
29 import javax.annotation.concurrent.GuardedBy;
30 import javax.annotation.concurrent.NotThreadSafe;
31 import org.opendaylight.controller.cluster.access.client.ConnectionEntry;
32 import org.opendaylight.controller.cluster.access.commands.AbstractLocalTransactionRequest;
33 import org.opendaylight.controller.cluster.access.commands.IncrementTransactionSequenceRequest;
34 import org.opendaylight.controller.cluster.access.commands.TransactionAbortRequest;
35 import org.opendaylight.controller.cluster.access.commands.TransactionAbortSuccess;
36 import org.opendaylight.controller.cluster.access.commands.TransactionCanCommitSuccess;
37 import org.opendaylight.controller.cluster.access.commands.TransactionCommitSuccess;
38 import org.opendaylight.controller.cluster.access.commands.TransactionDoCommitRequest;
39 import org.opendaylight.controller.cluster.access.commands.TransactionPreCommitRequest;
40 import org.opendaylight.controller.cluster.access.commands.TransactionPreCommitSuccess;
41 import org.opendaylight.controller.cluster.access.commands.TransactionPurgeRequest;
42 import org.opendaylight.controller.cluster.access.commands.TransactionRequest;
43 import org.opendaylight.controller.cluster.access.concepts.Request;
44 import org.opendaylight.controller.cluster.access.concepts.RequestFailure;
45 import org.opendaylight.controller.cluster.access.concepts.Response;
46 import org.opendaylight.controller.cluster.access.concepts.TransactionIdentifier;
47 import org.opendaylight.mdsal.common.api.ReadFailedException;
48 import org.opendaylight.yangtools.concepts.Identifiable;
49 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
50 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
51 import org.slf4j.Logger;
52 import org.slf4j.LoggerFactory;
55 * Class translating transaction operations towards a particular backend shard.
58 * This class is not safe to access from multiple application threads, as is usual for transactions. Internal state
59 * transitions coming from interactions with backend are expected to be thread-safe.
62 * This class interacts with the queueing mechanism in ClientActorBehavior, hence once we arrive at a decision
63 * to use either a local or remote implementation, we are stuck with it. We can re-evaluate on the next transaction.
65 * @author Robert Varga
67 abstract class AbstractProxyTransaction implements Identifiable<TransactionIdentifier> {
69 * Marker object used instead of read-type of requests, which are satisfied only once. This has a lower footprint
70 * and allows compressing multiple requests into a single entry.
73 private static final class IncrementSequence {
74 private final long sequence;
75 private long delta = 0;
77 IncrementSequence(final long sequence) {
78 this.sequence = sequence;
89 void incrementDelta() {
94 // Generic state base class. Direct instances are used for fast paths, sub-class is used for successor transitions
95 private static class State {
96 private final String string;
98 State(final String string) {
99 this.string = Preconditions.checkNotNull(string);
103 public final String toString() {
108 // State class used when a successor has interfered. Contains coordinator latch, the successor and previous state
109 private static final class SuccessorState extends State {
110 private final CountDownLatch latch = new CountDownLatch(1);
111 private AbstractProxyTransaction successor;
112 private State prevState;
118 // Synchronize with succession process and return the successor
119 AbstractProxyTransaction await() {
122 } catch (InterruptedException e) {
123 LOG.warn("Interrupted while waiting for latch of {}", successor);
124 throw Throwables.propagate(e);
133 State getPrevState() {
137 void setPrevState(final State prevState) {
138 Verify.verify(this.prevState == null);
139 this.prevState = Preconditions.checkNotNull(prevState);
142 // To be called from safe contexts, where successor is known to be completed
143 AbstractProxyTransaction getSuccessor() {
144 return Verify.verifyNotNull(successor);
147 void setSuccessor(final AbstractProxyTransaction successor) {
148 Verify.verify(this.successor == null);
149 this.successor = Preconditions.checkNotNull(successor);
153 private static final Logger LOG = LoggerFactory.getLogger(AbstractProxyTransaction.class);
154 private static final AtomicIntegerFieldUpdater<AbstractProxyTransaction> SEALED_UPDATER =
155 AtomicIntegerFieldUpdater.newUpdater(AbstractProxyTransaction.class, "sealed");
156 private static final AtomicReferenceFieldUpdater<AbstractProxyTransaction, State> STATE_UPDATER =
157 AtomicReferenceFieldUpdater.newUpdater(AbstractProxyTransaction.class, State.class, "state");
158 private static final State OPEN = new State("open");
159 private static final State SEALED = new State("sealed");
160 private static final State FLUSHED = new State("flushed");
162 // Touched from client actor thread only
163 private final Deque<Object> successfulRequests = new ArrayDeque<>();
164 private final ProxyHistory parent;
166 // Accessed from user thread only, which may not access this object concurrently
167 private long sequence;
170 * Atomic state-keeping is required to synchronize the process of propagating completed transaction state towards
171 * the backend -- which may include a successor.
173 * Successor, unlike {@link AbstractProxyTransaction#seal()} is triggered from the client actor thread, which means
174 * the successor placement needs to be atomic with regard to the application thread.
176 * In the common case, the application thread performs performs the seal operations and then "immediately" sends
177 * the corresponding message. The uncommon case is when the seal and send operations race with a connect completion
178 * or timeout, when a successor is injected.
180 * This leaves the problem of needing to completely transferring state just after all queued messages are replayed
181 * after a successor was injected, so that it can be properly sealed if we are racing. Further complication comes
182 * from lock ordering, where the successor injection works with a locked queue and locks proxy objects -- leading
183 * to a potential AB-BA deadlock in case of a naive implementation.
185 * For tracking user-visible state we use a single volatile int, which is flipped atomically from 0 to 1 exactly
186 * once in {@link AbstractProxyTransaction#seal()}. That keeps common operations fast, as they need to perform
187 * only a single volatile read to assert state correctness.
189 * For synchronizing client actor (successor-injecting) and user (commit-driving) thread, we keep a separate state
190 * variable. It uses pre-allocated objects for fast paths (i.e. no successor present) and a per-transition object
191 * for slow paths (when successor is injected/present).
193 private volatile int sealed = 0;
194 private volatile State state = OPEN;
196 AbstractProxyTransaction(final ProxyHistory parent) {
197 this.parent = Preconditions.checkNotNull(parent);
200 final void executeInActor(final Runnable command) {
201 parent.context().executeInActor(behavior -> {
207 final ActorRef localActor() {
208 return parent.localActor();
211 final void incrementSequence(final long delta) {
213 LOG.debug("Transaction {} incremented sequence to {}", this, sequence);
216 final long nextSequence() {
217 final long ret = sequence++;
218 LOG.debug("Transaction {} allocated sequence {}", this, ret);
222 final void delete(final YangInstanceIdentifier path) {
228 final void merge(final YangInstanceIdentifier path, final NormalizedNode<?, ?> data) {
234 final void write(final YangInstanceIdentifier path, final NormalizedNode<?, ?> data) {
240 final CheckedFuture<Boolean, ReadFailedException> exists(final YangInstanceIdentifier path) {
242 return doExists(path);
245 final CheckedFuture<Optional<NormalizedNode<?, ?>>, ReadFailedException> read(final YangInstanceIdentifier path) {
250 final void enqueueRequest(final TransactionRequest<?> request, final Consumer<Response<?, ?>> callback,
251 final long enqueuedTicks) {
252 LOG.debug("Transaction proxy {} enqueing request {} callback {}", this, request, callback);
253 parent.enqueueRequest(request, callback, enqueuedTicks);
256 final void sendRequest(final TransactionRequest<?> request, final Consumer<Response<?, ?>> callback) {
257 LOG.debug("Transaction proxy {} sending request {} callback {}", this, request, callback);
258 parent.sendRequest(request, callback);
262 * Seal this transaction before it is either committed or aborted.
265 // Transition user-visible state first
266 final boolean success = SEALED_UPDATER.compareAndSet(this, 0, 1);
267 Preconditions.checkState(success, "Proxy %s was already sealed", getIdentifier());
271 final void ensureSealed() {
272 if (SEALED_UPDATER.compareAndSet(this, 0, 1)) {
277 private void internalSeal() {
279 parent.onTransactionSealed(this);
281 // Now deal with state transfer, which can occur via successor or a follow-up canCommit() or directCommit().
282 if (!STATE_UPDATER.compareAndSet(this, OPEN, SEALED)) {
283 // Slow path: wait for the successor to complete
284 final AbstractProxyTransaction successor = awaitSuccessor();
286 // At this point the successor has completed transition and is possibly visible by the user thread, which is
287 // still stuck here. The successor has not seen final part of our state, nor the fact it is sealed.
288 // Propagate state and seal the successor.
289 flushState(successor);
290 successor.ensureSealed();
294 private void checkNotSealed() {
295 Preconditions.checkState(sealed == 0, "Transaction %s has already been sealed", getIdentifier());
298 private void checkSealed() {
299 Preconditions.checkState(sealed != 0, "Transaction %s has not been sealed yet", getIdentifier());
302 private SuccessorState getSuccessorState() {
303 final State local = state;
304 Verify.verify(local instanceof SuccessorState, "State %s has unexpected class", local);
305 return (SuccessorState) local;
308 private void checkReadWrite() {
309 if (isSnapshotOnly()) {
310 throw new UnsupportedOperationException("Transaction " + getIdentifier() + " is a read-only snapshot");
314 final void recordSuccessfulRequest(final @Nonnull TransactionRequest<?> req) {
315 successfulRequests.add(Verify.verifyNotNull(req));
318 final void recordFinishedRequest(final Response<?, ?> response) {
319 final Object last = successfulRequests.peekLast();
320 if (last instanceof IncrementSequence) {
321 ((IncrementSequence) last).incrementDelta();
323 successfulRequests.addLast(new IncrementSequence(response.getSequence()));
328 * Abort this transaction. This is invoked only for read-only transactions and will result in an explicit message
329 * being sent to the backend.
333 parent.abortTransaction(this);
335 sendRequest(abortRequest(), resp -> {
336 LOG.debug("Transaction {} abort completed with {}", getIdentifier(), resp);
341 final void abort(final VotingFuture<Void> ret) {
345 if (t instanceof TransactionAbortSuccess) {
347 } else if (t instanceof RequestFailure) {
348 ret.voteNo(((RequestFailure<?, ?>) t).getCause().unwrap());
350 ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass()));
353 // This is a terminal request, hence we do not need to record it
354 LOG.debug("Transaction {} abort completed", this);
359 final void enqueueAbort(final Consumer<Response<?, ?>> callback, final long enqueuedTicks) {
361 parent.abortTransaction(this);
363 enqueueRequest(abortRequest(), resp -> {
364 LOG.debug("Transaction {} abort completed with {}", getIdentifier(), resp);
365 // Purge will be sent by the predecessor's callback
366 if (callback != null) {
367 callback.accept(resp);
372 final void enqueueDoAbort(final Consumer<Response<?, ?>> callback, final long enqueuedTicks) {
373 enqueueRequest(new TransactionAbortRequest(getIdentifier(), nextSequence(), localActor()), callback,
377 final void sendDoAbort(final Consumer<Response<?, ?>> callback) {
378 sendRequest(new TransactionAbortRequest(getIdentifier(), nextSequence(), localActor()), callback);
382 * Commit this transaction, possibly in a coordinated fashion.
384 * @param coordinated True if this transaction should be coordinated across multiple participants.
385 * @return Future completion
387 final ListenableFuture<Boolean> directCommit() {
391 // Precludes startReconnect() from interfering with the fast path
392 synchronized (this) {
393 if (STATE_UPDATER.compareAndSet(this, SEALED, FLUSHED)) {
394 final SettableFuture<Boolean> ret = SettableFuture.create();
395 sendRequest(Verify.verifyNotNull(commitRequest(false)), t -> {
396 if (t instanceof TransactionCommitSuccess) {
397 ret.set(Boolean.TRUE);
398 } else if (t instanceof RequestFailure) {
399 ret.setException(((RequestFailure<?, ?>) t).getCause().unwrap());
401 ret.setException(new IllegalStateException("Unhandled response " + t.getClass()));
404 // This is a terminal request, hence we do not need to record it
405 LOG.debug("Transaction {} directCommit completed", this);
413 // We have had some interference with successor injection, wait for it to complete and defer to the successor.
414 return awaitSuccessor().directCommit();
417 final void canCommit(final VotingFuture<?> ret) {
421 // Precludes startReconnect() from interfering with the fast path
422 synchronized (this) {
423 if (STATE_UPDATER.compareAndSet(this, SEALED, FLUSHED)) {
424 final TransactionRequest<?> req = Verify.verifyNotNull(commitRequest(true));
426 sendRequest(req, t -> {
427 if (t instanceof TransactionCanCommitSuccess) {
429 } else if (t instanceof RequestFailure) {
430 ret.voteNo(((RequestFailure<?, ?>) t).getCause().unwrap());
432 ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass()));
435 recordSuccessfulRequest(req);
436 LOG.debug("Transaction {} canCommit completed", this);
443 // We have had some interference with successor injection, wait for it to complete and defer to the successor.
444 awaitSuccessor().canCommit(ret);
447 private AbstractProxyTransaction awaitSuccessor() {
448 return getSuccessorState().await();
451 final void preCommit(final VotingFuture<?> ret) {
455 final TransactionRequest<?> req = new TransactionPreCommitRequest(getIdentifier(), nextSequence(),
457 sendRequest(req, t -> {
458 if (t instanceof TransactionPreCommitSuccess) {
460 } else if (t instanceof RequestFailure) {
461 ret.voteNo(((RequestFailure<?, ?>) t).getCause().unwrap());
463 ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass()));
466 onPreCommitComplete(req);
470 private void onPreCommitComplete(final TransactionRequest<?> req) {
472 * The backend has agreed that the transaction has entered PRE_COMMIT phase, meaning it will be committed
473 * to storage after the timeout completes.
475 * All state has been replicated to the backend, hence we do not need to keep it around. Retain only
476 * the precommit request, so we know which request to use for resync.
478 LOG.debug("Transaction {} preCommit completed, clearing successfulRequests", this);
479 successfulRequests.clear();
481 // TODO: this works, but can contain some useless state (like batched operations). Create an empty
482 // equivalent of this request and store that.
483 recordSuccessfulRequest(req);
486 final void doCommit(final VotingFuture<?> ret) {
490 sendRequest(new TransactionDoCommitRequest(getIdentifier(), nextSequence(), localActor()), t -> {
491 if (t instanceof TransactionCommitSuccess) {
493 } else if (t instanceof RequestFailure) {
494 ret.voteNo(((RequestFailure<?, ?>) t).getCause().unwrap());
496 ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass()));
499 LOG.debug("Transaction {} doCommit completed", this);
504 private void enqueuePurge() {
508 final void enqueuePurge(final Consumer<Response<?, ?>> callback) {
509 // Purge request are dispatched internally, hence should not wait
510 enqueuePurge(callback, parent.currentTime());
513 final void enqueuePurge(final Consumer<Response<?, ?>> callback, final long enqueuedTicks) {
514 enqueueRequest(purgeRequest(), resp -> {
515 LOG.debug("Transaction {} purge completed", this);
516 parent.completeTransaction(this);
517 if (callback != null) {
518 callback.accept(resp);
523 private TransactionPurgeRequest purgeRequest() {
524 successfulRequests.clear();
525 return new TransactionPurgeRequest(getIdentifier(), nextSequence(), localActor());
528 // Called with the connection unlocked
529 final synchronized void startReconnect() {
530 // At this point canCommit/directCommit are blocked, we assert a new successor state, retrieving the previous
531 // state. This method is called with the queue still unlocked.
532 final SuccessorState nextState = new SuccessorState();
533 final State prevState = STATE_UPDATER.getAndSet(this, nextState);
535 LOG.debug("Start reconnect of proxy {} previous state {}", this, prevState);
536 Verify.verify(!(prevState instanceof SuccessorState), "Proxy %s duplicate reconnect attempt after %s", this,
539 // We have asserted a slow-path state, seal(), canCommit(), directCommit() are forced to slow paths, which will
540 // wait until we unblock nextState's latch before accessing state. Now we record prevState for later use and we
542 nextState.setPrevState(prevState);
545 // Called with the connection locked
546 final void replayMessages(final ProxyHistory successorHistory, final Iterable<ConnectionEntry> enqueuedEntries) {
547 final SuccessorState local = getSuccessorState();
548 final State prevState = local.getPrevState();
550 final AbstractProxyTransaction successor = successorHistory.createTransactionProxy(getIdentifier(),
552 LOG.debug("{} created successor transaction proxy {}", this, successor);
553 local.setSuccessor(successor);
555 // Replay successful requests first
556 if (!successfulRequests.isEmpty()) {
557 // We need to find a good timestamp to use for successful requests, as we do not want to time them out
558 // nor create timing inconsistencies in the queue -- requests are expected to be ordered by their enqueue
559 // time. We will pick the time of the first entry available. If there is none, we will just use current
560 // time, as all other requests will get enqueued afterwards.
561 final ConnectionEntry firstInQueue = Iterables.getFirst(enqueuedEntries, null);
562 final long now = firstInQueue != null ? firstInQueue.getEnqueuedTicks() : parent.currentTime();
564 for (Object obj : successfulRequests) {
565 if (obj instanceof TransactionRequest) {
566 LOG.debug("Forwarding successful request {} to successor {}", obj, successor);
567 successor.replayRequest((TransactionRequest<?>) obj, resp -> { }, now);
569 Verify.verify(obj instanceof IncrementSequence);
570 final IncrementSequence increment = (IncrementSequence) obj;
571 successor.replayRequest(new IncrementTransactionSequenceRequest(getIdentifier(),
572 increment.getSequence(), localActor(), isSnapshotOnly(), increment.getDelta()), resp -> { },
574 LOG.debug("Incrementing sequence {} to successor {}", obj, successor);
577 LOG.debug("{} replayed {} successful requests", getIdentifier(), successfulRequests.size());
578 successfulRequests.clear();
581 // Now replay whatever is in the connection
582 final Iterator<ConnectionEntry> it = enqueuedEntries.iterator();
583 while (it.hasNext()) {
584 final ConnectionEntry e = it.next();
585 final Request<?, ?> req = e.getRequest();
587 if (getIdentifier().equals(req.getTarget())) {
588 Verify.verify(req instanceof TransactionRequest, "Unhandled request %s", req);
589 LOG.debug("Replaying queued request {} to successor {}", req, successor);
590 successor.replayRequest((TransactionRequest<?>) req, e.getCallback(), e.getEnqueuedTicks());
596 * Check the state at which we have started the reconnect attempt. State transitions triggered while we were
597 * reconnecting have been forced to slow paths, which will be unlocked once we unblock the state latch
598 * at the end of this method.
600 if (SEALED.equals(prevState)) {
601 LOG.debug("Proxy {} reconnected while being sealed, propagating state to successor {}", this, successor);
602 flushState(successor);
603 successor.ensureSealed();
608 * Invoked from {@link #replayMessages(AbstractProxyTransaction, Iterable)} to have successor adopt an in-flight
612 * Note: this method is invoked by the predecessor on the successor.
614 * @param request Request which needs to be forwarded
615 * @param callback Callback to be invoked once the request completes
616 * @param enqueuedTicks ticker-based time stamp when the request was enqueued
618 private void replayRequest(final TransactionRequest<?> request, final Consumer<Response<?, ?>> callback,
619 final long enqueuedTicks) {
620 if (request instanceof AbstractLocalTransactionRequest) {
621 handleReplayedLocalRequest((AbstractLocalTransactionRequest<?>) request, callback, enqueuedTicks);
623 handleReplayedRemoteRequest(request, callback, enqueuedTicks);
627 // Called with the connection locked
628 final void finishReconnect() {
629 final SuccessorState local = getSuccessorState();
630 LOG.debug("Finishing reconnect of proxy {}", this);
632 // All done, release the latch, unblocking seal() and canCommit() slow paths
637 * Invoked from a retired connection for requests which have been in-flight and need to be re-adjusted
638 * and forwarded to the successor connection.
640 * @param request Request to be forwarded
641 * @param callback Original callback
643 final void forwardRequest(final TransactionRequest<?> request, final Consumer<Response<?, ?>> callback) {
644 forwardToSuccessor(getSuccessorState().getSuccessor(), request, callback);
647 final void forwardToSuccessor(final AbstractProxyTransaction successor, final TransactionRequest<?> request,
648 final Consumer<Response<?, ?>> callback) {
649 if (successor instanceof LocalProxyTransaction) {
650 forwardToLocal((LocalProxyTransaction)successor, request, callback);
651 } else if (successor instanceof RemoteProxyTransaction) {
652 forwardToRemote((RemoteProxyTransaction)successor, request, callback);
654 throw new IllegalStateException("Unhandled successor " + successor);
658 abstract boolean isSnapshotOnly();
660 abstract void doDelete(YangInstanceIdentifier path);
662 abstract void doMerge(YangInstanceIdentifier path, NormalizedNode<?, ?> data);
664 abstract void doWrite(YangInstanceIdentifier path, NormalizedNode<?, ?> data);
666 abstract CheckedFuture<Boolean, ReadFailedException> doExists(YangInstanceIdentifier path);
668 abstract CheckedFuture<Optional<NormalizedNode<?, ?>>, ReadFailedException> doRead(YangInstanceIdentifier path);
670 abstract void doSeal();
673 abstract void flushState(AbstractProxyTransaction successor);
675 abstract TransactionRequest<?> abortRequest();
677 abstract TransactionRequest<?> commitRequest(boolean coordinated);
680 * Replay a request originating in this proxy to a successor remote proxy.
682 abstract void forwardToRemote(RemoteProxyTransaction successor, TransactionRequest<?> request,
683 Consumer<Response<?, ?>> callback);
686 * Replay a request originating in this proxy to a successor local proxy.
688 abstract void forwardToLocal(LocalProxyTransaction successor, TransactionRequest<?> request,
689 Consumer<Response<?, ?>> callback);
692 * Invoked from {@link LocalProxyTransaction} when it replays its successful requests to its successor.
695 * Note: this method is invoked by the predecessor on the successor.
697 * @param request Request which needs to be forwarded
698 * @param callback Callback to be invoked once the request completes
699 * @param enqueuedTicks Time stamp to use for enqueue time
701 abstract void handleReplayedLocalRequest(AbstractLocalTransactionRequest<?> request,
702 @Nullable Consumer<Response<?, ?>> callback, long enqueuedTicks);
705 * Invoked from {@link RemoteProxyTransaction} when it replays its successful requests to its successor.
708 * Note: this method is invoked by the predecessor on the successor.
710 * @param request Request which needs to be forwarded
711 * @param callback Callback to be invoked once the request completes
712 * @param enqueuedTicks Time stamp to use for enqueue time
714 abstract void handleReplayedRemoteRequest(TransactionRequest<?> request,
715 @Nullable Consumer<Response<?, ?>> callback, long enqueuedTicks);
718 public final String toString() {
719 return MoreObjects.toStringHelper(this).add("identifier", getIdentifier()).add("state", state).toString();