2 * Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.controller.cluster.databroker.actors.dds;
10 import akka.actor.ActorRef;
11 import com.google.common.base.Optional;
12 import com.google.common.base.Preconditions;
13 import com.google.common.base.Throwables;
14 import com.google.common.base.Verify;
15 import com.google.common.util.concurrent.CheckedFuture;
16 import com.google.common.util.concurrent.ListenableFuture;
17 import com.google.common.util.concurrent.SettableFuture;
18 import java.util.ArrayDeque;
19 import java.util.Deque;
20 import java.util.Iterator;
21 import java.util.concurrent.CountDownLatch;
22 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
23 import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
24 import java.util.function.Consumer;
25 import javax.annotation.Nonnull;
26 import javax.annotation.Nullable;
27 import javax.annotation.concurrent.GuardedBy;
28 import javax.annotation.concurrent.NotThreadSafe;
29 import org.opendaylight.controller.cluster.access.client.ConnectionEntry;
30 import org.opendaylight.controller.cluster.access.commands.TransactionAbortRequest;
31 import org.opendaylight.controller.cluster.access.commands.TransactionAbortSuccess;
32 import org.opendaylight.controller.cluster.access.commands.TransactionCanCommitSuccess;
33 import org.opendaylight.controller.cluster.access.commands.TransactionCommitSuccess;
34 import org.opendaylight.controller.cluster.access.commands.TransactionDoCommitRequest;
35 import org.opendaylight.controller.cluster.access.commands.TransactionPreCommitRequest;
36 import org.opendaylight.controller.cluster.access.commands.TransactionPreCommitSuccess;
37 import org.opendaylight.controller.cluster.access.commands.TransactionRequest;
38 import org.opendaylight.controller.cluster.access.concepts.Request;
39 import org.opendaylight.controller.cluster.access.concepts.RequestFailure;
40 import org.opendaylight.controller.cluster.access.concepts.Response;
41 import org.opendaylight.controller.cluster.access.concepts.TransactionIdentifier;
42 import org.opendaylight.mdsal.common.api.ReadFailedException;
43 import org.opendaylight.yangtools.concepts.Identifiable;
44 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
45 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
46 import org.slf4j.Logger;
47 import org.slf4j.LoggerFactory;
50 * Class translating transaction operations towards a particular backend shard.
53 * This class is not safe to access from multiple application threads, as is usual for transactions. Internal state
54 * transitions coming from interactions with backend are expected to be thread-safe.
57 * This class interacts with the queueing mechanism in ClientActorBehavior, hence once we arrive at a decision
58 * to use either a local or remote implementation, we are stuck with it. We can re-evaluate on the next transaction.
60 * @author Robert Varga
62 abstract class AbstractProxyTransaction implements Identifiable<TransactionIdentifier> {
64 * Marker object used instead of read-type of requests, which are satisfied only once. This has a lower footprint
65 * and allows compressing multiple requests into a single entry.
68 private static final class IncrementSequence {
69 private long delta = 1;
75 void incrementDelta() {
80 // Generic state base class. Direct instances are used for fast paths, sub-class is used for successor transitions
81 private static class State {
82 private final String string;
84 State(final String string) {
85 this.string = Preconditions.checkNotNull(string);
89 public final String toString() {
94 // State class used when a successor has interfered. Contains coordinator latch, the successor and previous state
95 private static final class SuccessorState extends State {
96 private final CountDownLatch latch = new CountDownLatch(1);
97 private AbstractProxyTransaction successor;
98 private State prevState;
104 // Synchronize with succession process and return the successor
105 AbstractProxyTransaction await() {
108 } catch (InterruptedException e) {
109 LOG.warn("Interrupted while waiting for latch of {}", successor);
110 throw Throwables.propagate(e);
119 State getPrevState() {
123 void setPrevState(final State prevState) {
124 Verify.verify(this.prevState == null);
125 this.prevState = Preconditions.checkNotNull(prevState);
128 // To be called from safe contexts, where successor is known to be completed
129 AbstractProxyTransaction getSuccessor() {
130 return Verify.verifyNotNull(successor);
133 void setSuccessor(final AbstractProxyTransaction successor) {
134 Verify.verify(this.successor == null);
135 this.successor = Preconditions.checkNotNull(successor);
139 private static final Logger LOG = LoggerFactory.getLogger(AbstractProxyTransaction.class);
140 private static final AtomicIntegerFieldUpdater<AbstractProxyTransaction> SEALED_UPDATER =
141 AtomicIntegerFieldUpdater.newUpdater(AbstractProxyTransaction.class, "sealed");
142 private static final AtomicReferenceFieldUpdater<AbstractProxyTransaction, State> STATE_UPDATER =
143 AtomicReferenceFieldUpdater.newUpdater(AbstractProxyTransaction.class, State.class, "state");
144 private static final State OPEN = new State("open");
145 private static final State SEALED = new State("sealed");
146 private static final State FLUSHED = new State("flushed");
148 // Touched from client actor thread only
149 private final Deque<Object> successfulRequests = new ArrayDeque<>();
150 private final ProxyHistory parent;
152 // Accessed from user thread only, which may not access this object concurrently
153 private long sequence;
156 * Atomic state-keeping is required to synchronize the process of propagating completed transaction state towards
157 * the backend -- which may include a successor.
159 * Successor, unlike {@link AbstractProxyTransaction#seal()} is triggered from the client actor thread, which means
160 * the successor placement needs to be atomic with regard to the application thread.
162 * In the common case, the application thread performs performs the seal operations and then "immediately" sends
163 * the corresponding message. The uncommon case is when the seal and send operations race with a connect completion
164 * or timeout, when a successor is injected.
166 * This leaves the problem of needing to completely transferring state just after all queued messages are replayed
167 * after a successor was injected, so that it can be properly sealed if we are racing. Further complication comes
168 * from lock ordering, where the successor injection works with a locked queue and locks proxy objects -- leading
169 * to a potential AB-BA deadlock in case of a naive implementation.
171 * For tracking user-visible state we use a single volatile int, which is flipped atomically from 0 to 1 exactly
172 * once in {@link AbstractProxyTransaction#seal()}. That keeps common operations fast, as they need to perform
173 * only a single volatile read to assert state correctness.
175 * For synchronizing client actor (successor-injecting) and user (commit-driving) thread, we keep a separate state
176 * variable. It uses pre-allocated objects for fast paths (i.e. no successor present) and a per-transition object
177 * for slow paths (when successor is injected/present).
179 private volatile int sealed = 0;
180 private volatile State state = OPEN;
182 AbstractProxyTransaction(final ProxyHistory parent) {
183 this.parent = Preconditions.checkNotNull(parent);
186 final ActorRef localActor() {
187 return parent.localActor();
190 private void incrementSequence(final long delta) {
192 LOG.debug("Transaction {} incremented sequence to {}", this, sequence);
195 final long nextSequence() {
196 final long ret = sequence++;
197 LOG.debug("Transaction {} allocated sequence {}", this, ret);
201 final void delete(final YangInstanceIdentifier path) {
207 final void merge(final YangInstanceIdentifier path, final NormalizedNode<?, ?> data) {
213 final void write(final YangInstanceIdentifier path, final NormalizedNode<?, ?> data) {
219 final CheckedFuture<Boolean, ReadFailedException> exists(final YangInstanceIdentifier path) {
221 return doExists(path);
224 final CheckedFuture<Optional<NormalizedNode<?, ?>>, ReadFailedException> read(final YangInstanceIdentifier path) {
229 final void sendRequest(final TransactionRequest<?> request, final Consumer<Response<?, ?>> callback) {
230 LOG.debug("Transaction proxy {} sending request {} callback {}", this, request, callback);
231 parent.sendRequest(request, callback);
235 * Seal this transaction before it is either committed or aborted.
238 // Transition user-visible state first
239 final boolean success = SEALED_UPDATER.compareAndSet(this, 0, 1);
240 Preconditions.checkState(success, "Proxy %s was already sealed", getIdentifier());
244 final void ensureSealed() {
245 if (SEALED_UPDATER.compareAndSet(this, 0, 1)) {
250 private void internalSeal() {
252 parent.onTransactionSealed(this);
254 // Now deal with state transfer, which can occur via successor or a follow-up canCommit() or directCommit().
255 if (!STATE_UPDATER.compareAndSet(this, OPEN, SEALED)) {
256 // Slow path: wait for the successor to complete
257 final AbstractProxyTransaction successor = awaitSuccessor();
259 // At this point the successor has completed transition and is possibly visible by the user thread, which is
260 // still stuck here. The successor has not seen final part of our state, nor the fact it is sealed.
261 // Propagate state and seal the successor.
262 flushState(successor);
263 successor.ensureSealed();
267 private void checkNotSealed() {
268 Preconditions.checkState(sealed == 0, "Transaction %s has already been sealed", getIdentifier());
271 private void checkSealed() {
272 Preconditions.checkState(sealed != 0, "Transaction %s has not been sealed yet", getIdentifier());
275 private SuccessorState getSuccessorState() {
276 final State local = state;
277 Verify.verify(local instanceof SuccessorState, "State %s has unexpected class", local);
278 return (SuccessorState) local;
281 private void checkReadWrite() {
282 if (isSnapshotOnly()) {
283 throw new UnsupportedOperationException("Transaction " + getIdentifier() + " is a read-only snapshot");
287 final void recordSuccessfulRequest(final @Nonnull TransactionRequest<?> req) {
288 successfulRequests.add(Verify.verifyNotNull(req));
291 final void recordFinishedRequest() {
292 final Object last = successfulRequests.peekLast();
293 if (last instanceof IncrementSequence) {
294 ((IncrementSequence) last).incrementDelta();
296 successfulRequests.addLast(new IncrementSequence());
301 * Abort this transaction. This is invoked only for read-only transactions and will result in an explicit message
302 * being sent to the backend.
307 parent.abortTransaction(this);
310 final void abort(final VotingFuture<Void> ret) {
314 if (t instanceof TransactionAbortSuccess) {
316 } else if (t instanceof RequestFailure) {
317 ret.voteNo(((RequestFailure<?, ?>) t).getCause());
319 ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass()));
322 // This is a terminal request, hence we do not need to record it
323 LOG.debug("Transaction {} abort completed", this);
324 parent.completeTransaction(this);
328 final void sendAbort(final Consumer<Response<?, ?>> callback) {
329 sendRequest(new TransactionAbortRequest(getIdentifier(), nextSequence(), localActor()), callback);
333 * Commit this transaction, possibly in a coordinated fashion.
335 * @param coordinated True if this transaction should be coordinated across multiple participants.
336 * @return Future completion
338 final ListenableFuture<Boolean> directCommit() {
342 // Precludes startReconnect() from interfering with the fast path
343 synchronized (this) {
344 if (STATE_UPDATER.compareAndSet(this, SEALED, FLUSHED)) {
345 final SettableFuture<Boolean> ret = SettableFuture.create();
346 sendRequest(Verify.verifyNotNull(commitRequest(false)), t -> {
347 if (t instanceof TransactionCommitSuccess) {
348 ret.set(Boolean.TRUE);
349 } else if (t instanceof RequestFailure) {
350 ret.setException(((RequestFailure<?, ?>) t).getCause());
352 ret.setException(new IllegalStateException("Unhandled response " + t.getClass()));
355 // This is a terminal request, hence we do not need to record it
356 LOG.debug("Transaction {} directCommit completed", this);
357 parent.completeTransaction(this);
364 // We have had some interference with successor injection, wait for it to complete and defer to the successor.
365 return awaitSuccessor().directCommit();
368 final void canCommit(final VotingFuture<?> ret) {
372 // Precludes startReconnect() from interfering with the fast path
373 synchronized (this) {
374 if (STATE_UPDATER.compareAndSet(this, SEALED, FLUSHED)) {
375 final TransactionRequest<?> req = Verify.verifyNotNull(commitRequest(true));
377 sendRequest(req, t -> {
378 if (t instanceof TransactionCanCommitSuccess) {
380 } else if (t instanceof RequestFailure) {
381 ret.voteNo(((RequestFailure<?, ?>) t).getCause());
383 ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass()));
386 recordSuccessfulRequest(req);
387 LOG.debug("Transaction {} canCommit completed", this);
394 // We have had some interference with successor injection, wait for it to complete and defer to the successor.
395 awaitSuccessor().canCommit(ret);
398 private AbstractProxyTransaction awaitSuccessor() {
399 return getSuccessorState().await();
402 final void preCommit(final VotingFuture<?> ret) {
406 final TransactionRequest<?> req = new TransactionPreCommitRequest(getIdentifier(), nextSequence(),
408 sendRequest(req, t -> {
409 if (t instanceof TransactionPreCommitSuccess) {
411 } else if (t instanceof RequestFailure) {
412 ret.voteNo(((RequestFailure<?, ?>) t).getCause());
414 ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass()));
417 recordSuccessfulRequest(req);
418 LOG.debug("Transaction {} preCommit completed", this);
422 final void doCommit(final VotingFuture<?> ret) {
426 sendRequest(new TransactionDoCommitRequest(getIdentifier(), nextSequence(), localActor()), t -> {
427 if (t instanceof TransactionCommitSuccess) {
429 } else if (t instanceof RequestFailure) {
430 ret.voteNo(((RequestFailure<?, ?>) t).getCause());
432 ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass()));
435 LOG.debug("Transaction {} doCommit completed", this);
436 parent.completeTransaction(this);
440 // Called with the connection unlocked
441 final synchronized void startReconnect() {
442 // At this point canCommit/directCommit are blocked, we assert a new successor state, retrieving the previous
443 // state. This method is called with the queue still unlocked.
444 final SuccessorState nextState = new SuccessorState();
445 final State prevState = STATE_UPDATER.getAndSet(this, nextState);
447 LOG.debug("Start reconnect of proxy {} previous state {}", this, prevState);
448 Verify.verify(!(prevState instanceof SuccessorState), "Proxy %s duplicate reconnect attempt after %s", this,
451 // We have asserted a slow-path state, seal(), canCommit(), directCommit() are forced to slow paths, which will
452 // wait until we unblock nextState's latch before accessing state. Now we record prevState for later use and we
454 nextState.setPrevState(prevState);
457 // Called with the connection locked
458 final void replayMessages(final AbstractProxyTransaction successor,
459 final Iterable<ConnectionEntry> enqueuedEntries) {
460 final SuccessorState local = getSuccessorState();
461 local.setSuccessor(successor);
463 // Replay successful requests first
464 for (Object obj : successfulRequests) {
465 if (obj instanceof TransactionRequest) {
466 LOG.debug("Forwarding successful request {} to successor {}", obj, successor);
467 successor.handleForwardedRemoteRequest((TransactionRequest<?>) obj, null);
469 Verify.verify(obj instanceof IncrementSequence);
470 successor.incrementSequence(((IncrementSequence) obj).getDelta());
473 LOG.debug("{} replayed {} successful requests", getIdentifier(), successfulRequests.size());
474 successfulRequests.clear();
476 // Now replay whatever is in the connection
477 final Iterator<ConnectionEntry> it = enqueuedEntries.iterator();
478 while (it.hasNext()) {
479 final ConnectionEntry e = it.next();
480 final Request<?, ?> req = e.getRequest();
482 if (getIdentifier().equals(req.getTarget())) {
483 Verify.verify(req instanceof TransactionRequest, "Unhandled request %s", req);
484 LOG.debug("Forwarding queued request{} to successor {}", req, successor);
485 successor.handleForwardedRemoteRequest((TransactionRequest<?>) req, e.getCallback());
491 * Check the state at which we have started the reconnect attempt. State transitions triggered while we were
492 * reconnecting have been forced to slow paths, which will be unlocked once we unblock the state latch
493 * at the end of this method.
495 final State prevState = local.getPrevState();
496 if (SEALED.equals(prevState)) {
497 LOG.debug("Proxy {} reconnected while being sealed, propagating state to successor {}", this, successor);
498 flushState(successor);
499 successor.ensureSealed();
503 // Called with the connection locked
504 final void finishReconnect() {
505 final SuccessorState local = getSuccessorState();
506 LOG.debug("Finishing reconnect of proxy {}", this);
508 // All done, release the latch, unblocking seal() and canCommit() slow paths
513 * Invoked from a retired connection for requests which have been in-flight and need to be re-adjusted
514 * and forwarded to the successor connection.
516 * @param request Request to be forwarded
517 * @param callback Original callback
519 final void replayRequest(final TransactionRequest<?> request, final Consumer<Response<?, ?>> callback) {
520 final AbstractProxyTransaction successor = getSuccessorState().getSuccessor();
522 if (successor instanceof LocalProxyTransaction) {
523 forwardToLocal((LocalProxyTransaction)successor, request, callback);
524 } else if (successor instanceof RemoteProxyTransaction) {
525 forwardToRemote((RemoteProxyTransaction)successor, request, callback);
527 throw new IllegalStateException("Unhandled successor " + successor);
531 abstract boolean isSnapshotOnly();
533 abstract void doDelete(final YangInstanceIdentifier path);
535 abstract void doMerge(final YangInstanceIdentifier path, final NormalizedNode<?, ?> data);
537 abstract void doWrite(final YangInstanceIdentifier path, final NormalizedNode<?, ?> data);
539 abstract CheckedFuture<Boolean, ReadFailedException> doExists(final YangInstanceIdentifier path);
541 abstract CheckedFuture<Optional<NormalizedNode<?, ?>>, ReadFailedException> doRead(
542 final YangInstanceIdentifier path);
544 abstract void doSeal();
546 abstract void doAbort();
549 abstract void flushState(AbstractProxyTransaction successor);
551 abstract TransactionRequest<?> commitRequest(boolean coordinated);
554 * Invoked from {@link RemoteProxyTransaction} when it replays its successful requests to its successor. There is
555 * no equivalent of this call from {@link LocalProxyTransaction} because it does not send a request until all
556 * operations are packaged in the message.
559 * Note: this method is invoked by the predecessor on the successor.
561 * @param request Request which needs to be forwarded
562 * @param callback Callback to be invoked once the request completes
564 abstract void handleForwardedRemoteRequest(TransactionRequest<?> request,
565 @Nullable Consumer<Response<?, ?>> callback);
568 * Replay a request originating in this proxy to a successor remote proxy.
570 abstract void forwardToRemote(RemoteProxyTransaction successor, TransactionRequest<?> request,
571 Consumer<Response<?, ?>> callback);
574 * Replay a request originating in this proxy to a successor local proxy.
576 abstract void forwardToLocal(LocalProxyTransaction successor, TransactionRequest<?> request,
577 Consumer<Response<?, ?>> callback);