/*
* Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved.
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License v1.0 which accompanies this distribution,
* and is available at http://www.eclipse.org/legal/epl-v10.html
*/
package org.opendaylight.controller.cluster.databroker.actors.dds;
import akka.actor.ActorRef;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
import com.google.common.base.Verify;
import com.google.common.util.concurrent.CheckedFuture;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.SettableFuture;
import java.util.ArrayDeque;
import java.util.Deque;
import java.util.concurrent.CountDownLatch;
import java.util.function.Consumer;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.annotation.concurrent.GuardedBy;
import javax.annotation.concurrent.NotThreadSafe;
import org.opendaylight.controller.cluster.access.commands.TransactionAbortRequest;
import org.opendaylight.controller.cluster.access.commands.TransactionAbortSuccess;
import org.opendaylight.controller.cluster.access.commands.TransactionCanCommitSuccess;
import org.opendaylight.controller.cluster.access.commands.TransactionCommitSuccess;
import org.opendaylight.controller.cluster.access.commands.TransactionDoCommitRequest;
import org.opendaylight.controller.cluster.access.commands.TransactionPreCommitRequest;
import org.opendaylight.controller.cluster.access.commands.TransactionPreCommitSuccess;
import org.opendaylight.controller.cluster.access.commands.TransactionRequest;
import org.opendaylight.controller.cluster.access.concepts.RequestException;
import org.opendaylight.controller.cluster.access.concepts.RequestFailure;
import org.opendaylight.controller.cluster.access.concepts.Response;
import org.opendaylight.controller.cluster.access.concepts.TransactionIdentifier;
import org.opendaylight.mdsal.common.api.ReadFailedException;
import org.opendaylight.yangtools.concepts.Identifiable;
import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Class translating transaction operations towards a particular backend shard.
*
*
* This class is not safe to access from multiple application threads, as is usual for transactions. Internal state
* transitions coming from interactions with backend are expected to be thread-safe.
*
*
* This class interacts with the queueing mechanism in ClientActorBehavior, hence once we arrive at a decision
* to use either a local or remote implementation, we are stuck with it. We can re-evaluate on the next transaction.
*
* @author Robert Varga
*/
abstract class AbstractProxyTransaction implements Identifiable {
/**
* Marker object used instead of read-type of requests, which are satisfied only once. This has a lower footprint
* and allows compressing multiple requests into a single entry.
*/
@NotThreadSafe
private static final class IncrementSequence {
private long delta = 1;
long getDelta() {
return delta;
}
void incrementDelta() {
delta++;
}
}
private enum SealState {
/**
* The user has not sealed the transaction yet.
*/
OPEN,
/**
* The user has sealed the transaction, but has not issued a canCommit().
*/
SEALED,
/**
* The user has sealed the transaction and has issued a canCommit().
*/
FLUSHED,
}
private static final Logger LOG = LoggerFactory.getLogger(AbstractProxyTransaction.class);
private final Deque successfulRequests = new ArrayDeque<>();
private final ProxyHistory parent;
/*
* Atomic state-keeping is required to synchronize the process of propagating completed transaction state towards
* the backend -- which may include a successor.
*
* Successor, unlike {@link AbstractProxyTransaction#seal()} is triggered from the client actor thread, which means
* the successor placement needs to be atomic with regard to the application thread.
*
* In the common case, the application thread performs performs the seal operations and then "immediately" sends
* the corresponding message. The uncommon case is when the seal and send operations race with a connect completion
* or timeout, when a successor is injected.
*
* This leaves the problem of needing to completely transferring state just after all queued messages are replayed
* after a successor was injected, so that it can be properly sealed if we are racing.
*/
private volatile SealState sealed = SealState.OPEN;
@GuardedBy("this")
private AbstractProxyTransaction successor;
@GuardedBy("this")
private CountDownLatch successorLatch;
// Accessed from user thread only, which may not access this object concurrently
private long sequence;
AbstractProxyTransaction(final ProxyHistory parent) {
this.parent = Preconditions.checkNotNull(parent);
}
final ActorRef localActor() {
return parent.localActor();
}
private void incrementSequence(final long delta) {
sequence += delta;
LOG.debug("Transaction {} incremented sequence to {}", this, sequence);
}
final long nextSequence() {
final long ret = sequence++;
LOG.debug("Transaction {} allocated sequence {}", this, ret);
return ret;
}
final void delete(final YangInstanceIdentifier path) {
checkNotSealed();
doDelete(path);
}
final void merge(final YangInstanceIdentifier path, final NormalizedNode, ?> data) {
checkNotSealed();
doMerge(path, data);
}
final void write(final YangInstanceIdentifier path, final NormalizedNode, ?> data) {
checkNotSealed();
doWrite(path, data);
}
final CheckedFuture exists(final YangInstanceIdentifier path) {
checkNotSealed();
return doExists(path);
}
final CheckedFuture>, ReadFailedException> read(final YangInstanceIdentifier path) {
checkNotSealed();
return doRead(path);
}
final void sendRequest(final TransactionRequest> request, final Consumer> callback) {
LOG.debug("Transaction proxy {} sending request {} callback {}", this, request, callback);
parent.sendRequest(request, callback);
}
/**
* Seal this transaction before it is either committed or aborted.
*/
final void seal() {
final CountDownLatch localLatch;
synchronized (this) {
checkNotSealed();
doSeal();
// Fast path: no successor
if (successor == null) {
sealed = SealState.SEALED;
parent.onTransactionSealed(this);
return;
}
localLatch = successorLatch;
}
// Slow path: wait for the latch
LOG.debug("{} waiting on successor latch", getIdentifier());
try {
localLatch.await();
} catch (InterruptedException e) {
LOG.warn("{} interrupted while waiting for latch", getIdentifier());
throw Throwables.propagate(e);
}
synchronized (this) {
LOG.debug("{} reacquired lock", getIdentifier());
flushState(successor);
successor.seal();
sealed = SealState.FLUSHED;
parent.onTransactionSealed(this);
}
}
private void checkNotSealed() {
Preconditions.checkState(sealed == SealState.OPEN, "Transaction %s has already been sealed", getIdentifier());
}
private SealState checkSealed() {
final SealState local = sealed;
Preconditions.checkState(local != SealState.OPEN, "Transaction %s has not been sealed yet", getIdentifier());
return local;
}
final void recordSuccessfulRequest(final @Nonnull TransactionRequest> req) {
successfulRequests.add(Verify.verifyNotNull(req));
}
final void recordFinishedRequest() {
final Object last = successfulRequests.peekLast();
if (last instanceof IncrementSequence) {
((IncrementSequence) last).incrementDelta();
} else {
successfulRequests.addLast(new IncrementSequence());
}
}
/**
* Abort this transaction. This is invoked only for read-only transactions and will result in an explicit message
* being sent to the backend.
*/
final void abort() {
checkNotSealed();
doAbort();
parent.abortTransaction(this);
}
final void abort(final VotingFuture ret) {
checkSealed();
sendAbort(t -> {
if (t instanceof TransactionAbortSuccess) {
ret.voteYes();
} else if (t instanceof RequestFailure) {
ret.voteNo(((RequestFailure, ?>) t).getCause());
} else {
ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass()));
}
// This is a terminal request, hence we do not need to record it
LOG.debug("Transaction {} abort completed", this);
parent.completeTransaction(this);
});
}
final void sendAbort(final Consumer> callback) {
sendRequest(new TransactionAbortRequest(getIdentifier(), nextSequence(), localActor()), callback);
}
/**
* Commit this transaction, possibly in a coordinated fashion.
*
* @param coordinated True if this transaction should be coordinated across multiple participants.
* @return Future completion
*/
final ListenableFuture directCommit() {
final CountDownLatch localLatch;
synchronized (this) {
final SealState local = checkSealed();
// Fast path: no successor asserted
if (successor == null) {
Verify.verify(local == SealState.SEALED);
final SettableFuture ret = SettableFuture.create();
sendRequest(Verify.verifyNotNull(commitRequest(false)), t -> {
if (t instanceof TransactionCommitSuccess) {
ret.set(Boolean.TRUE);
} else if (t instanceof RequestFailure) {
ret.setException(((RequestFailure, ?>) t).getCause());
} else {
ret.setException(new IllegalStateException("Unhandled response " + t.getClass()));
}
// This is a terminal request, hence we do not need to record it
LOG.debug("Transaction {} directCommit completed", this);
parent.completeTransaction(this);
});
sealed = SealState.FLUSHED;
return ret;
}
// We have a successor, take its latch
localLatch = successorLatch;
}
// Slow path: we need to wait for the successor to completely propagate
LOG.debug("{} waiting on successor latch", getIdentifier());
try {
localLatch.await();
} catch (InterruptedException e) {
LOG.warn("{} interrupted while waiting for latch", getIdentifier());
throw Throwables.propagate(e);
}
synchronized (this) {
LOG.debug("{} reacquired lock", getIdentifier());
final SealState local = sealed;
Verify.verify(local == SealState.FLUSHED);
return successor.directCommit();
}
}
final void canCommit(final VotingFuture> ret) {
final CountDownLatch localLatch;
synchronized (this) {
final SealState local = checkSealed();
// Fast path: no successor asserted
if (successor == null) {
Verify.verify(local == SealState.SEALED);
final TransactionRequest> req = Verify.verifyNotNull(commitRequest(true));
sendRequest(req, t -> {
if (t instanceof TransactionCanCommitSuccess) {
ret.voteYes();
} else if (t instanceof RequestFailure) {
ret.voteNo(((RequestFailure, ?>) t).getCause());
} else {
ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass()));
}
recordSuccessfulRequest(req);
LOG.debug("Transaction {} canCommit completed", this);
});
sealed = SealState.FLUSHED;
return;
}
// We have a successor, take its latch
localLatch = successorLatch;
}
// Slow path: we need to wait for the successor to completely propagate
LOG.debug("{} waiting on successor latch", getIdentifier());
try {
localLatch.await();
} catch (InterruptedException e) {
LOG.warn("{} interrupted while waiting for latch", getIdentifier());
throw Throwables.propagate(e);
}
synchronized (this) {
LOG.debug("{} reacquired lock", getIdentifier());
final SealState local = sealed;
Verify.verify(local == SealState.FLUSHED);
successor.canCommit(ret);
}
}
final void preCommit(final VotingFuture> ret) {
checkSealed();
final TransactionRequest> req = new TransactionPreCommitRequest(getIdentifier(), nextSequence(),
localActor());
sendRequest(req, t -> {
if (t instanceof TransactionPreCommitSuccess) {
ret.voteYes();
} else if (t instanceof RequestFailure) {
ret.voteNo(((RequestFailure, ?>) t).getCause());
} else {
ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass()));
}
recordSuccessfulRequest(req);
LOG.debug("Transaction {} preCommit completed", this);
});
}
void doCommit(final VotingFuture> ret) {
checkSealed();
sendRequest(new TransactionDoCommitRequest(getIdentifier(), nextSequence(), localActor()), t -> {
if (t instanceof TransactionCommitSuccess) {
ret.voteYes();
} else if (t instanceof RequestFailure) {
ret.voteNo(((RequestFailure, ?>) t).getCause());
} else {
ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass()));
}
LOG.debug("Transaction {} doCommit completed", this);
parent.completeTransaction(this);
});
}
final synchronized void startReconnect(final AbstractProxyTransaction successor) {
Preconditions.checkState(this.successor == null);
this.successor = Preconditions.checkNotNull(successor);
for (Object obj : successfulRequests) {
if (obj instanceof TransactionRequest) {
LOG.debug("Forwarding request {} to successor {}", obj, successor);
successor.handleForwardedRemoteRequest((TransactionRequest>) obj, null);
} else {
Verify.verify(obj instanceof IncrementSequence);
successor.incrementSequence(((IncrementSequence) obj).getDelta());
}
}
LOG.debug("{} replayed {} successful requests", getIdentifier(), successfulRequests.size());
successfulRequests.clear();
/*
* Before releasing the lock we need to make sure that a call to seal() blocks until we have completed
* finishConnect().
*/
successorLatch = new CountDownLatch(1);
}
final synchronized void finishReconnect() {
Preconditions.checkState(successorLatch != null);
if (sealed == SealState.SEALED) {
/*
* If this proxy is in the 'sealed, have not sent canCommit' state. If so, we need to forward current
* leftover state to the successor now.
*/
flushState(successor);
successor.seal();
sealed = SealState.FLUSHED;
}
// All done, release the latch, unblocking seal() and canCommit()
successorLatch.countDown();
}
/**
* Invoked from a retired connection for requests which have been in-flight and need to be re-adjusted
* and forwarded to the successor connection.
*
* @param request Request to be forwarded
* @param callback Original callback
* @throws RequestException when the request is unhandled by the successor
*/
final synchronized void replayRequest(final TransactionRequest> request,
final Consumer> callback) {
Preconditions.checkState(successor != null, "%s does not have a successor set", this);
if (successor instanceof LocalProxyTransaction) {
forwardToLocal((LocalProxyTransaction)successor, request, callback);
} else if (successor instanceof RemoteProxyTransaction) {
forwardToRemote((RemoteProxyTransaction)successor, request, callback);
} else {
throw new IllegalStateException("Unhandled successor " + successor);
}
}
abstract void doDelete(final YangInstanceIdentifier path);
abstract void doMerge(final YangInstanceIdentifier path, final NormalizedNode, ?> data);
abstract void doWrite(final YangInstanceIdentifier path, final NormalizedNode, ?> data);
abstract CheckedFuture doExists(final YangInstanceIdentifier path);
abstract CheckedFuture>, ReadFailedException> doRead(
final YangInstanceIdentifier path);
abstract void doSeal();
abstract void doAbort();
@GuardedBy("this")
abstract void flushState(AbstractProxyTransaction successor);
abstract TransactionRequest> commitRequest(boolean coordinated);
/**
* Invoked from {@link RemoteProxyTransaction} when it replays its successful requests to its successor. There is
* no equivalent of this call from {@link LocalProxyTransaction} because it does not send a request until all
* operations are packaged in the message.
*
*
* Note: this method is invoked by the predecessor on the successor.
*
* @param request Request which needs to be forwarded
* @param callback Callback to be invoked once the request completes
*/
abstract void handleForwardedRemoteRequest(TransactionRequest> request,
@Nullable Consumer> callback);
/**
* Replay a request originating in this proxy to a successor remote proxy.
*/
abstract void forwardToRemote(RemoteProxyTransaction successor, TransactionRequest> request,
Consumer> callback);
/**
* Replay a request originating in this proxy to a successor local proxy.
*/
abstract void forwardToLocal(LocalProxyTransaction successor, TransactionRequest> request,
Consumer> callback);
}