X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?p=controller.git;a=blobdiff_plain;f=opendaylight%2Fmd-sal%2Fsal-distributed-datastore%2Fsrc%2Fmain%2Fjava%2Forg%2Fopendaylight%2Fcontroller%2Fcluster%2Fdatastore%2FTransactionProxy.java;h=d93bae22e08d9fddb3f1ac7d9c12aa4a278d0ff6;hp=6cf16b44268c6c16e26e0658632f61994ee33971;hb=08dd5c2c443ff53f56af88a0e8dc8f34e36d2245;hpb=05a8052a457b2e53f06233f1a0b056d162118566 diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/TransactionProxy.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/TransactionProxy.java index 6cf16b4426..f1ba4eabb9 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/TransactionProxy.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/TransactionProxy.java @@ -8,10 +8,9 @@ package org.opendaylight.controller.cluster.datastore; -import akka.actor.ActorPath; import akka.actor.ActorSelection; +import akka.dispatch.Mapper; import akka.dispatch.OnComplete; - import com.google.common.annotations.VisibleForTesting; import com.google.common.base.FinalizablePhantomReference; import com.google.common.base.FinalizableReferenceQueue; @@ -19,22 +18,26 @@ import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.util.concurrent.CheckedFuture; -import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; - +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import javax.annotation.concurrent.GuardedBy; +import org.opendaylight.controller.cluster.datastore.compat.PreLithiumTransactionContextImpl; +import org.opendaylight.controller.cluster.datastore.exceptions.NoShardLeaderException; import org.opendaylight.controller.cluster.datastore.identifiers.TransactionIdentifier; import org.opendaylight.controller.cluster.datastore.messages.CloseTransaction; import org.opendaylight.controller.cluster.datastore.messages.CreateTransaction; import org.opendaylight.controller.cluster.datastore.messages.CreateTransactionReply; -import org.opendaylight.controller.cluster.datastore.messages.DataExists; -import org.opendaylight.controller.cluster.datastore.messages.DataExistsReply; -import org.opendaylight.controller.cluster.datastore.messages.DeleteData; -import org.opendaylight.controller.cluster.datastore.messages.MergeData; -import org.opendaylight.controller.cluster.datastore.messages.ReadData; -import org.opendaylight.controller.cluster.datastore.messages.ReadDataReply; -import org.opendaylight.controller.cluster.datastore.messages.ReadyTransaction; -import org.opendaylight.controller.cluster.datastore.messages.ReadyTransactionReply; -import org.opendaylight.controller.cluster.datastore.messages.WriteData; import org.opendaylight.controller.cluster.datastore.shardstrategy.ShardStrategyFactory; import org.opendaylight.controller.cluster.datastore.utils.ActorContext; import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException; @@ -46,17 +49,9 @@ import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode; import org.opendaylight.yangtools.yang.model.api.SchemaContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - -import scala.Function1; import scala.concurrent.Future; -import scala.runtime.AbstractFunction1; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; +import scala.concurrent.Promise; +import scala.concurrent.duration.FiniteDuration; /** * TransactionProxy acts as a proxy for one or more transactions that were created on a remote shard @@ -72,18 +67,26 @@ import java.util.concurrent.atomic.AtomicLong; */ public class TransactionProxy implements DOMStoreReadWriteTransaction { - private final TransactionChainProxy transactionChainProxy; - - - - public enum TransactionType { + public static enum TransactionType { READ_ONLY, WRITE_ONLY, - READ_WRITE + READ_WRITE; + + public static TransactionType fromInt(int type) { + if(type == WRITE_ONLY.ordinal()) { + return WRITE_ONLY; + } else if(type == READ_WRITE.ordinal()) { + return READ_WRITE; + } else if(type == READ_ONLY.ordinal()) { + return READ_ONLY; + } else { + throw new IllegalArgumentException("In TransactionType enum value" + type); + } + } } - static Function1 SAME_FAILURE_TRANSFORMER = new AbstractFunction1< - Throwable, Throwable>() { + static final Mapper SAME_FAILURE_TRANSFORMER = + new Mapper() { @Override public Throwable apply(Throwable failure) { return failure; @@ -92,9 +95,13 @@ public class TransactionProxy implements DOMStoreReadWriteTransaction { private static final AtomicLong counter = new AtomicLong(); - private static final Logger - LOG = LoggerFactory.getLogger(TransactionProxy.class); + private static final Logger LOG = LoggerFactory.getLogger(TransactionProxy.class); + /** + * Time interval in between transaction create retries. + */ + private static final FiniteDuration CREATE_TX_TRY_INTERVAL = + FiniteDuration.create(1, TimeUnit.SECONDS); /** * Used to enqueue the PhantomReferences for read-only TransactionProxy instances. The @@ -156,8 +163,7 @@ public class TransactionProxy implements DOMStoreReadWriteTransaction { if(remoteTransactionActorsMB.get()) { for(ActorSelection actor : remoteTransactionActors) { LOG.trace("Sending CloseTransaction to {}", actor); - actorContext.sendRemoteOperationAsync(actor, - new CloseTransaction().toSerializable()); + actorContext.sendOperationAsync(actor, CloseTransaction.INSTANCE.toSerializable()); } } } @@ -171,91 +177,117 @@ public class TransactionProxy implements DOMStoreReadWriteTransaction { * PhantomReference. */ private List remoteTransactionActors; - private AtomicBoolean remoteTransactionActorsMB; + private volatile AtomicBoolean remoteTransactionActorsMB; - private final Map remoteTransactionPaths = new HashMap<>(); + /** + * Stores the create transaction results per shard. + */ + private final Map txFutureCallbackMap = new HashMap<>(); private final TransactionType transactionType; private final ActorContext actorContext; private final TransactionIdentifier identifier; + private final String transactionChainId; private final SchemaContext schemaContext; private boolean inReadyState; - public TransactionProxy(ActorContext actorContext, TransactionType transactionType) { - this(actorContext, transactionType, null); - } - - @VisibleForTesting - List> getRecordedOperationFutures() { - List> recordedOperationFutures = Lists.newArrayList(); - for(TransactionContext transactionContext : remoteTransactionPaths.values()) { - recordedOperationFutures.addAll(transactionContext.getRecordedOperationFutures()); - } + private volatile boolean initialized; + private Semaphore operationLimiter; + private OperationCompleter operationCompleter; - return recordedOperationFutures; + public TransactionProxy(ActorContext actorContext, TransactionType transactionType) { + this(actorContext, transactionType, ""); } - public TransactionProxy(ActorContext actorContext, TransactionType transactionType, TransactionChainProxy transactionChainProxy) { + public TransactionProxy(ActorContext actorContext, TransactionType transactionType, + String transactionChainId) { this.actorContext = Preconditions.checkNotNull(actorContext, "actorContext should not be null"); this.transactionType = Preconditions.checkNotNull(transactionType, "transactionType should not be null"); this.schemaContext = Preconditions.checkNotNull(actorContext.getSchemaContext(), "schemaContext should not be null"); - this.transactionChainProxy = transactionChainProxy; + this.transactionChainId = transactionChainId; String memberName = actorContext.getCurrentMemberName(); if(memberName == null){ memberName = "UNKNOWN-MEMBER"; } - this.identifier = TransactionIdentifier.builder().memberName(memberName).counter( - counter.getAndIncrement()).build(); + this.identifier = new TransactionIdentifier(memberName, counter.getAndIncrement()); - if(transactionType == TransactionType.READ_ONLY) { - // Read-only Tx's aren't explicitly closed by the client so we create a PhantomReference - // to close the remote Tx's when this instance is no longer in use and is garbage - // collected. - - remoteTransactionActors = Lists.newArrayList(); - remoteTransactionActorsMB = new AtomicBoolean(); + LOG.debug("Created txn {} of type {} on chain {}", identifier, transactionType, transactionChainId); + } - TransactionProxyCleanupPhantomReference cleanup = - new TransactionProxyCleanupPhantomReference(this); - phantomReferenceCache.put(cleanup, cleanup); + @VisibleForTesting + List> getRecordedOperationFutures() { + List> recordedOperationFutures = Lists.newArrayList(); + for(TransactionFutureCallback txFutureCallback : txFutureCallbackMap.values()) { + TransactionContext transactionContext = txFutureCallback.getTransactionContext(); + if(transactionContext != null) { + recordedOperationFutures.addAll(transactionContext.getRecordedOperationFutures()); + } } - if(LOG.isDebugEnabled()) { - LOG.debug("Created txn {} of type {}", identifier, transactionType); + + return recordedOperationFutures; + } + + @VisibleForTesting + boolean hasTransactionContext() { + for(TransactionFutureCallback txFutureCallback : txFutureCallbackMap.values()) { + TransactionContext transactionContext = txFutureCallback.getTransactionContext(); + if(transactionContext != null) { + return true; + } } + + return false; } @Override - public CheckedFuture>, ReadFailedException> read( - final YangInstanceIdentifier path) { + public CheckedFuture>, ReadFailedException> read(final YangInstanceIdentifier path) { Preconditions.checkState(transactionType != TransactionType.WRITE_ONLY, "Read operation on write-only transaction is not allowed"); - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} read {}", identifier, path); - } - createTransactionIfMissing(actorContext, path); + LOG.debug("Tx {} read {}", identifier, path); + + throttleOperation(); + + final SettableFuture>> proxyFuture = SettableFuture.create(); - return transactionContext(path).readData(path); + TransactionFutureCallback txFutureCallback = getOrCreateTxFutureCallback(path); + txFutureCallback.enqueueTransactionOperation(new TransactionOperation() { + @Override + public void invoke(TransactionContext transactionContext) { + transactionContext.readData(path, proxyFuture); + } + }); + + return MappingCheckedFuture.create(proxyFuture, ReadFailedException.MAPPER); } @Override - public CheckedFuture exists(YangInstanceIdentifier path) { + public CheckedFuture exists(final YangInstanceIdentifier path) { Preconditions.checkState(transactionType != TransactionType.WRITE_ONLY, "Exists operation on write-only transaction is not allowed"); - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} exists {}", identifier, path); - } - createTransactionIfMissing(actorContext, path); + LOG.debug("Tx {} exists {}", identifier, path); + + throttleOperation(); + + final SettableFuture proxyFuture = SettableFuture.create(); + + TransactionFutureCallback txFutureCallback = getOrCreateTxFutureCallback(path); + txFutureCallback.enqueueTransactionOperation(new TransactionOperation() { + @Override + public void invoke(TransactionContext transactionContext) { + transactionContext.dataExists(path, proxyFuture); + } + }); - return transactionContext(path).dataExists(path); + return MappingCheckedFuture.create(proxyFuture, ReadFailedException.MAPPER); } private void checkModificationState() { @@ -265,42 +297,88 @@ public class TransactionProxy implements DOMStoreReadWriteTransaction { "Transaction is sealed - further modifications are not allowed"); } + private void throttleOperation() { + throttleOperation(1); + } + + private void throttleOperation(int acquirePermits) { + if(!initialized) { + // Note : Currently mailbox-capacity comes from akka.conf and not from the config-subsystem + operationLimiter = new Semaphore(actorContext.getTransactionOutstandingOperationLimit()); + operationCompleter = new OperationCompleter(operationLimiter); + + // Make sure we write this last because it's volatile and will also publish the non-volatile writes + // above as well so they'll be visible to other threads. + initialized = true; + } + + try { + if(!operationLimiter.tryAcquire(acquirePermits, + actorContext.getDatastoreContext().getOperationTimeoutInSeconds(), TimeUnit.SECONDS)){ + LOG.warn("Failed to acquire operation permit for transaction {}", getIdentifier()); + } + } catch (InterruptedException e) { + if(LOG.isDebugEnabled()) { + LOG.debug("Interrupted when trying to acquire operation permit for transaction " + getIdentifier().toString(), e); + } else { + LOG.warn("Interrupted when trying to acquire operation permit for transaction {}", getIdentifier()); + } + } + } + + @Override - public void write(YangInstanceIdentifier path, NormalizedNode data) { + public void write(final YangInstanceIdentifier path, final NormalizedNode data) { checkModificationState(); - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} write {}", identifier, path); - } - createTransactionIfMissing(actorContext, path); + LOG.debug("Tx {} write {}", identifier, path); - transactionContext(path).writeData(path, data); + throttleOperation(); + + TransactionFutureCallback txFutureCallback = getOrCreateTxFutureCallback(path); + txFutureCallback.enqueueTransactionOperation(new TransactionOperation() { + @Override + public void invoke(TransactionContext transactionContext) { + transactionContext.writeData(path, data); + } + }); } @Override - public void merge(YangInstanceIdentifier path, NormalizedNode data) { + public void merge(final YangInstanceIdentifier path, final NormalizedNode data) { checkModificationState(); - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} merge {}", identifier, path); - } - createTransactionIfMissing(actorContext, path); + LOG.debug("Tx {} merge {}", identifier, path); + + throttleOperation(); - transactionContext(path).mergeData(path, data); + TransactionFutureCallback txFutureCallback = getOrCreateTxFutureCallback(path); + txFutureCallback.enqueueTransactionOperation(new TransactionOperation() { + @Override + public void invoke(TransactionContext transactionContext) { + transactionContext.mergeData(path, data); + } + }); } @Override - public void delete(YangInstanceIdentifier path) { + public void delete(final YangInstanceIdentifier path) { checkModificationState(); - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} delete {}", identifier, path); - } - createTransactionIfMissing(actorContext, path); - transactionContext(path).deleteData(path); + LOG.debug("Tx {} delete {}", identifier, path); + + throttleOperation(); + + TransactionFutureCallback txFutureCallback = getOrCreateTxFutureCallback(path); + txFutureCallback.enqueueTransactionOperation(new TransactionOperation() { + @Override + public void invoke(TransactionContext transactionContext) { + transactionContext.deleteData(path); + } + }); } @Override @@ -310,29 +388,57 @@ public class TransactionProxy implements DOMStoreReadWriteTransaction { inReadyState = true; - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} Trying to get {} transactions ready for commit", identifier, - remoteTransactionPaths.size()); + LOG.debug("Tx {} Readying {} transactions for commit", identifier, + txFutureCallbackMap.size()); + + if(txFutureCallbackMap.size() == 0) { + onTransactionReady(Collections.>emptyList()); + TransactionRateLimitingCallback.adjustRateLimitForUnusedTransaction(actorContext); + return NoOpDOMStoreThreePhaseCommitCohort.INSTANCE; } - List> cohortPathFutures = Lists.newArrayList(); - for(TransactionContext transactionContext : remoteTransactionPaths.values()) { + throttleOperation(txFutureCallbackMap.size()); - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} Readying transaction for shard {}", identifier, - transactionContext.getShardName()); + List> cohortFutures = Lists.newArrayList(); + + for(TransactionFutureCallback txFutureCallback : txFutureCallbackMap.values()) { + + LOG.debug("Tx {} Readying transaction for shard {} chain {}", identifier, + txFutureCallback.getShardName(), transactionChainId); + + final TransactionContext transactionContext = txFutureCallback.getTransactionContext(); + final Future future; + if (transactionContext != null) { + // avoid the creation of a promise and a TransactionOperation + future = transactionContext.readyTransaction(); + } else { + final Promise promise = akka.dispatch.Futures.promise(); + txFutureCallback.enqueueTransactionOperation(new TransactionOperation() { + @Override + public void invoke(TransactionContext transactionContext) { + promise.completeWith(transactionContext.readyTransaction()); + } + }); + future = promise.future(); } - cohortPathFutures.add(transactionContext.readyTransaction()); - } - if(transactionChainProxy != null){ - transactionChainProxy.onTransactionReady(cohortPathFutures); + cohortFutures.add(future); } - return new ThreePhaseCommitCohortProxy(actorContext, cohortPathFutures, + onTransactionReady(cohortFutures); + + return new ThreePhaseCommitCohortProxy(actorContext, cohortFutures, identifier.toString()); } + /** + * Method for derived classes to be notified when the transaction has been readied. + * + * @param cohortFutures the cohort Futures for each shard transaction. + */ + protected void onTransactionReady(List> cohortFutures) { + } + @Override public Object getIdentifier() { return this.identifier; @@ -340,511 +446,358 @@ public class TransactionProxy implements DOMStoreReadWriteTransaction { @Override public void close() { - for(TransactionContext transactionContext : remoteTransactionPaths.values()) { - transactionContext.closeTransaction(); + for (TransactionFutureCallback txFutureCallback : txFutureCallbackMap.values()) { + txFutureCallback.enqueueTransactionOperation(new TransactionOperation() { + @Override + public void invoke(TransactionContext transactionContext) { + transactionContext.closeTransaction(); + } + }); } - remoteTransactionPaths.clear(); + txFutureCallbackMap.clear(); - if(transactionType == TransactionType.READ_ONLY) { + if(remoteTransactionActorsMB != null) { remoteTransactionActors.clear(); remoteTransactionActorsMB.set(true); } } - private TransactionContext transactionContext(YangInstanceIdentifier path){ - String shardName = shardNameFromIdentifier(path); - return remoteTransactionPaths.get(shardName); - } - private String shardNameFromIdentifier(YangInstanceIdentifier path){ return ShardStrategyFactory.getStrategy(path).findShard(path); } - private void createTransactionIfMissing(ActorContext actorContext, - YangInstanceIdentifier path) { - - if(transactionChainProxy != null){ - transactionChainProxy.waitTillCurrentTransactionReady(); - } - - String shardName = ShardStrategyFactory.getStrategy(path).findShard(path); - - TransactionContext transactionContext = - remoteTransactionPaths.get(shardName); - - if (transactionContext != null) { - // A transaction already exists with that shard - return; - } - - try { - Object response = actorContext.executeShardOperation(shardName, - new CreateTransaction(identifier.toString(), this.transactionType.ordinal(), - getTransactionChainId()).toSerializable()); - if (response.getClass().equals(CreateTransactionReply.SERIALIZABLE_CLASS)) { - CreateTransactionReply reply = - CreateTransactionReply.fromSerializable(response); + protected Future sendFindPrimaryShardAsync(String shardName) { + return actorContext.findPrimaryShardAsync(shardName); + } - String transactionPath = reply.getTransactionPath(); + private TransactionFutureCallback getOrCreateTxFutureCallback(YangInstanceIdentifier path) { + String shardName = shardNameFromIdentifier(path); + TransactionFutureCallback txFutureCallback = txFutureCallbackMap.get(shardName); + if(txFutureCallback == null) { + Future findPrimaryFuture = sendFindPrimaryShardAsync(shardName); - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} Received transaction path = {}", identifier, transactionPath); - } - ActorSelection transactionActor = actorContext.actorSelection(transactionPath); + final TransactionFutureCallback newTxFutureCallback = new TransactionFutureCallback(shardName); - if (transactionType == TransactionType.READ_ONLY) { - // Add the actor to the remoteTransactionActors list for access by the - // cleanup PhantonReference. - remoteTransactionActors.add(transactionActor); + txFutureCallback = newTxFutureCallback; + txFutureCallbackMap.put(shardName, txFutureCallback); - // Write to the memory barrier volatile to publish the above update to the - // remoteTransactionActors list for thread visibility. - remoteTransactionActorsMB.set(true); + findPrimaryFuture.onComplete(new OnComplete() { + @Override + public void onComplete(Throwable failure, ActorSelection primaryShard) { + if(failure != null) { + newTxFutureCallback.createTransactionContext(failure, null); + } else { + newTxFutureCallback.setPrimaryShard(primaryShard); + } } - - transactionContext = new TransactionContextImpl(shardName, transactionPath, - transactionActor, identifier, actorContext, schemaContext); - - remoteTransactionPaths.put(shardName, transactionContext); - } else { - throw new IllegalArgumentException(String.format( - "Invalid reply type {} for CreateTransaction", response.getClass())); - } - } catch (Exception e) { - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} Creating NoOpTransaction because of : {}", identifier, e.getMessage()); - } - remoteTransactionPaths - .put(shardName, new NoOpTransactionContext(shardName, e, identifier)); + }, actorContext.getClientDispatcher()); } + + return txFutureCallback; } public String getTransactionChainId() { - if(transactionChainProxy == null){ - return ""; - } - return transactionChainProxy.getTransactionChainId(); + return transactionChainId; } + protected ActorContext getActorContext() { + return actorContext; + } - private interface TransactionContext { - String getShardName(); - - void closeTransaction(); - - Future readyTransaction(); - - void writeData(YangInstanceIdentifier path, NormalizedNode data); - - void deleteData(YangInstanceIdentifier path); + /** + * Interfaces for transaction operations to be invoked later. + */ + private static interface TransactionOperation { + void invoke(TransactionContext transactionContext); + } - void mergeData(YangInstanceIdentifier path, NormalizedNode data); + /** + * Implements a Future OnComplete callback for a CreateTransaction message. This class handles + * retries, up to a limit, if the shard doesn't have a leader yet. This is done by scheduling a + * retry task after a short delay. + *

+ * The end result from a completed CreateTransaction message is a TransactionContext that is + * used to perform transaction operations. Transaction operations that occur before the + * CreateTransaction completes are cache and executed once the CreateTransaction completes, + * successfully or not. + */ + private class TransactionFutureCallback extends OnComplete { - CheckedFuture>, ReadFailedException> readData( - final YangInstanceIdentifier path); + /** + * The list of transaction operations to execute once the CreateTransaction completes. + */ + @GuardedBy("txOperationsOnComplete") + private final List txOperationsOnComplete = Lists.newArrayList(); - CheckedFuture dataExists(YangInstanceIdentifier path); + /** + * The TransactionContext resulting from the CreateTransaction reply. + */ + private volatile TransactionContext transactionContext; - List> getRecordedOperationFutures(); - } + /** + * The target primary shard. + */ + private volatile ActorSelection primaryShard; - private static abstract class AbstractTransactionContext implements TransactionContext { + private volatile int createTxTries = (int) (actorContext.getDatastoreContext(). + getShardLeaderElectionTimeout().duration().toMillis() / + CREATE_TX_TRY_INTERVAL.toMillis()); - protected final TransactionIdentifier identifier; - protected final String shardName; - protected final List> recordedOperationFutures = Lists.newArrayList(); + private final String shardName; - AbstractTransactionContext(String shardName, TransactionIdentifier identifier) { + TransactionFutureCallback(String shardName) { this.shardName = shardName; - this.identifier = identifier; } - @Override - public String getShardName() { + String getShardName() { return shardName; } - @Override - public List> getRecordedOperationFutures() { - return recordedOperationFutures; + TransactionContext getTransactionContext() { + return transactionContext; } - } - private static class TransactionContextImpl extends AbstractTransactionContext { - private final Logger LOG = LoggerFactory.getLogger(TransactionContextImpl.class); - private final ActorContext actorContext; - private final SchemaContext schemaContext; - private final String actorPath; - private final ActorSelection actor; - - private TransactionContextImpl(String shardName, String actorPath, - ActorSelection actor, TransactionIdentifier identifier, ActorContext actorContext, - SchemaContext schemaContext) { - super(shardName, identifier); - this.actorPath = actorPath; - this.actor = actor; - this.actorContext = actorContext; - this.schemaContext = schemaContext; - } + /** + * Sets the target primary shard and initiates a CreateTransaction try. + */ + void setPrimaryShard(ActorSelection primaryShard) { + this.primaryShard = primaryShard; - private ActorSelection getActor() { - return actor; - } + if(transactionType == TransactionType.WRITE_ONLY && + actorContext.getDatastoreContext().isWriteOnlyTransactionOptimizationsEnabled()) { + LOG.debug("Tx {} Primary shard {} found - creating WRITE_ONLY transaction context", + identifier, primaryShard); - private String getResolvedCohortPath(String cohortPath) { - return actorContext.resolvePath(actorPath, cohortPath); - } - - @Override - public void closeTransaction() { - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} closeTransaction called", identifier); + // For write-only Tx's we prepare the transaction modifications directly on the shard actor + // to avoid the overhead of creating a separate transaction actor. + // FIXME: can't assume the shard version is LITHIUM_VERSION - need to obtain it somehow. + executeTxOperatonsOnComplete(createValidTransactionContext(this.primaryShard, + this.primaryShard.path().toString(), DataStoreVersions.LITHIUM_VERSION)); + } else { + tryCreateTransaction(); } - actorContext.sendRemoteOperationAsync(getActor(), new CloseTransaction().toSerializable()); } - @Override - public Future readyTransaction() { - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} readyTransaction called with {} previous recorded operations pending", - identifier, recordedOperationFutures.size()); - } - // Send the ReadyTransaction message to the Tx actor. - - final Future replyFuture = actorContext.executeRemoteOperationAsync(getActor(), - new ReadyTransaction().toSerializable()); - - // Combine all the previously recorded put/merge/delete operation reply Futures and the - // ReadyTransactionReply Future into one Future. If any one fails then the combined - // Future will fail. We need all prior operations and the ready operation to succeed - // in order to attempt commit. - - List> futureList = - Lists.newArrayListWithCapacity(recordedOperationFutures.size() + 1); - futureList.addAll(recordedOperationFutures); - futureList.add(replyFuture); - - Future> combinedFutures = akka.dispatch.Futures.sequence(futureList, - actorContext.getActorSystem().dispatcher()); + /** + * Adds a TransactionOperation to be executed after the CreateTransaction completes. + */ + void addTxOperationOnComplete(TransactionOperation operation) { + boolean invokeOperation = true; + synchronized(txOperationsOnComplete) { + if(transactionContext == null) { + LOG.debug("Tx {} Adding operation on complete", identifier); - // Transform the combined Future into a Future that returns the cohort actor path from - // the ReadyTransactionReply. That's the end result of the ready operation. - - return combinedFutures.transform(new AbstractFunction1, ActorPath>() { - @Override - public ActorPath apply(Iterable notUsed) { - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} readyTransaction: pending recorded operations succeeded", - identifier); - } - // At this point all the Futures succeeded and we need to extract the cohort - // actor path from the ReadyTransactionReply. For the recorded operations, they - // don't return any data so we're only interested that they completed - // successfully. We could be paranoid and verify the correct reply types but - // that really should never happen so it's not worth the overhead of - // de-serializing each reply. - - // Note the Future get call here won't block as it's complete. - Object serializedReadyReply = replyFuture.value().get().get(); - if(serializedReadyReply.getClass().equals( - ReadyTransactionReply.SERIALIZABLE_CLASS)) { - ReadyTransactionReply reply = ReadyTransactionReply.fromSerializable( - actorContext.getActorSystem(), serializedReadyReply); - - String resolvedCohortPath = getResolvedCohortPath( - reply.getCohortPath().toString()); - - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} readyTransaction: resolved cohort path {}", - identifier, resolvedCohortPath); - } - return actorContext.actorFor(resolvedCohortPath); - } else { - // Throwing an exception here will fail the Future. - - throw new IllegalArgumentException(String.format("Invalid reply type {}", - serializedReadyReply.getClass())); - } + invokeOperation = false; + txOperationsOnComplete.add(operation); } - }, SAME_FAILURE_TRANSFORMER, actorContext.getActorSystem().dispatcher()); - } - - @Override - public void deleteData(YangInstanceIdentifier path) { - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} deleteData called path = {}", identifier, path); - } - recordedOperationFutures.add(actorContext.executeRemoteOperationAsync(getActor(), - new DeleteData(path).toSerializable() )); - } - - @Override - public void mergeData(YangInstanceIdentifier path, NormalizedNode data) { - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} mergeData called path = {}", identifier, path); } - recordedOperationFutures.add(actorContext.executeRemoteOperationAsync(getActor(), - new MergeData(path, data, schemaContext).toSerializable())); - } - @Override - public void writeData(YangInstanceIdentifier path, NormalizedNode data) { - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} writeData called path = {}", identifier, path); + if(invokeOperation) { + operation.invoke(transactionContext); } - recordedOperationFutures.add(actorContext.executeRemoteOperationAsync(getActor(), - new WriteData(path, data, schemaContext).toSerializable())); } - @Override - public CheckedFuture>, ReadFailedException> readData( - final YangInstanceIdentifier path) { + void enqueueTransactionOperation(final TransactionOperation op) { - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} readData called path = {}", identifier, path); - } - final SettableFuture>> returnFuture = SettableFuture.create(); - - // If there were any previous recorded put/merge/delete operation reply Futures then we - // must wait for them to successfully complete. This is necessary to honor the read - // uncommitted semantics of the public API contract. If any one fails then fail the read. - - if(recordedOperationFutures.isEmpty()) { - finishReadData(path, returnFuture); + if (transactionContext != null) { + op.invoke(transactionContext); } else { - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} readData: verifying {} previous recorded operations", - identifier, recordedOperationFutures.size()); - } - // Note: we make a copy of recordedOperationFutures to be on the safe side in case - // Futures#sequence accesses the passed List on a different thread, as - // recordedOperationFutures is not synchronized. - - Future> combinedFutures = akka.dispatch.Futures.sequence( - Lists.newArrayList(recordedOperationFutures), - actorContext.getActorSystem().dispatcher()); - OnComplete> onComplete = new OnComplete>() { - @Override - public void onComplete(Throwable failure, Iterable notUsed) - throws Throwable { - if(failure != null) { - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} readData: a recorded operation failed: {}", - identifier, failure); - } - returnFuture.setException(new ReadFailedException( - "The read could not be performed because a previous put, merge," - + "or delete operation failed", failure)); - } else { - finishReadData(path, returnFuture); - } - } - }; - - combinedFutures.onComplete(onComplete, actorContext.getActorSystem().dispatcher()); + // The shard Tx hasn't been created yet so add the Tx operation to the Tx Future + // callback to be executed after the Tx is created. + addTxOperationOnComplete(op); } - - return MappingCheckedFuture.create(returnFuture, ReadFailedException.MAPPER); } - private void finishReadData(final YangInstanceIdentifier path, - final SettableFuture>> returnFuture) { - + /** + * Performs a CreateTransaction try async. + */ + private void tryCreateTransaction() { if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} finishReadData called path = {}", identifier, path); + LOG.debug("Tx {} Primary shard {} found - trying create transaction", identifier, primaryShard); } - OnComplete onComplete = new OnComplete() { - @Override - public void onComplete(Throwable failure, Object readResponse) throws Throwable { - if(failure != null) { - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} read operation failed: {}", identifier, failure); - } - returnFuture.setException(new ReadFailedException( - "Error reading data for path " + path, failure)); - } else { - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} read operation succeeded", identifier, failure); - } - if (readResponse.getClass().equals(ReadDataReply.SERIALIZABLE_CLASS)) { - ReadDataReply reply = ReadDataReply.fromSerializable(schemaContext, - path, readResponse); - if (reply.getNormalizedNode() == null) { - returnFuture.set(Optional.>absent()); - } else { - returnFuture.set(Optional.>of( - reply.getNormalizedNode())); - } - } else { - returnFuture.setException(new ReadFailedException( - "Invalid response reading data for path " + path)); - } - } - } - }; + Object serializedCreateMessage = new CreateTransaction(identifier.toString(), + TransactionProxy.this.transactionType.ordinal(), + getTransactionChainId()).toSerializable(); + + Future createTxFuture = actorContext.executeOperationAsync(primaryShard, serializedCreateMessage); - Future readFuture = actorContext.executeRemoteOperationAsync(getActor(), - new ReadData(path).toSerializable()); - readFuture.onComplete(onComplete, actorContext.getActorSystem().dispatcher()); + createTxFuture.onComplete(this, actorContext.getClientDispatcher()); } @Override - public CheckedFuture dataExists( - final YangInstanceIdentifier path) { - - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} dataExists called path = {}", identifier, path); + public void onComplete(Throwable failure, Object response) { + if(failure instanceof NoShardLeaderException) { + // There's no leader for the shard yet - schedule and try again, unless we're out + // of retries. Note: createTxTries is volatile as it may be written by different + // threads however not concurrently, therefore decrementing it non-atomically here + // is ok. + if(--createTxTries > 0) { + LOG.debug("Tx {} Shard {} has no leader yet - scheduling create Tx retry", + identifier, shardName); + + actorContext.getActorSystem().scheduler().scheduleOnce(CREATE_TX_TRY_INTERVAL, + new Runnable() { + @Override + public void run() { + tryCreateTransaction(); + } + }, actorContext.getClientDispatcher()); + return; + } } - final SettableFuture returnFuture = SettableFuture.create(); - // If there were any previous recorded put/merge/delete operation reply Futures then we - // must wait for them to successfully complete. This is necessary to honor the read - // uncommitted semantics of the public API contract. If any one fails then fail this - // request. - - if(recordedOperationFutures.isEmpty()) { - finishDataExists(path, returnFuture); + createTransactionContext(failure, response); + } + + private void createTransactionContext(Throwable failure, Object response) { + // Mainly checking for state violation here to perform a volatile read of "initialized" to + // ensure updates to operationLimter et al are visible to this thread (ie we're doing + // "piggy-back" synchronization here). + Preconditions.checkState(initialized, "Tx was not propertly initialized."); + + // Create the TransactionContext from the response or failure. Store the new + // TransactionContext locally until we've completed invoking the + // TransactionOperations. This avoids thread timing issues which could cause + // out-of-order TransactionOperations. Eg, on a modification operation, if the + // TransactionContext is non-null, then we directly call the TransactionContext. + // However, at the same time, the code may be executing the cached + // TransactionOperations. So to avoid thus timing, we don't publish the + // TransactionContext until after we've executed all cached TransactionOperations. + TransactionContext localTransactionContext; + if(failure != null) { + LOG.debug("Tx {} Creating NoOpTransaction because of error", identifier, failure); + + localTransactionContext = new NoOpTransactionContext(failure, identifier, operationLimiter); + } else if (response.getClass().equals(CreateTransactionReply.SERIALIZABLE_CLASS)) { + localTransactionContext = createValidTransactionContext( + CreateTransactionReply.fromSerializable(response)); } else { - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} dataExists: verifying {} previous recorded operations", - identifier, recordedOperationFutures.size()); - } - // Note: we make a copy of recordedOperationFutures to be on the safe side in case - // Futures#sequence accesses the passed List on a different thread, as - // recordedOperationFutures is not synchronized. - - Future> combinedFutures = akka.dispatch.Futures.sequence( - Lists.newArrayList(recordedOperationFutures), - actorContext.getActorSystem().dispatcher()); - OnComplete> onComplete = new OnComplete>() { - @Override - public void onComplete(Throwable failure, Iterable notUsed) - throws Throwable { - if(failure != null) { - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} dataExists: a recorded operation failed: {}", - identifier, failure); - } - returnFuture.setException(new ReadFailedException( - "The data exists could not be performed because a previous " - + "put, merge, or delete operation failed", failure)); - } else { - finishDataExists(path, returnFuture); - } - } - }; + IllegalArgumentException exception = new IllegalArgumentException(String.format( + "Invalid reply type %s for CreateTransaction", response.getClass())); - combinedFutures.onComplete(onComplete, actorContext.getActorSystem().dispatcher()); + localTransactionContext = new NoOpTransactionContext(exception, identifier, operationLimiter); } - return MappingCheckedFuture.create(returnFuture, ReadFailedException.MAPPER); - } + executeTxOperatonsOnComplete(localTransactionContext); + } + + private void executeTxOperatonsOnComplete(TransactionContext localTransactionContext) { + while(true) { + // Access to txOperationsOnComplete and transactionContext must be protected and atomic + // (ie synchronized) with respect to #addTxOperationOnComplete to handle timing + // issues and ensure no TransactionOperation is missed and that they are processed + // in the order they occurred. + + // We'll make a local copy of the txOperationsOnComplete list to handle re-entrancy + // in case a TransactionOperation results in another transaction operation being + // queued (eg a put operation from a client read Future callback that is notified + // synchronously). + Collection operationsBatch = null; + synchronized(txOperationsOnComplete) { + if(txOperationsOnComplete.isEmpty()) { + // We're done invoking the TransactionOperations so we can now publish the + // TransactionContext. + transactionContext = localTransactionContext; + break; + } - private void finishDataExists(final YangInstanceIdentifier path, - final SettableFuture returnFuture) { + operationsBatch = new ArrayList<>(txOperationsOnComplete); + txOperationsOnComplete.clear(); + } - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} finishDataExists called path = {}", identifier, path); - } - OnComplete onComplete = new OnComplete() { - @Override - public void onComplete(Throwable failure, Object response) throws Throwable { - if(failure != null) { - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} dataExists operation failed: {}", identifier, failure); - } - returnFuture.setException(new ReadFailedException( - "Error checking data exists for path " + path, failure)); - } else { - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} dataExists operation succeeded", identifier, failure); - } - if (response.getClass().equals(DataExistsReply.SERIALIZABLE_CLASS)) { - returnFuture.set(Boolean.valueOf(DataExistsReply. - fromSerializable(response).exists())); - } else { - returnFuture.setException(new ReadFailedException( - "Invalid response checking exists for path " + path)); - } - } + // Invoke TransactionOperations outside the sync block to avoid unnecessary blocking. + // A slight down-side is that we need to re-acquire the lock below but this should + // be negligible. + for(TransactionOperation oper: operationsBatch) { + oper.invoke(localTransactionContext); } - }; + } + } + + private TransactionContext createValidTransactionContext(CreateTransactionReply reply) { + LOG.debug("Tx {} Received {}", identifier, reply); - Future future = actorContext.executeRemoteOperationAsync(getActor(), - new DataExists(path).toSerializable()); - future.onComplete(onComplete, actorContext.getActorSystem().dispatcher()); + return createValidTransactionContext(actorContext.actorSelection(reply.getTransactionPath()), + reply.getTransactionPath(), reply.getVersion()); } - } - private static class NoOpTransactionContext extends AbstractTransactionContext { + private TransactionContext createValidTransactionContext(ActorSelection transactionActor, + String transactionPath, short remoteTransactionVersion) { - private final Logger LOG = LoggerFactory.getLogger(NoOpTransactionContext.class); + if (transactionType == TransactionType.READ_ONLY) { + // Read-only Tx's aren't explicitly closed by the client so we create a PhantomReference + // to close the remote Tx's when this instance is no longer in use and is garbage + // collected. - private final Exception failure; + if(remoteTransactionActorsMB == null) { + remoteTransactionActors = Lists.newArrayList(); + remoteTransactionActorsMB = new AtomicBoolean(); - public NoOpTransactionContext(String shardName, Exception failure, - TransactionIdentifier identifier){ - super(shardName, identifier); - this.failure = failure; - } + TransactionProxyCleanupPhantomReference cleanup = + new TransactionProxyCleanupPhantomReference(TransactionProxy.this); + phantomReferenceCache.put(cleanup, cleanup); + } - @Override - public void closeTransaction() { - if(LOG.isDebugEnabled()) { - LOG.debug("NoOpTransactionContext {} closeTransaction called", identifier); + // Add the actor to the remoteTransactionActors list for access by the + // cleanup PhantonReference. + remoteTransactionActors.add(transactionActor); + + // Write to the memory barrier volatile to publish the above update to the + // remoteTransactionActors list for thread visibility. + remoteTransactionActorsMB.set(true); } - } - @Override - public Future readyTransaction() { - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} readyTransaction called", identifier); + // TxActor is always created where the leader of the shard is. + // Check if TxActor is created in the same node + boolean isTxActorLocal = actorContext.isPathLocal(transactionPath); + + if(remoteTransactionVersion < DataStoreVersions.LITHIUM_VERSION) { + return new PreLithiumTransactionContextImpl(transactionPath, transactionActor, identifier, + transactionChainId, actorContext, schemaContext, isTxActorLocal, remoteTransactionVersion, + operationCompleter); + } else if (transactionType == TransactionType.WRITE_ONLY && + actorContext.getDatastoreContext().isWriteOnlyTransactionOptimizationsEnabled()) { + return new WriteOnlyTransactionContextImpl(transactionActor, identifier, transactionChainId, + actorContext, schemaContext, isTxActorLocal, remoteTransactionVersion, operationCompleter); + } else { + return new TransactionContextImpl(transactionActor, identifier, transactionChainId, + actorContext, schemaContext, isTxActorLocal, remoteTransactionVersion, operationCompleter); } - return akka.dispatch.Futures.failed(failure); } + } - @Override - public void deleteData(YangInstanceIdentifier path) { - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} deleteData called path = {}", identifier, path); - } + private static class NoOpDOMStoreThreePhaseCommitCohort implements DOMStoreThreePhaseCommitCohort { + static NoOpDOMStoreThreePhaseCommitCohort INSTANCE = new NoOpDOMStoreThreePhaseCommitCohort(); + + private static final ListenableFuture IMMEDIATE_VOID_SUCCESS = + com.google.common.util.concurrent.Futures.immediateFuture(null); + private static final ListenableFuture IMMEDIATE_BOOLEAN_SUCCESS = + com.google.common.util.concurrent.Futures.immediateFuture(Boolean.TRUE); + + private NoOpDOMStoreThreePhaseCommitCohort() { } @Override - public void mergeData(YangInstanceIdentifier path, NormalizedNode data) { - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} mergeData called path = {}", identifier, path); - } + public ListenableFuture canCommit() { + return IMMEDIATE_BOOLEAN_SUCCESS; } @Override - public void writeData(YangInstanceIdentifier path, NormalizedNode data) { - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} writeData called path = {}", identifier, path); - } + public ListenableFuture preCommit() { + return IMMEDIATE_VOID_SUCCESS; } @Override - public CheckedFuture>, ReadFailedException> readData( - YangInstanceIdentifier path) { - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} readData called path = {}", identifier, path); - } - return Futures.immediateFailedCheckedFuture(new ReadFailedException( - "Error reading data for path " + path, failure)); + public ListenableFuture abort() { + return IMMEDIATE_VOID_SUCCESS; } @Override - public CheckedFuture dataExists( - YangInstanceIdentifier path) { - if(LOG.isDebugEnabled()) { - LOG.debug("Tx {} dataExists called path = {}", identifier, path); - } - return Futures.immediateFailedCheckedFuture(new ReadFailedException( - "Error checking exists for path " + path, failure)); + public ListenableFuture commit() { + return IMMEDIATE_VOID_SUCCESS; } } }