X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?p=controller.git;a=blobdiff_plain;f=opendaylight%2Fmd-sal%2Fsal-distributed-datastore%2Fsrc%2Fmain%2Fjava%2Forg%2Fopendaylight%2Fcontroller%2Fcluster%2Fdatastore%2FTransactionProxy.java;h=f28a1e5f73d4823fe31ba20d14fa4d986f9700da;hp=cbd61b2087ac3c2bfd38f77ea3a34ed51d6786d8;hb=37f0504d391efd8b7d61403759fcc22a6dd3a093;hpb=dcbf09a50b0f7fb437386f0433bd4464db1193de diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/TransactionProxy.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/TransactionProxy.java index cbd61b2087..f28a1e5f73 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/TransactionProxy.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/TransactionProxy.java @@ -8,27 +8,33 @@ package org.opendaylight.controller.cluster.datastore; -import akka.actor.ActorPath; -import akka.actor.ActorRef; import akka.actor.ActorSelection; -import akka.actor.Props; - +import akka.dispatch.Mapper; +import akka.dispatch.OnComplete; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.FinalizablePhantomReference; +import com.google.common.base.FinalizableReferenceQueue; import com.google.common.base.Optional; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import com.google.common.util.concurrent.CheckedFuture; +import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListeningExecutorService; - -import org.opendaylight.controller.cluster.datastore.exceptions.TimeoutException; +import com.google.common.util.concurrent.SettableFuture; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import javax.annotation.concurrent.GuardedBy; +import org.opendaylight.controller.cluster.datastore.exceptions.NoShardLeaderException; +import org.opendaylight.controller.cluster.datastore.identifiers.TransactionIdentifier; import org.opendaylight.controller.cluster.datastore.messages.CloseTransaction; import org.opendaylight.controller.cluster.datastore.messages.CreateTransaction; import org.opendaylight.controller.cluster.datastore.messages.CreateTransactionReply; -import org.opendaylight.controller.cluster.datastore.messages.DeleteData; -import org.opendaylight.controller.cluster.datastore.messages.MergeData; -import org.opendaylight.controller.cluster.datastore.messages.ReadData; -import org.opendaylight.controller.cluster.datastore.messages.ReadDataReply; -import org.opendaylight.controller.cluster.datastore.messages.ReadyTransaction; -import org.opendaylight.controller.cluster.datastore.messages.ReadyTransactionReply; -import org.opendaylight.controller.cluster.datastore.messages.WriteData; import org.opendaylight.controller.cluster.datastore.shardstrategy.ShardStrategyFactory; import org.opendaylight.controller.cluster.datastore.utils.ActorContext; import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException; @@ -40,13 +46,9 @@ import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode; import org.opendaylight.yangtools.yang.model.api.SchemaContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.atomic.AtomicLong; +import scala.concurrent.Future; +import scala.concurrent.Promise; +import scala.concurrent.duration.FiniteDuration; /** * TransactionProxy acts as a proxy for one or more transactions that were created on a remote shard @@ -61,90 +63,361 @@ import java.util.concurrent.atomic.AtomicLong; *

*/ public class TransactionProxy implements DOMStoreReadWriteTransaction { - public enum TransactionType { + + public static enum TransactionType { READ_ONLY, WRITE_ONLY, READ_WRITE } + static final Mapper SAME_FAILURE_TRANSFORMER = + new Mapper() { + @Override + public Throwable apply(Throwable failure) { + return failure; + } + }; + private static final AtomicLong counter = new AtomicLong(); - private static final Logger - LOG = LoggerFactory.getLogger(TransactionProxy.class); + private static final Logger LOG = LoggerFactory.getLogger(TransactionProxy.class); + + /** + * Time interval in between transaction create retries. + */ + private static final FiniteDuration CREATE_TX_TRY_INTERVAL = + FiniteDuration.create(1, TimeUnit.SECONDS); + + /** + * Used to enqueue the PhantomReferences for read-only TransactionProxy instances. The + * FinalizableReferenceQueue is safe to use statically in an OSGi environment as it uses some + * trickery to clean up its internal thread when the bundle is unloaded. + */ + private static final FinalizableReferenceQueue phantomReferenceQueue = + new FinalizableReferenceQueue(); + + /** + * This stores the TransactionProxyCleanupPhantomReference instances statically, This is + * necessary because PhantomReferences need a hard reference so they're not garbage collected. + * Once finalized, the TransactionProxyCleanupPhantomReference removes itself from this map + * and thus becomes eligible for garbage collection. + */ + private static final Map phantomReferenceCache = + new ConcurrentHashMap<>(); + + /** + * A PhantomReference that closes remote transactions for a TransactionProxy when it's + * garbage collected. This is used for read-only transactions as they're not explicitly closed + * by clients. So the only way to detect that a transaction is no longer in use and it's safe + * to clean up is when it's garbage collected. It's inexact as to when an instance will be GC'ed + * but TransactionProxy instances should generally be short-lived enough to avoid being moved + * to the old generation space and thus should be cleaned up in a timely manner as the GC + * runs on the young generation (eden, swap1...) space much more frequently. + */ + private static class TransactionProxyCleanupPhantomReference + extends FinalizablePhantomReference { + + private final List remoteTransactionActors; + private final AtomicBoolean remoteTransactionActorsMB; + private final ActorContext actorContext; + private final TransactionIdentifier identifier; + + protected TransactionProxyCleanupPhantomReference(TransactionProxy referent) { + super(referent, phantomReferenceQueue); + + // Note we need to cache the relevant fields from the TransactionProxy as we can't + // have a hard reference to the TransactionProxy instance itself. + + remoteTransactionActors = referent.remoteTransactionActors; + remoteTransactionActorsMB = referent.remoteTransactionActorsMB; + actorContext = referent.actorContext; + identifier = referent.identifier; + } + + @Override + public void finalizeReferent() { + LOG.trace("Cleaning up {} Tx actors for TransactionProxy {}", + remoteTransactionActors.size(), identifier); + + phantomReferenceCache.remove(this); + // Access the memory barrier volatile to ensure all previous updates to the + // remoteTransactionActors list are visible to this thread. + + if(remoteTransactionActorsMB.get()) { + for(ActorSelection actor : remoteTransactionActors) { + LOG.trace("Sending CloseTransaction to {}", actor); + actorContext.sendOperationAsync(actor, CloseTransaction.INSTANCE.toSerializable()); + } + } + } + } + + /** + * Stores the remote Tx actors for each requested data store path to be used by the + * PhantomReference to close the remote Tx's. This is only used for read-only Tx's. The + * remoteTransactionActorsMB volatile serves as a memory barrier to publish updates to the + * remoteTransactionActors list so they will be visible to the thread accessing the + * PhantomReference. + */ + private List remoteTransactionActors; + private AtomicBoolean remoteTransactionActorsMB; + + /** + * Stores the create transaction results per shard. + */ + private final Map txFutureCallbackMap = new HashMap<>(); private final TransactionType transactionType; private final ActorContext actorContext; - private final Map remoteTransactionPaths = new HashMap<>(); - private final String identifier; - private final ListeningExecutorService executor; + private final TransactionIdentifier identifier; + private final String transactionChainId; private final SchemaContext schemaContext; + private boolean inReadyState; + private final Semaphore operationLimiter; + private final OperationCompleter operationCompleter; + + public TransactionProxy(ActorContext actorContext, TransactionType transactionType) { + this(actorContext, transactionType, ""); + } + + public TransactionProxy(ActorContext actorContext, TransactionType transactionType, + String transactionChainId) { + this.actorContext = Preconditions.checkNotNull(actorContext, + "actorContext should not be null"); + this.transactionType = Preconditions.checkNotNull(transactionType, + "transactionType should not be null"); + this.schemaContext = Preconditions.checkNotNull(actorContext.getSchemaContext(), + "schemaContext should not be null"); + this.transactionChainId = transactionChainId; + + String memberName = actorContext.getCurrentMemberName(); + if(memberName == null){ + memberName = "UNKNOWN-MEMBER"; + } - public TransactionProxy( - ActorContext actorContext, - TransactionType transactionType, - ListeningExecutorService executor, - SchemaContext schemaContext - ) { + this.identifier = TransactionIdentifier.builder().memberName(memberName).counter( + counter.getAndIncrement()).build(); - this.identifier = actorContext.getCurrentMemberName() + "-txn-" + counter.getAndIncrement(); - this.transactionType = transactionType; - this.actorContext = actorContext; - this.executor = executor; - this.schemaContext = schemaContext; + if(transactionType == TransactionType.READ_ONLY) { + // Read-only Tx's aren't explicitly closed by the client so we create a PhantomReference + // to close the remote Tx's when this instance is no longer in use and is garbage + // collected. + remoteTransactionActors = Lists.newArrayList(); + remoteTransactionActorsMB = new AtomicBoolean(); + TransactionProxyCleanupPhantomReference cleanup = + new TransactionProxyCleanupPhantomReference(this); + phantomReferenceCache.put(cleanup, cleanup); + } + + // Note : Currently mailbox-capacity comes from akka.conf and not from the config-subsystem + this.operationLimiter = new Semaphore(actorContext.getTransactionOutstandingOperationLimit()); + this.operationCompleter = new OperationCompleter(operationLimiter); + + LOG.debug("Created txn {} of type {} on chain {}", identifier, transactionType, transactionChainId); + } + + @VisibleForTesting + List> getRecordedOperationFutures() { + List> recordedOperationFutures = Lists.newArrayList(); + for(TransactionFutureCallback txFutureCallback : txFutureCallbackMap.values()) { + TransactionContext transactionContext = txFutureCallback.getTransactionContext(); + if(transactionContext != null) { + recordedOperationFutures.addAll(transactionContext.getRecordedOperationFutures()); + } + } + + return recordedOperationFutures; + } + + @VisibleForTesting + boolean hasTransactionContext() { + for(TransactionFutureCallback txFutureCallback : txFutureCallbackMap.values()) { + TransactionContext transactionContext = txFutureCallback.getTransactionContext(); + if(transactionContext != null) { + return true; + } + } + + return false; } @Override - public CheckedFuture>, ReadFailedException> read( - final YangInstanceIdentifier path) { + public CheckedFuture>, ReadFailedException> read(final YangInstanceIdentifier path) { + + Preconditions.checkState(transactionType != TransactionType.WRITE_ONLY, + "Read operation on write-only transaction is not allowed"); - createTransactionIfMissing(actorContext, path); + LOG.debug("Tx {} read {}", identifier, path); - return transactionContext(path).readData(path); + throttleOperation(); + + TransactionFutureCallback txFutureCallback = getOrCreateTxFutureCallback(path); + return txFutureCallback.enqueueReadOperation(new ReadOperation>>() { + @Override + public CheckedFuture>, ReadFailedException> invoke( + TransactionContext transactionContext) { + return transactionContext.readData(path); + } + }); } @Override - public void write(YangInstanceIdentifier path, NormalizedNode data) { + public CheckedFuture exists(final YangInstanceIdentifier path) { + + Preconditions.checkState(transactionType != TransactionType.WRITE_ONLY, + "Exists operation on write-only transaction is not allowed"); - createTransactionIfMissing(actorContext, path); + LOG.debug("Tx {} exists {}", identifier, path); - transactionContext(path).writeData(path, data); + throttleOperation(); + + TransactionFutureCallback txFutureCallback = getOrCreateTxFutureCallback(path); + return txFutureCallback.enqueueReadOperation(new ReadOperation() { + @Override + public CheckedFuture invoke(TransactionContext transactionContext) { + return transactionContext.dataExists(path); + } + }); + } + + + private void checkModificationState() { + Preconditions.checkState(transactionType != TransactionType.READ_ONLY, + "Modification operation on read-only transaction is not allowed"); + Preconditions.checkState(!inReadyState, + "Transaction is sealed - further modifications are not allowed"); + } + + private void throttleOperation() { + throttleOperation(1); + } + + private void throttleOperation(int acquirePermits) { + try { + if(!operationLimiter.tryAcquire(acquirePermits, actorContext.getDatastoreContext().getOperationTimeoutInSeconds(), TimeUnit.SECONDS)){ + LOG.warn("Failed to acquire operation permit for transaction {}", getIdentifier()); + } + } catch (InterruptedException e) { + if(LOG.isDebugEnabled()) { + LOG.debug("Interrupted when trying to acquire operation permit for transaction " + getIdentifier().toString(), e); + } else { + LOG.warn("Interrupted when trying to acquire operation permit for transaction {}", getIdentifier()); + } + } + } + + + @Override + public void write(final YangInstanceIdentifier path, final NormalizedNode data) { + + checkModificationState(); + + LOG.debug("Tx {} write {}", identifier, path); + + throttleOperation(); + + TransactionFutureCallback txFutureCallback = getOrCreateTxFutureCallback(path); + txFutureCallback.enqueueModifyOperation(new TransactionOperation() { + @Override + public void invoke(TransactionContext transactionContext) { + transactionContext.writeData(path, data); + } + }); } @Override - public void merge(YangInstanceIdentifier path, NormalizedNode data) { + public void merge(final YangInstanceIdentifier path, final NormalizedNode data) { + + checkModificationState(); - createTransactionIfMissing(actorContext, path); + LOG.debug("Tx {} merge {}", identifier, path); - transactionContext(path).mergeData(path, data); + throttleOperation(); + + TransactionFutureCallback txFutureCallback = getOrCreateTxFutureCallback(path); + txFutureCallback.enqueueModifyOperation(new TransactionOperation() { + @Override + public void invoke(TransactionContext transactionContext) { + transactionContext.mergeData(path, data); + } + }); } @Override - public void delete(YangInstanceIdentifier path) { + public void delete(final YangInstanceIdentifier path) { - createTransactionIfMissing(actorContext, path); + checkModificationState(); - transactionContext(path).deleteData(path); + LOG.debug("Tx {} delete {}", identifier, path); + + throttleOperation(); + + TransactionFutureCallback txFutureCallback = getOrCreateTxFutureCallback(path); + txFutureCallback.enqueueModifyOperation(new TransactionOperation() { + @Override + public void invoke(TransactionContext transactionContext) { + transactionContext.deleteData(path); + } + }); } @Override public DOMStoreThreePhaseCommitCohort ready() { - List cohortPaths = new ArrayList<>(); - for(TransactionContext transactionContext : remoteTransactionPaths.values()) { - Object result = transactionContext.readyTransaction(); + checkModificationState(); - if(result.getClass().equals(ReadyTransactionReply.SERIALIZABLE_CLASS)){ - ReadyTransactionReply reply = ReadyTransactionReply.fromSerializable(actorContext.getActorSystem(),result); - String resolvedCohortPath = transactionContext - .getResolvedCohortPath(reply.getCohortPath().toString()); - cohortPaths.add(actorContext.actorFor(resolvedCohortPath)); - } + throttleOperation(txFutureCallbackMap.size()); + + inReadyState = true; + + LOG.debug("Tx {} Readying {} transactions for commit", identifier, + txFutureCallbackMap.size()); + + List> cohortFutures = Lists.newArrayList(); + + for(TransactionFutureCallback txFutureCallback : txFutureCallbackMap.values()) { + + LOG.debug("Tx {} Readying transaction for shard {} chain {}", identifier, + txFutureCallback.getShardName(), transactionChainId); + + Future future = txFutureCallback.enqueueFutureOperation(new FutureOperation() { + @Override + public Future invoke(TransactionContext transactionContext) { + return transactionContext.readyTransaction(); + } + }); + + cohortFutures.add(future); } - return new ThreePhaseCommitCohortProxy(actorContext, cohortPaths, identifier, executor); + onTransactionReady(cohortFutures); + + return new ThreePhaseCommitCohortProxy(actorContext, cohortFutures, + identifier.toString()); + } + + /** + * Method for derived classes to be notified when the transaction has been readied. + * + * @param cohortFutures the cohort Futures for each shard transaction. + */ + protected void onTransactionReady(List> cohortFutures) { + } + + /** + * Method called to send a CreateTransaction message to a shard. + * + * @param shard the shard actor to send to + * @param serializedCreateMessage the serialized message to send + * @return the response Future + */ + protected Future sendCreateTransaction(ActorSelection shard, + Object serializedCreateMessage) { + return actorContext.executeOperationAsync(shard, serializedCreateMessage); } @Override @@ -154,208 +427,320 @@ public class TransactionProxy implements DOMStoreReadWriteTransaction { @Override public void close() { - for(TransactionContext transactionContext : remoteTransactionPaths.values()) { - transactionContext.closeTransaction(); + for (TransactionFutureCallback txFutureCallback : txFutureCallbackMap.values()) { + txFutureCallback.enqueueModifyOperation(new TransactionOperation() { + @Override + public void invoke(TransactionContext transactionContext) { + transactionContext.closeTransaction(); + } + }); } - } - private TransactionContext transactionContext(YangInstanceIdentifier path){ - String shardName = shardNameFromIdentifier(path); - return remoteTransactionPaths.get(shardName); + txFutureCallbackMap.clear(); + + if(transactionType == TransactionType.READ_ONLY) { + remoteTransactionActors.clear(); + remoteTransactionActorsMB.set(true); + } } private String shardNameFromIdentifier(YangInstanceIdentifier path){ return ShardStrategyFactory.getStrategy(path).findShard(path); } - private void createTransactionIfMissing(ActorContext actorContext, YangInstanceIdentifier path) { - String shardName = ShardStrategyFactory.getStrategy(path).findShard(path); - - TransactionContext transactionContext = - remoteTransactionPaths.get(shardName); - - if(transactionContext != null){ - // A transaction already exists with that shard - return; + private TransactionFutureCallback getOrCreateTxFutureCallback(YangInstanceIdentifier path) { + String shardName = shardNameFromIdentifier(path); + TransactionFutureCallback txFutureCallback = txFutureCallbackMap.get(shardName); + if(txFutureCallback == null) { + Future findPrimaryFuture = actorContext.findPrimaryShardAsync(shardName); + + final TransactionFutureCallback newTxFutureCallback = + new TransactionFutureCallback(shardName); + + txFutureCallback = newTxFutureCallback; + txFutureCallbackMap.put(shardName, txFutureCallback); + + findPrimaryFuture.onComplete(new OnComplete() { + @Override + public void onComplete(Throwable failure, ActorSelection primaryShard) { + if(failure != null) { + newTxFutureCallback.onComplete(failure, null); + } else { + newTxFutureCallback.setPrimaryShard(primaryShard); + } + } + }, actorContext.getActorSystem().dispatcher()); } - try { - Object response = actorContext.executeShardOperation(shardName, - new CreateTransaction(identifier).toSerializable(), - ActorContext.ASK_DURATION); - if (response.getClass() - .equals(CreateTransactionReply.SERIALIZABLE_CLASS)) { - CreateTransactionReply reply = - CreateTransactionReply.fromSerializable(response); - - String transactionPath = reply.getTransactionPath(); - - LOG.info("Received transaction path = {}" , transactionPath ); - - ActorSelection transactionActor = - actorContext.actorSelection(transactionPath); - transactionContext = - new TransactionContextImpl(shardName, transactionPath, - transactionActor); - - remoteTransactionPaths.put(shardName, transactionContext); - } - } catch(TimeoutException e){ - LOG.warn("Timed out trying to create transaction on shard {}: {}", shardName, e); - remoteTransactionPaths.put(shardName, new NoOpTransactionContext(shardName)); - } + return txFutureCallback; } - private interface TransactionContext { - String getShardName(); - - String getResolvedCohortPath(String cohortPath); - - public void closeTransaction(); - - public Object readyTransaction(); + public String getTransactionChainId() { + return transactionChainId; + } - void deleteData(YangInstanceIdentifier path); + protected ActorContext getActorContext() { + return actorContext; + } - void mergeData(YangInstanceIdentifier path, NormalizedNode data); + /** + * Interfaces for transaction operations to be invoked later. + */ + private static interface TransactionOperation { + void invoke(TransactionContext transactionContext); + } - CheckedFuture>, ReadFailedException> readData( - final YangInstanceIdentifier path); + /** + * This interface returns a Guava Future + */ + private static interface ReadOperation { + CheckedFuture invoke(TransactionContext transactionContext); + } - void writeData(YangInstanceIdentifier path, NormalizedNode data); + /** + * This interface returns a Scala Future + */ + private static interface FutureOperation { + Future invoke(TransactionContext transactionContext); } + /** + * Implements a Future OnComplete callback for a CreateTransaction message. This class handles + * retries, up to a limit, if the shard doesn't have a leader yet. This is done by scheduling a + * retry task after a short delay. + *

+ * The end result from a completed CreateTransaction message is a TransactionContext that is + * used to perform transaction operations. Transaction operations that occur before the + * CreateTransaction completes are cache and executed once the CreateTransaction completes, + * successfully or not. + */ + private class TransactionFutureCallback extends OnComplete { + + /** + * The list of transaction operations to execute once the CreateTransaction completes. + */ + @GuardedBy("txOperationsOnComplete") + private final List txOperationsOnComplete = Lists.newArrayList(); + + /** + * The TransactionContext resulting from the CreateTransaction reply. + */ + private volatile TransactionContext transactionContext; + + /** + * The target primary shard. + */ + private volatile ActorSelection primaryShard; + + private volatile int createTxTries = (int) (actorContext.getDatastoreContext(). + getShardLeaderElectionTimeout().duration().toMillis() / + CREATE_TX_TRY_INTERVAL.toMillis()); - private class TransactionContextImpl implements TransactionContext{ private final String shardName; - private final String actorPath; - private final ActorSelection actor; - - private TransactionContextImpl(String shardName, String actorPath, - ActorSelection actor) { + TransactionFutureCallback(String shardName) { this.shardName = shardName; - this.actorPath = actorPath; - this.actor = actor; } - @Override public String getShardName() { + String getShardName() { return shardName; } - private ActorSelection getActor() { - return actor; + TransactionContext getTransactionContext() { + return transactionContext; } - @Override public String getResolvedCohortPath(String cohortPath){ - return actorContext.resolvePath(actorPath, cohortPath); + + /** + * Sets the target primary shard and initiates a CreateTransaction try. + */ + void setPrimaryShard(ActorSelection primaryShard) { + LOG.debug("Tx {} Primary shard found - trying create transaction", identifier); + + this.primaryShard = primaryShard; + tryCreateTransaction(); } - @Override public void closeTransaction() { - getActor().tell( - new CloseTransaction().toSerializable(), null); + /** + * Adds a TransactionOperation to be executed after the CreateTransaction completes. + */ + void addTxOperationOnComplete(TransactionOperation operation) { + synchronized(txOperationsOnComplete) { + if(transactionContext == null) { + LOG.debug("Tx {} Adding operation on complete {}", identifier); + + txOperationsOnComplete.add(operation); + } else { + operation.invoke(transactionContext); + } + } } - @Override public Object readyTransaction() { - return actorContext.executeRemoteOperation(getActor(), - new ReadyTransaction().toSerializable(), - ActorContext.ASK_DURATION - ); - } + Future enqueueFutureOperation(final FutureOperation op) { - @Override public void deleteData(YangInstanceIdentifier path) { - getActor().tell(new DeleteData(path).toSerializable(), null); - } + Future future; - @Override public void mergeData(YangInstanceIdentifier path, NormalizedNode data){ - getActor().tell(new MergeData(path, data, schemaContext).toSerializable(), null); + if (transactionContext != null) { + future = op.invoke(transactionContext); + } else { + // The shard Tx hasn't been created yet so add the Tx operation to the Tx Future + // callback to be executed after the Tx is created. + final Promise promise = akka.dispatch.Futures.promise(); + addTxOperationOnComplete(new TransactionOperation() { + @Override + public void invoke(TransactionContext transactionContext) { + promise.completeWith(op.invoke(transactionContext)); + } + }); + + future = promise.future(); + } + + return future; } - @Override public CheckedFuture>, ReadFailedException> readData( - final YangInstanceIdentifier path) { - - Callable>> call = new Callable>>() { - - @Override public Optional> call() throws Exception { - Object response = actorContext - .executeRemoteOperation(getActor(), new ReadData(path).toSerializable(), - ActorContext.ASK_DURATION); - if(response.getClass().equals(ReadDataReply.SERIALIZABLE_CLASS)){ - ReadDataReply reply = ReadDataReply.fromSerializable(schemaContext,path, response); - if(reply.getNormalizedNode() == null){ - return Optional.absent(); - } - return Optional.>of(reply.getNormalizedNode()); + CheckedFuture enqueueReadOperation(final ReadOperation op) { + + CheckedFuture future; + + if (transactionContext != null) { + future = op.invoke(transactionContext); + } else { + // The shard Tx hasn't been created yet so add the Tx operation to the Tx Future + // callback to be executed after the Tx is created. + final SettableFuture proxyFuture = SettableFuture.create(); + addTxOperationOnComplete(new TransactionOperation() { + @Override + public void invoke(TransactionContext transactionContext) { + Futures.addCallback(op.invoke(transactionContext), new FutureCallback() { + @Override + public void onSuccess(T data) { + proxyFuture.set(data); + } + + @Override + public void onFailure(Throwable t) { + proxyFuture.setException(t); + } + }); } + }); - return Optional.absent(); - } - }; + future = MappingCheckedFuture.create(proxyFuture, ReadFailedException.MAPPER); + } - return MappingCheckedFuture.create(executor.submit(call), ReadFailedException.MAPPER); + return future; } - @Override public void writeData(YangInstanceIdentifier path, NormalizedNode data) { - getActor().tell(new WriteData(path, data, schemaContext).toSerializable(), null); + void enqueueModifyOperation(final TransactionOperation op) { + + if (transactionContext != null) { + op.invoke(transactionContext); + } else { + // The shard Tx hasn't been created yet so add the Tx operation to the Tx Future + // callback to be executed after the Tx is created. + addTxOperationOnComplete(op); + } } - } + /** + * Performs a CreateTransaction try async. + */ + private void tryCreateTransaction() { + Future createTxFuture = sendCreateTransaction(primaryShard, + new CreateTransaction(identifier.toString(), + TransactionProxy.this.transactionType.ordinal(), + getTransactionChainId()).toSerializable()); - private class NoOpTransactionContext implements TransactionContext { + createTxFuture.onComplete(this, actorContext.getActorSystem().dispatcher()); + } - private final Logger - LOG = LoggerFactory.getLogger(NoOpTransactionContext.class); + @Override + public void onComplete(Throwable failure, Object response) { + if(failure instanceof NoShardLeaderException) { + // There's no leader for the shard yet - schedule and try again, unless we're out + // of retries. Note: createTxTries is volatile as it may be written by different + // threads however not concurrently, therefore decrementing it non-atomically here + // is ok. + if(--createTxTries > 0) { + LOG.debug("Tx {} Shard {} has no leader yet - scheduling create Tx retry", + identifier, shardName); + + actorContext.getActorSystem().scheduler().scheduleOnce(CREATE_TX_TRY_INTERVAL, + new Runnable() { + @Override + public void run() { + tryCreateTransaction(); + } + }, actorContext.getActorSystem().dispatcher()); + return; + } + } - private final String shardName; + // Create the TransactionContext from the response or failure and execute delayed + // TransactionOperations. This entire section is done atomically (ie synchronized) with + // respect to #addTxOperationOnComplete to handle timing issues and ensure no + // TransactionOperation is missed and that they are processed in the order they occurred. + synchronized(txOperationsOnComplete) { + // Store the new TransactionContext locally until we've completed invoking the + // TransactionOperations. This avoids thread timing issues which could cause + // out-of-order TransactionOperations. Eg, on a modification operation, if the + // TransactionContext is non-null, then we directly call the TransactionContext. + // However, at the same time, the code may be executing the cached + // TransactionOperations. So to avoid thus timing, we don't publish the + // TransactionContext until after we've executed all cached TransactionOperations. + TransactionContext localTransactionContext; + if(failure != null) { + LOG.debug("Tx {} Creating NoOpTransaction because of error: {}", identifier, + failure.getMessage()); + + localTransactionContext = new NoOpTransactionContext(failure, identifier, operationLimiter); + } else if (response.getClass().equals(CreateTransactionReply.SERIALIZABLE_CLASS)) { + localTransactionContext = createValidTransactionContext( + CreateTransactionReply.fromSerializable(response)); + } else { + IllegalArgumentException exception = new IllegalArgumentException(String.format( + "Invalid reply type %s for CreateTransaction", response.getClass())); + + localTransactionContext = new NoOpTransactionContext(exception, identifier, operationLimiter); + } - private ActorRef cohort; + for(TransactionOperation oper: txOperationsOnComplete) { + oper.invoke(localTransactionContext); + } - public NoOpTransactionContext(String shardName){ - this.shardName = shardName; - } - @Override public String getShardName() { - return shardName; + txOperationsOnComplete.clear(); + // We're done invoking the TransactionOperations so we can now publish the + // TransactionContext. + transactionContext = localTransactionContext; + } } - @Override public String getResolvedCohortPath(String cohortPath) { - return cohort.path().toString(); - } + private TransactionContext createValidTransactionContext(CreateTransactionReply reply) { + String transactionPath = reply.getTransactionPath(); - @Override public void closeTransaction() { - LOG.error("closeTransaction called"); - } + LOG.debug("Tx {} Received transaction actor path {}", identifier, transactionPath); - @Override public Object readyTransaction() { - LOG.error("readyTransaction called"); - cohort = actorContext.getActorSystem().actorOf(Props.create(NoOpCohort.class)); - return new ReadyTransactionReply(cohort.path()).toSerializable(); - } + ActorSelection transactionActor = actorContext.actorSelection(transactionPath); - @Override public void deleteData(YangInstanceIdentifier path) { - LOG.error("deleteData called path = {}", path); - } + if (transactionType == TransactionType.READ_ONLY) { + // Add the actor to the remoteTransactionActors list for access by the + // cleanup PhantonReference. + remoteTransactionActors.add(transactionActor); - @Override public void mergeData(YangInstanceIdentifier path, - NormalizedNode data) { - LOG.error("mergeData called path = {}", path); - } + // Write to the memory barrier volatile to publish the above update to the + // remoteTransactionActors list for thread visibility. + remoteTransactionActorsMB.set(true); + } - @Override - public CheckedFuture>, ReadFailedException> readData( - YangInstanceIdentifier path) { - LOG.error("readData called path = {}", path); - return Futures.immediateCheckedFuture( - Optional.>absent()); - } + // TxActor is always created where the leader of the shard is. + // Check if TxActor is created in the same node + boolean isTxActorLocal = actorContext.isPathLocal(transactionPath); - @Override public void writeData(YangInstanceIdentifier path, - NormalizedNode data) { - LOG.error("writeData called path = {}", path); + return new TransactionContextImpl(transactionPath, transactionActor, identifier, + actorContext, schemaContext, isTxActorLocal, reply.getVersion(), operationCompleter); } } - - - }