2 * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
9 package org.opendaylight.controller.cluster.datastore;
11 import akka.actor.ActorSelection;
12 import akka.dispatch.Mapper;
13 import akka.dispatch.OnComplete;
14 import com.google.common.annotations.VisibleForTesting;
15 import com.google.common.base.FinalizablePhantomReference;
16 import com.google.common.base.FinalizableReferenceQueue;
17 import com.google.common.base.Optional;
18 import com.google.common.base.Preconditions;
19 import com.google.common.collect.Lists;
20 import com.google.common.util.concurrent.CheckedFuture;
21 import com.google.common.util.concurrent.SettableFuture;
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.HashMap;
25 import java.util.List;
27 import java.util.concurrent.ConcurrentHashMap;
28 import java.util.concurrent.Semaphore;
29 import java.util.concurrent.TimeUnit;
30 import java.util.concurrent.atomic.AtomicBoolean;
31 import java.util.concurrent.atomic.AtomicLong;
32 import javax.annotation.concurrent.GuardedBy;
33 import org.opendaylight.controller.cluster.datastore.exceptions.NoShardLeaderException;
34 import org.opendaylight.controller.cluster.datastore.identifiers.TransactionIdentifier;
35 import org.opendaylight.controller.cluster.datastore.messages.CloseTransaction;
36 import org.opendaylight.controller.cluster.datastore.messages.CreateTransaction;
37 import org.opendaylight.controller.cluster.datastore.messages.CreateTransactionReply;
38 import org.opendaylight.controller.cluster.datastore.shardstrategy.ShardStrategyFactory;
39 import org.opendaylight.controller.cluster.datastore.utils.ActorContext;
40 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
41 import org.opendaylight.controller.sal.core.spi.data.DOMStoreReadWriteTransaction;
42 import org.opendaylight.controller.sal.core.spi.data.DOMStoreThreePhaseCommitCohort;
43 import org.opendaylight.yangtools.util.concurrent.MappingCheckedFuture;
44 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
45 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
46 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
47 import org.slf4j.Logger;
48 import org.slf4j.LoggerFactory;
49 import scala.concurrent.Future;
50 import scala.concurrent.Promise;
51 import scala.concurrent.duration.FiniteDuration;
54 * TransactionProxy acts as a proxy for one or more transactions that were created on a remote shard
56 * Creating a transaction on the consumer side will create one instance of a transaction proxy. If during
57 * the transaction reads and writes are done on data that belongs to different shards then a separate transaction will
58 * be created on each of those shards by the TransactionProxy
61 * The TransactionProxy does not make any guarantees about atomicity or order in which the transactions on the various
62 * shards will be executed.
65 public class TransactionProxy implements DOMStoreReadWriteTransaction {
67 public static enum TransactionType {
73 static final Mapper<Throwable, Throwable> SAME_FAILURE_TRANSFORMER =
74 new Mapper<Throwable, Throwable>() {
76 public Throwable apply(Throwable failure) {
81 private static final AtomicLong counter = new AtomicLong();
83 private static final Logger LOG = LoggerFactory.getLogger(TransactionProxy.class);
86 * Time interval in between transaction create retries.
88 private static final FiniteDuration CREATE_TX_TRY_INTERVAL =
89 FiniteDuration.create(1, TimeUnit.SECONDS);
92 * Used to enqueue the PhantomReferences for read-only TransactionProxy instances. The
93 * FinalizableReferenceQueue is safe to use statically in an OSGi environment as it uses some
94 * trickery to clean up its internal thread when the bundle is unloaded.
96 private static final FinalizableReferenceQueue phantomReferenceQueue =
97 new FinalizableReferenceQueue();
100 * This stores the TransactionProxyCleanupPhantomReference instances statically, This is
101 * necessary because PhantomReferences need a hard reference so they're not garbage collected.
102 * Once finalized, the TransactionProxyCleanupPhantomReference removes itself from this map
103 * and thus becomes eligible for garbage collection.
105 private static final Map<TransactionProxyCleanupPhantomReference,
106 TransactionProxyCleanupPhantomReference> phantomReferenceCache =
107 new ConcurrentHashMap<>();
110 * A PhantomReference that closes remote transactions for a TransactionProxy when it's
111 * garbage collected. This is used for read-only transactions as they're not explicitly closed
112 * by clients. So the only way to detect that a transaction is no longer in use and it's safe
113 * to clean up is when it's garbage collected. It's inexact as to when an instance will be GC'ed
114 * but TransactionProxy instances should generally be short-lived enough to avoid being moved
115 * to the old generation space and thus should be cleaned up in a timely manner as the GC
116 * runs on the young generation (eden, swap1...) space much more frequently.
118 private static class TransactionProxyCleanupPhantomReference
119 extends FinalizablePhantomReference<TransactionProxy> {
121 private final List<ActorSelection> remoteTransactionActors;
122 private final AtomicBoolean remoteTransactionActorsMB;
123 private final ActorContext actorContext;
124 private final TransactionIdentifier identifier;
126 protected TransactionProxyCleanupPhantomReference(TransactionProxy referent) {
127 super(referent, phantomReferenceQueue);
129 // Note we need to cache the relevant fields from the TransactionProxy as we can't
130 // have a hard reference to the TransactionProxy instance itself.
132 remoteTransactionActors = referent.remoteTransactionActors;
133 remoteTransactionActorsMB = referent.remoteTransactionActorsMB;
134 actorContext = referent.actorContext;
135 identifier = referent.identifier;
139 public void finalizeReferent() {
140 LOG.trace("Cleaning up {} Tx actors for TransactionProxy {}",
141 remoteTransactionActors.size(), identifier);
143 phantomReferenceCache.remove(this);
145 // Access the memory barrier volatile to ensure all previous updates to the
146 // remoteTransactionActors list are visible to this thread.
148 if(remoteTransactionActorsMB.get()) {
149 for(ActorSelection actor : remoteTransactionActors) {
150 LOG.trace("Sending CloseTransaction to {}", actor);
151 actorContext.sendOperationAsync(actor, CloseTransaction.INSTANCE.toSerializable());
158 * Stores the remote Tx actors for each requested data store path to be used by the
159 * PhantomReference to close the remote Tx's. This is only used for read-only Tx's. The
160 * remoteTransactionActorsMB volatile serves as a memory barrier to publish updates to the
161 * remoteTransactionActors list so they will be visible to the thread accessing the
164 private List<ActorSelection> remoteTransactionActors;
165 private AtomicBoolean remoteTransactionActorsMB;
168 * Stores the create transaction results per shard.
170 private final Map<String, TransactionFutureCallback> txFutureCallbackMap = new HashMap<>();
172 private final TransactionType transactionType;
173 private final ActorContext actorContext;
174 private final TransactionIdentifier identifier;
175 private final String transactionChainId;
176 private final SchemaContext schemaContext;
177 private boolean inReadyState;
178 private final Semaphore operationLimiter;
179 private final OperationCompleter operationCompleter;
181 public TransactionProxy(ActorContext actorContext, TransactionType transactionType) {
182 this(actorContext, transactionType, "");
185 public TransactionProxy(ActorContext actorContext, TransactionType transactionType,
186 String transactionChainId) {
187 this.actorContext = Preconditions.checkNotNull(actorContext,
188 "actorContext should not be null");
189 this.transactionType = Preconditions.checkNotNull(transactionType,
190 "transactionType should not be null");
191 this.schemaContext = Preconditions.checkNotNull(actorContext.getSchemaContext(),
192 "schemaContext should not be null");
193 this.transactionChainId = transactionChainId;
195 String memberName = actorContext.getCurrentMemberName();
196 if(memberName == null){
197 memberName = "UNKNOWN-MEMBER";
200 this.identifier = TransactionIdentifier.builder().memberName(memberName).counter(
201 counter.getAndIncrement()).build();
203 if(transactionType == TransactionType.READ_ONLY) {
204 // Read-only Tx's aren't explicitly closed by the client so we create a PhantomReference
205 // to close the remote Tx's when this instance is no longer in use and is garbage
208 remoteTransactionActors = Lists.newArrayList();
209 remoteTransactionActorsMB = new AtomicBoolean();
211 TransactionProxyCleanupPhantomReference cleanup =
212 new TransactionProxyCleanupPhantomReference(this);
213 phantomReferenceCache.put(cleanup, cleanup);
216 // Note : Currently mailbox-capacity comes from akka.conf and not from the config-subsystem
217 this.operationLimiter = new Semaphore(actorContext.getTransactionOutstandingOperationLimit());
218 this.operationCompleter = new OperationCompleter(operationLimiter);
220 LOG.debug("Created txn {} of type {} on chain {}", identifier, transactionType, transactionChainId);
224 List<Future<Object>> getRecordedOperationFutures() {
225 List<Future<Object>> recordedOperationFutures = Lists.newArrayList();
226 for(TransactionFutureCallback txFutureCallback : txFutureCallbackMap.values()) {
227 TransactionContext transactionContext = txFutureCallback.getTransactionContext();
228 if(transactionContext != null) {
229 recordedOperationFutures.addAll(transactionContext.getRecordedOperationFutures());
233 return recordedOperationFutures;
237 boolean hasTransactionContext() {
238 for(TransactionFutureCallback txFutureCallback : txFutureCallbackMap.values()) {
239 TransactionContext transactionContext = txFutureCallback.getTransactionContext();
240 if(transactionContext != null) {
249 public CheckedFuture<Optional<NormalizedNode<?, ?>>, ReadFailedException> read(final YangInstanceIdentifier path) {
251 Preconditions.checkState(transactionType != TransactionType.WRITE_ONLY,
252 "Read operation on write-only transaction is not allowed");
254 LOG.debug("Tx {} read {}", identifier, path);
258 final SettableFuture<Optional<NormalizedNode<?, ?>>> proxyFuture = SettableFuture.create();
260 TransactionFutureCallback txFutureCallback = getOrCreateTxFutureCallback(path);
261 txFutureCallback.enqueueTransactionOperation(new TransactionOperation() {
263 public void invoke(TransactionContext transactionContext) {
264 transactionContext.readData(path, proxyFuture);
268 return MappingCheckedFuture.create(proxyFuture, ReadFailedException.MAPPER);
272 public CheckedFuture<Boolean, ReadFailedException> exists(final YangInstanceIdentifier path) {
274 Preconditions.checkState(transactionType != TransactionType.WRITE_ONLY,
275 "Exists operation on write-only transaction is not allowed");
277 LOG.debug("Tx {} exists {}", identifier, path);
281 final SettableFuture<Boolean> proxyFuture = SettableFuture.create();
283 TransactionFutureCallback txFutureCallback = getOrCreateTxFutureCallback(path);
284 txFutureCallback.enqueueTransactionOperation(new TransactionOperation() {
286 public void invoke(TransactionContext transactionContext) {
287 transactionContext.dataExists(path, proxyFuture);
291 return MappingCheckedFuture.create(proxyFuture, ReadFailedException.MAPPER);
294 private void checkModificationState() {
295 Preconditions.checkState(transactionType != TransactionType.READ_ONLY,
296 "Modification operation on read-only transaction is not allowed");
297 Preconditions.checkState(!inReadyState,
298 "Transaction is sealed - further modifications are not allowed");
301 private void throttleOperation() {
302 throttleOperation(1);
305 private void throttleOperation(int acquirePermits) {
307 if(!operationLimiter.tryAcquire(acquirePermits,
308 actorContext.getDatastoreContext().getOperationTimeoutInSeconds(), TimeUnit.SECONDS)){
309 LOG.warn("Failed to acquire operation permit for transaction {}", getIdentifier());
311 } catch (InterruptedException e) {
312 if(LOG.isDebugEnabled()) {
313 LOG.debug("Interrupted when trying to acquire operation permit for transaction " + getIdentifier().toString(), e);
315 LOG.warn("Interrupted when trying to acquire operation permit for transaction {}", getIdentifier());
322 public void write(final YangInstanceIdentifier path, final NormalizedNode<?, ?> data) {
324 checkModificationState();
326 LOG.debug("Tx {} write {}", identifier, path);
330 TransactionFutureCallback txFutureCallback = getOrCreateTxFutureCallback(path);
331 txFutureCallback.enqueueTransactionOperation(new TransactionOperation() {
333 public void invoke(TransactionContext transactionContext) {
334 transactionContext.writeData(path, data);
340 public void merge(final YangInstanceIdentifier path, final NormalizedNode<?, ?> data) {
342 checkModificationState();
344 LOG.debug("Tx {} merge {}", identifier, path);
348 TransactionFutureCallback txFutureCallback = getOrCreateTxFutureCallback(path);
349 txFutureCallback.enqueueTransactionOperation(new TransactionOperation() {
351 public void invoke(TransactionContext transactionContext) {
352 transactionContext.mergeData(path, data);
358 public void delete(final YangInstanceIdentifier path) {
360 checkModificationState();
362 LOG.debug("Tx {} delete {}", identifier, path);
366 TransactionFutureCallback txFutureCallback = getOrCreateTxFutureCallback(path);
367 txFutureCallback.enqueueTransactionOperation(new TransactionOperation() {
369 public void invoke(TransactionContext transactionContext) {
370 transactionContext.deleteData(path);
376 public DOMStoreThreePhaseCommitCohort ready() {
378 checkModificationState();
380 throttleOperation(txFutureCallbackMap.size());
384 LOG.debug("Tx {} Readying {} transactions for commit", identifier,
385 txFutureCallbackMap.size());
387 List<Future<ActorSelection>> cohortFutures = Lists.newArrayList();
389 for(TransactionFutureCallback txFutureCallback : txFutureCallbackMap.values()) {
391 LOG.debug("Tx {} Readying transaction for shard {} chain {}", identifier,
392 txFutureCallback.getShardName(), transactionChainId);
394 final TransactionContext transactionContext = txFutureCallback.getTransactionContext();
395 final Future<ActorSelection> future;
396 if (transactionContext != null) {
397 // avoid the creation of a promise and a TransactionOperation
398 future = transactionContext.readyTransaction();
400 final Promise<ActorSelection> promise = akka.dispatch.Futures.promise();
401 txFutureCallback.enqueueTransactionOperation(new TransactionOperation() {
403 public void invoke(TransactionContext transactionContext) {
404 promise.completeWith(transactionContext.readyTransaction());
407 future = promise.future();
410 cohortFutures.add(future);
413 onTransactionReady(cohortFutures);
415 return new ThreePhaseCommitCohortProxy(actorContext, cohortFutures,
416 identifier.toString());
420 * Method for derived classes to be notified when the transaction has been readied.
422 * @param cohortFutures the cohort Futures for each shard transaction.
424 protected void onTransactionReady(List<Future<ActorSelection>> cohortFutures) {
428 * Method called to send a CreateTransaction message to a shard.
430 * @param shard the shard actor to send to
431 * @param serializedCreateMessage the serialized message to send
432 * @return the response Future
434 protected Future<Object> sendCreateTransaction(ActorSelection shard,
435 Object serializedCreateMessage) {
436 return actorContext.executeOperationAsync(shard, serializedCreateMessage);
440 public Object getIdentifier() {
441 return this.identifier;
445 public void close() {
446 for (TransactionFutureCallback txFutureCallback : txFutureCallbackMap.values()) {
447 txFutureCallback.enqueueTransactionOperation(new TransactionOperation() {
449 public void invoke(TransactionContext transactionContext) {
450 transactionContext.closeTransaction();
455 txFutureCallbackMap.clear();
457 if(transactionType == TransactionType.READ_ONLY) {
458 remoteTransactionActors.clear();
459 remoteTransactionActorsMB.set(true);
463 private String shardNameFromIdentifier(YangInstanceIdentifier path){
464 return ShardStrategyFactory.getStrategy(path).findShard(path);
467 private TransactionFutureCallback getOrCreateTxFutureCallback(YangInstanceIdentifier path) {
468 String shardName = shardNameFromIdentifier(path);
469 TransactionFutureCallback txFutureCallback = txFutureCallbackMap.get(shardName);
470 if(txFutureCallback == null) {
471 Future<ActorSelection> findPrimaryFuture = actorContext.findPrimaryShardAsync(shardName);
473 final TransactionFutureCallback newTxFutureCallback =
474 new TransactionFutureCallback(shardName);
476 txFutureCallback = newTxFutureCallback;
477 txFutureCallbackMap.put(shardName, txFutureCallback);
479 findPrimaryFuture.onComplete(new OnComplete<ActorSelection>() {
481 public void onComplete(Throwable failure, ActorSelection primaryShard) {
482 if(failure != null) {
483 newTxFutureCallback.onComplete(failure, null);
485 newTxFutureCallback.setPrimaryShard(primaryShard);
488 }, actorContext.getClientDispatcher());
491 return txFutureCallback;
494 public String getTransactionChainId() {
495 return transactionChainId;
498 protected ActorContext getActorContext() {
503 * Interfaces for transaction operations to be invoked later.
505 private static interface TransactionOperation {
506 void invoke(TransactionContext transactionContext);
510 * Implements a Future OnComplete callback for a CreateTransaction message. This class handles
511 * retries, up to a limit, if the shard doesn't have a leader yet. This is done by scheduling a
512 * retry task after a short delay.
514 * The end result from a completed CreateTransaction message is a TransactionContext that is
515 * used to perform transaction operations. Transaction operations that occur before the
516 * CreateTransaction completes are cache and executed once the CreateTransaction completes,
517 * successfully or not.
519 private class TransactionFutureCallback extends OnComplete<Object> {
522 * The list of transaction operations to execute once the CreateTransaction completes.
524 @GuardedBy("txOperationsOnComplete")
525 private final List<TransactionOperation> txOperationsOnComplete = Lists.newArrayList();
528 * The TransactionContext resulting from the CreateTransaction reply.
530 private volatile TransactionContext transactionContext;
533 * The target primary shard.
535 private volatile ActorSelection primaryShard;
537 private volatile int createTxTries = (int) (actorContext.getDatastoreContext().
538 getShardLeaderElectionTimeout().duration().toMillis() /
539 CREATE_TX_TRY_INTERVAL.toMillis());
541 private final String shardName;
543 TransactionFutureCallback(String shardName) {
544 this.shardName = shardName;
547 String getShardName() {
551 TransactionContext getTransactionContext() {
552 return transactionContext;
557 * Sets the target primary shard and initiates a CreateTransaction try.
559 void setPrimaryShard(ActorSelection primaryShard) {
560 LOG.debug("Tx {} Primary shard found - trying create transaction", identifier);
562 this.primaryShard = primaryShard;
563 tryCreateTransaction();
567 * Adds a TransactionOperation to be executed after the CreateTransaction completes.
569 void addTxOperationOnComplete(TransactionOperation operation) {
570 boolean invokeOperation = true;
571 synchronized(txOperationsOnComplete) {
572 if(transactionContext == null) {
573 LOG.debug("Tx {} Adding operation on complete {}", identifier);
575 invokeOperation = false;
576 txOperationsOnComplete.add(operation);
580 if(invokeOperation) {
581 operation.invoke(transactionContext);
585 void enqueueTransactionOperation(final TransactionOperation op) {
587 if (transactionContext != null) {
588 op.invoke(transactionContext);
590 // The shard Tx hasn't been created yet so add the Tx operation to the Tx Future
591 // callback to be executed after the Tx is created.
592 addTxOperationOnComplete(op);
597 * Performs a CreateTransaction try async.
599 private void tryCreateTransaction() {
600 Future<Object> createTxFuture = sendCreateTransaction(primaryShard,
601 new CreateTransaction(identifier.toString(),
602 TransactionProxy.this.transactionType.ordinal(),
603 getTransactionChainId()).toSerializable());
605 createTxFuture.onComplete(this, actorContext.getClientDispatcher());
609 public void onComplete(Throwable failure, Object response) {
610 if(failure instanceof NoShardLeaderException) {
611 // There's no leader for the shard yet - schedule and try again, unless we're out
612 // of retries. Note: createTxTries is volatile as it may be written by different
613 // threads however not concurrently, therefore decrementing it non-atomically here
615 if(--createTxTries > 0) {
616 LOG.debug("Tx {} Shard {} has no leader yet - scheduling create Tx retry",
617 identifier, shardName);
619 actorContext.getActorSystem().scheduler().scheduleOnce(CREATE_TX_TRY_INTERVAL,
623 tryCreateTransaction();
625 }, actorContext.getClientDispatcher());
630 // Create the TransactionContext from the response or failure. Store the new
631 // TransactionContext locally until we've completed invoking the
632 // TransactionOperations. This avoids thread timing issues which could cause
633 // out-of-order TransactionOperations. Eg, on a modification operation, if the
634 // TransactionContext is non-null, then we directly call the TransactionContext.
635 // However, at the same time, the code may be executing the cached
636 // TransactionOperations. So to avoid thus timing, we don't publish the
637 // TransactionContext until after we've executed all cached TransactionOperations.
638 TransactionContext localTransactionContext;
639 if(failure != null) {
640 LOG.debug("Tx {} Creating NoOpTransaction because of error: {}", identifier,
641 failure.getMessage());
643 localTransactionContext = new NoOpTransactionContext(failure, identifier, operationLimiter);
644 } else if (response.getClass().equals(CreateTransactionReply.SERIALIZABLE_CLASS)) {
645 localTransactionContext = createValidTransactionContext(
646 CreateTransactionReply.fromSerializable(response));
648 IllegalArgumentException exception = new IllegalArgumentException(String.format(
649 "Invalid reply type %s for CreateTransaction", response.getClass()));
651 localTransactionContext = new NoOpTransactionContext(exception, identifier, operationLimiter);
654 executeTxOperatonsOnComplete(localTransactionContext);
657 private void executeTxOperatonsOnComplete(TransactionContext localTransactionContext) {
659 // Access to txOperationsOnComplete and transactionContext must be protected and atomic
660 // (ie synchronized) with respect to #addTxOperationOnComplete to handle timing
661 // issues and ensure no TransactionOperation is missed and that they are processed
662 // in the order they occurred.
664 // We'll make a local copy of the txOperationsOnComplete list to handle re-entrancy
665 // in case a TransactionOperation results in another transaction operation being
666 // queued (eg a put operation from a client read Future callback that is notified
668 Collection<TransactionOperation> operationsBatch = null;
669 synchronized(txOperationsOnComplete) {
670 if(txOperationsOnComplete.isEmpty()) {
671 // We're done invoking the TransactionOperations so we can now publish the
672 // TransactionContext.
673 transactionContext = localTransactionContext;
677 operationsBatch = new ArrayList<>(txOperationsOnComplete);
678 txOperationsOnComplete.clear();
681 // Invoke TransactionOperations outside the sync block to avoid unnecessary blocking.
682 // A slight down-side is that we need to re-acquire the lock below but this should
684 for(TransactionOperation oper: operationsBatch) {
685 oper.invoke(localTransactionContext);
690 private TransactionContext createValidTransactionContext(CreateTransactionReply reply) {
691 String transactionPath = reply.getTransactionPath();
693 LOG.debug("Tx {} Received {}", identifier, reply);
695 ActorSelection transactionActor = actorContext.actorSelection(transactionPath);
697 if (transactionType == TransactionType.READ_ONLY) {
698 // Add the actor to the remoteTransactionActors list for access by the
699 // cleanup PhantonReference.
700 remoteTransactionActors.add(transactionActor);
702 // Write to the memory barrier volatile to publish the above update to the
703 // remoteTransactionActors list for thread visibility.
704 remoteTransactionActorsMB.set(true);
707 // TxActor is always created where the leader of the shard is.
708 // Check if TxActor is created in the same node
709 boolean isTxActorLocal = actorContext.isPathLocal(transactionPath);
711 if(reply.getVersion() >= DataStoreVersions.LITHIUM_VERSION) {
712 return new TransactionContextImpl(transactionPath, transactionActor, identifier,
713 actorContext, schemaContext, isTxActorLocal, reply.getVersion(), operationCompleter);
715 return new LegacyTransactionContextImpl(transactionPath, transactionActor, identifier,
716 actorContext, schemaContext, isTxActorLocal, reply.getVersion(), operationCompleter);