2 * Copyright (c) 2014 Cisco Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
9 package org.opendaylight.controller.cluster.datastore;
11 import akka.actor.ActorSelection;
12 import akka.dispatch.Mapper;
13 import akka.dispatch.OnComplete;
14 import com.google.common.annotations.VisibleForTesting;
15 import com.google.common.base.FinalizablePhantomReference;
16 import com.google.common.base.FinalizableReferenceQueue;
17 import com.google.common.base.Optional;
18 import com.google.common.base.Preconditions;
19 import com.google.common.collect.Lists;
20 import com.google.common.util.concurrent.CheckedFuture;
21 import com.google.common.util.concurrent.SettableFuture;
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.HashMap;
25 import java.util.List;
27 import java.util.concurrent.ConcurrentHashMap;
28 import java.util.concurrent.Semaphore;
29 import java.util.concurrent.TimeUnit;
30 import java.util.concurrent.atomic.AtomicBoolean;
31 import java.util.concurrent.atomic.AtomicLong;
32 import javax.annotation.concurrent.GuardedBy;
33 import org.opendaylight.controller.cluster.datastore.exceptions.NoShardLeaderException;
34 import org.opendaylight.controller.cluster.datastore.identifiers.TransactionIdentifier;
35 import org.opendaylight.controller.cluster.datastore.messages.CloseTransaction;
36 import org.opendaylight.controller.cluster.datastore.messages.CreateTransaction;
37 import org.opendaylight.controller.cluster.datastore.messages.CreateTransactionReply;
38 import org.opendaylight.controller.cluster.datastore.shardstrategy.ShardStrategyFactory;
39 import org.opendaylight.controller.cluster.datastore.utils.ActorContext;
40 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
41 import org.opendaylight.controller.sal.core.spi.data.DOMStoreReadWriteTransaction;
42 import org.opendaylight.controller.sal.core.spi.data.DOMStoreThreePhaseCommitCohort;
43 import org.opendaylight.yangtools.util.concurrent.MappingCheckedFuture;
44 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
45 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
46 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
47 import org.slf4j.Logger;
48 import org.slf4j.LoggerFactory;
49 import scala.concurrent.Future;
50 import scala.concurrent.Promise;
51 import scala.concurrent.duration.FiniteDuration;
54 * TransactionProxy acts as a proxy for one or more transactions that were created on a remote shard
56 * Creating a transaction on the consumer side will create one instance of a transaction proxy. If during
57 * the transaction reads and writes are done on data that belongs to different shards then a separate transaction will
58 * be created on each of those shards by the TransactionProxy
61 * The TransactionProxy does not make any guarantees about atomicity or order in which the transactions on the various
62 * shards will be executed.
65 public class TransactionProxy implements DOMStoreReadWriteTransaction {
67 public static enum TransactionType {
73 static final Mapper<Throwable, Throwable> SAME_FAILURE_TRANSFORMER =
74 new Mapper<Throwable, Throwable>() {
76 public Throwable apply(Throwable failure) {
81 private static final AtomicLong counter = new AtomicLong();
83 private static final Logger LOG = LoggerFactory.getLogger(TransactionProxy.class);
86 * Time interval in between transaction create retries.
88 private static final FiniteDuration CREATE_TX_TRY_INTERVAL =
89 FiniteDuration.create(1, TimeUnit.SECONDS);
92 * Used to enqueue the PhantomReferences for read-only TransactionProxy instances. The
93 * FinalizableReferenceQueue is safe to use statically in an OSGi environment as it uses some
94 * trickery to clean up its internal thread when the bundle is unloaded.
96 private static final FinalizableReferenceQueue phantomReferenceQueue =
97 new FinalizableReferenceQueue();
100 * This stores the TransactionProxyCleanupPhantomReference instances statically, This is
101 * necessary because PhantomReferences need a hard reference so they're not garbage collected.
102 * Once finalized, the TransactionProxyCleanupPhantomReference removes itself from this map
103 * and thus becomes eligible for garbage collection.
105 private static final Map<TransactionProxyCleanupPhantomReference,
106 TransactionProxyCleanupPhantomReference> phantomReferenceCache =
107 new ConcurrentHashMap<>();
110 * A PhantomReference that closes remote transactions for a TransactionProxy when it's
111 * garbage collected. This is used for read-only transactions as they're not explicitly closed
112 * by clients. So the only way to detect that a transaction is no longer in use and it's safe
113 * to clean up is when it's garbage collected. It's inexact as to when an instance will be GC'ed
114 * but TransactionProxy instances should generally be short-lived enough to avoid being moved
115 * to the old generation space and thus should be cleaned up in a timely manner as the GC
116 * runs on the young generation (eden, swap1...) space much more frequently.
118 private static class TransactionProxyCleanupPhantomReference
119 extends FinalizablePhantomReference<TransactionProxy> {
121 private final List<ActorSelection> remoteTransactionActors;
122 private final AtomicBoolean remoteTransactionActorsMB;
123 private final ActorContext actorContext;
124 private final TransactionIdentifier identifier;
126 protected TransactionProxyCleanupPhantomReference(TransactionProxy referent) {
127 super(referent, phantomReferenceQueue);
129 // Note we need to cache the relevant fields from the TransactionProxy as we can't
130 // have a hard reference to the TransactionProxy instance itself.
132 remoteTransactionActors = referent.remoteTransactionActors;
133 remoteTransactionActorsMB = referent.remoteTransactionActorsMB;
134 actorContext = referent.actorContext;
135 identifier = referent.identifier;
139 public void finalizeReferent() {
140 LOG.trace("Cleaning up {} Tx actors for TransactionProxy {}",
141 remoteTransactionActors.size(), identifier);
143 phantomReferenceCache.remove(this);
145 // Access the memory barrier volatile to ensure all previous updates to the
146 // remoteTransactionActors list are visible to this thread.
148 if(remoteTransactionActorsMB.get()) {
149 for(ActorSelection actor : remoteTransactionActors) {
150 LOG.trace("Sending CloseTransaction to {}", actor);
151 actorContext.sendOperationAsync(actor, CloseTransaction.INSTANCE.toSerializable());
158 * Stores the remote Tx actors for each requested data store path to be used by the
159 * PhantomReference to close the remote Tx's. This is only used for read-only Tx's. The
160 * remoteTransactionActorsMB volatile serves as a memory barrier to publish updates to the
161 * remoteTransactionActors list so they will be visible to the thread accessing the
164 private List<ActorSelection> remoteTransactionActors;
165 private AtomicBoolean remoteTransactionActorsMB;
168 * Stores the create transaction results per shard.
170 private final Map<String, TransactionFutureCallback> txFutureCallbackMap = new HashMap<>();
172 private final TransactionType transactionType;
173 private final ActorContext actorContext;
174 private final TransactionIdentifier identifier;
175 private final String transactionChainId;
176 private final SchemaContext schemaContext;
177 private boolean inReadyState;
178 private final Semaphore operationLimiter;
179 private final OperationCompleter operationCompleter;
181 public TransactionProxy(ActorContext actorContext, TransactionType transactionType) {
182 this(actorContext, transactionType, "");
185 public TransactionProxy(ActorContext actorContext, TransactionType transactionType,
186 String transactionChainId) {
187 this.actorContext = Preconditions.checkNotNull(actorContext,
188 "actorContext should not be null");
189 this.transactionType = Preconditions.checkNotNull(transactionType,
190 "transactionType should not be null");
191 this.schemaContext = Preconditions.checkNotNull(actorContext.getSchemaContext(),
192 "schemaContext should not be null");
193 this.transactionChainId = transactionChainId;
195 String memberName = actorContext.getCurrentMemberName();
196 if(memberName == null){
197 memberName = "UNKNOWN-MEMBER";
200 this.identifier = TransactionIdentifier.builder().memberName(memberName).counter(
201 counter.getAndIncrement()).build();
203 if(transactionType == TransactionType.READ_ONLY) {
204 // Read-only Tx's aren't explicitly closed by the client so we create a PhantomReference
205 // to close the remote Tx's when this instance is no longer in use and is garbage
208 remoteTransactionActors = Lists.newArrayList();
209 remoteTransactionActorsMB = new AtomicBoolean();
211 TransactionProxyCleanupPhantomReference cleanup =
212 new TransactionProxyCleanupPhantomReference(this);
213 phantomReferenceCache.put(cleanup, cleanup);
216 // Note : Currently mailbox-capacity comes from akka.conf and not from the config-subsystem
217 this.operationLimiter = new Semaphore(actorContext.getTransactionOutstandingOperationLimit());
218 this.operationCompleter = new OperationCompleter(operationLimiter);
220 LOG.debug("Created txn {} of type {} on chain {}", identifier, transactionType, transactionChainId);
224 List<Future<Object>> getRecordedOperationFutures() {
225 List<Future<Object>> recordedOperationFutures = Lists.newArrayList();
226 for(TransactionFutureCallback txFutureCallback : txFutureCallbackMap.values()) {
227 TransactionContext transactionContext = txFutureCallback.getTransactionContext();
228 if(transactionContext != null) {
229 recordedOperationFutures.addAll(transactionContext.getRecordedOperationFutures());
233 return recordedOperationFutures;
237 boolean hasTransactionContext() {
238 for(TransactionFutureCallback txFutureCallback : txFutureCallbackMap.values()) {
239 TransactionContext transactionContext = txFutureCallback.getTransactionContext();
240 if(transactionContext != null) {
249 public CheckedFuture<Optional<NormalizedNode<?, ?>>, ReadFailedException> read(final YangInstanceIdentifier path) {
251 Preconditions.checkState(transactionType != TransactionType.WRITE_ONLY,
252 "Read operation on write-only transaction is not allowed");
254 LOG.debug("Tx {} read {}", identifier, path);
258 final SettableFuture<Optional<NormalizedNode<?, ?>>> proxyFuture = SettableFuture.create();
260 TransactionFutureCallback txFutureCallback = getOrCreateTxFutureCallback(path);
261 txFutureCallback.enqueueTransactionOperation(new TransactionOperation() {
263 public void invoke(TransactionContext transactionContext) {
264 transactionContext.readData(path, proxyFuture);
268 return MappingCheckedFuture.create(proxyFuture, ReadFailedException.MAPPER);
272 public CheckedFuture<Boolean, ReadFailedException> exists(final YangInstanceIdentifier path) {
274 Preconditions.checkState(transactionType != TransactionType.WRITE_ONLY,
275 "Exists operation on write-only transaction is not allowed");
277 LOG.debug("Tx {} exists {}", identifier, path);
281 final SettableFuture<Boolean> proxyFuture = SettableFuture.create();
283 TransactionFutureCallback txFutureCallback = getOrCreateTxFutureCallback(path);
284 txFutureCallback.enqueueTransactionOperation(new TransactionOperation() {
286 public void invoke(TransactionContext transactionContext) {
287 transactionContext.dataExists(path, proxyFuture);
291 return MappingCheckedFuture.create(proxyFuture, ReadFailedException.MAPPER);
294 private void checkModificationState() {
295 Preconditions.checkState(transactionType != TransactionType.READ_ONLY,
296 "Modification operation on read-only transaction is not allowed");
297 Preconditions.checkState(!inReadyState,
298 "Transaction is sealed - further modifications are not allowed");
301 private void throttleOperation() {
302 throttleOperation(1);
305 private void throttleOperation(int acquirePermits) {
307 if(!operationLimiter.tryAcquire(acquirePermits, actorContext.getDatastoreContext().getOperationTimeoutInSeconds(), TimeUnit.SECONDS)){
308 LOG.warn("Failed to acquire operation permit for transaction {}", getIdentifier());
310 } catch (InterruptedException e) {
311 if(LOG.isDebugEnabled()) {
312 LOG.debug("Interrupted when trying to acquire operation permit for transaction " + getIdentifier().toString(), e);
314 LOG.warn("Interrupted when trying to acquire operation permit for transaction {}", getIdentifier());
321 public void write(final YangInstanceIdentifier path, final NormalizedNode<?, ?> data) {
323 checkModificationState();
325 LOG.debug("Tx {} write {}", identifier, path);
329 TransactionFutureCallback txFutureCallback = getOrCreateTxFutureCallback(path);
330 txFutureCallback.enqueueTransactionOperation(new TransactionOperation() {
332 public void invoke(TransactionContext transactionContext) {
333 transactionContext.writeData(path, data);
339 public void merge(final YangInstanceIdentifier path, final NormalizedNode<?, ?> data) {
341 checkModificationState();
343 LOG.debug("Tx {} merge {}", identifier, path);
347 TransactionFutureCallback txFutureCallback = getOrCreateTxFutureCallback(path);
348 txFutureCallback.enqueueTransactionOperation(new TransactionOperation() {
350 public void invoke(TransactionContext transactionContext) {
351 transactionContext.mergeData(path, data);
357 public void delete(final YangInstanceIdentifier path) {
359 checkModificationState();
361 LOG.debug("Tx {} delete {}", identifier, path);
365 TransactionFutureCallback txFutureCallback = getOrCreateTxFutureCallback(path);
366 txFutureCallback.enqueueTransactionOperation(new TransactionOperation() {
368 public void invoke(TransactionContext transactionContext) {
369 transactionContext.deleteData(path);
375 public DOMStoreThreePhaseCommitCohort ready() {
377 checkModificationState();
379 throttleOperation(txFutureCallbackMap.size());
383 LOG.debug("Tx {} Readying {} transactions for commit", identifier,
384 txFutureCallbackMap.size());
386 List<Future<ActorSelection>> cohortFutures = Lists.newArrayList();
388 for(TransactionFutureCallback txFutureCallback : txFutureCallbackMap.values()) {
390 LOG.debug("Tx {} Readying transaction for shard {} chain {}", identifier,
391 txFutureCallback.getShardName(), transactionChainId);
393 final TransactionContext transactionContext = txFutureCallback.getTransactionContext();
394 final Future<ActorSelection> future;
395 if (transactionContext != null) {
396 // avoid the creation of a promise and a TransactionOperation
397 future = transactionContext.readyTransaction();
399 final Promise<ActorSelection> promise = akka.dispatch.Futures.promise();
400 txFutureCallback.enqueueTransactionOperation(new TransactionOperation() {
402 public void invoke(TransactionContext transactionContext) {
403 promise.completeWith(transactionContext.readyTransaction());
406 future = promise.future();
409 cohortFutures.add(future);
412 onTransactionReady(cohortFutures);
414 return new ThreePhaseCommitCohortProxy(actorContext, cohortFutures,
415 identifier.toString());
419 * Method for derived classes to be notified when the transaction has been readied.
421 * @param cohortFutures the cohort Futures for each shard transaction.
423 protected void onTransactionReady(List<Future<ActorSelection>> cohortFutures) {
427 * Method called to send a CreateTransaction message to a shard.
429 * @param shard the shard actor to send to
430 * @param serializedCreateMessage the serialized message to send
431 * @return the response Future
433 protected Future<Object> sendCreateTransaction(ActorSelection shard,
434 Object serializedCreateMessage) {
435 return actorContext.executeOperationAsync(shard, serializedCreateMessage);
439 public Object getIdentifier() {
440 return this.identifier;
444 public void close() {
445 for (TransactionFutureCallback txFutureCallback : txFutureCallbackMap.values()) {
446 txFutureCallback.enqueueTransactionOperation(new TransactionOperation() {
448 public void invoke(TransactionContext transactionContext) {
449 transactionContext.closeTransaction();
454 txFutureCallbackMap.clear();
456 if(transactionType == TransactionType.READ_ONLY) {
457 remoteTransactionActors.clear();
458 remoteTransactionActorsMB.set(true);
462 private String shardNameFromIdentifier(YangInstanceIdentifier path){
463 return ShardStrategyFactory.getStrategy(path).findShard(path);
466 private TransactionFutureCallback getOrCreateTxFutureCallback(YangInstanceIdentifier path) {
467 String shardName = shardNameFromIdentifier(path);
468 TransactionFutureCallback txFutureCallback = txFutureCallbackMap.get(shardName);
469 if(txFutureCallback == null) {
470 Future<ActorSelection> findPrimaryFuture = actorContext.findPrimaryShardAsync(shardName);
472 final TransactionFutureCallback newTxFutureCallback =
473 new TransactionFutureCallback(shardName);
475 txFutureCallback = newTxFutureCallback;
476 txFutureCallbackMap.put(shardName, txFutureCallback);
478 findPrimaryFuture.onComplete(new OnComplete<ActorSelection>() {
480 public void onComplete(Throwable failure, ActorSelection primaryShard) {
481 if(failure != null) {
482 newTxFutureCallback.onComplete(failure, null);
484 newTxFutureCallback.setPrimaryShard(primaryShard);
487 }, actorContext.getActorSystem().dispatcher());
490 return txFutureCallback;
493 public String getTransactionChainId() {
494 return transactionChainId;
497 protected ActorContext getActorContext() {
502 * Interfaces for transaction operations to be invoked later.
504 private static interface TransactionOperation {
505 void invoke(TransactionContext transactionContext);
509 * Implements a Future OnComplete callback for a CreateTransaction message. This class handles
510 * retries, up to a limit, if the shard doesn't have a leader yet. This is done by scheduling a
511 * retry task after a short delay.
513 * The end result from a completed CreateTransaction message is a TransactionContext that is
514 * used to perform transaction operations. Transaction operations that occur before the
515 * CreateTransaction completes are cache and executed once the CreateTransaction completes,
516 * successfully or not.
518 private class TransactionFutureCallback extends OnComplete<Object> {
521 * The list of transaction operations to execute once the CreateTransaction completes.
523 @GuardedBy("txOperationsOnComplete")
524 private final List<TransactionOperation> txOperationsOnComplete = Lists.newArrayList();
527 * The TransactionContext resulting from the CreateTransaction reply.
529 private volatile TransactionContext transactionContext;
532 * The target primary shard.
534 private volatile ActorSelection primaryShard;
536 private volatile int createTxTries = (int) (actorContext.getDatastoreContext().
537 getShardLeaderElectionTimeout().duration().toMillis() /
538 CREATE_TX_TRY_INTERVAL.toMillis());
540 private final String shardName;
542 TransactionFutureCallback(String shardName) {
543 this.shardName = shardName;
546 String getShardName() {
550 TransactionContext getTransactionContext() {
551 return transactionContext;
556 * Sets the target primary shard and initiates a CreateTransaction try.
558 void setPrimaryShard(ActorSelection primaryShard) {
559 LOG.debug("Tx {} Primary shard found - trying create transaction", identifier);
561 this.primaryShard = primaryShard;
562 tryCreateTransaction();
566 * Adds a TransactionOperation to be executed after the CreateTransaction completes.
568 void addTxOperationOnComplete(TransactionOperation operation) {
569 boolean invokeOperation = true;
570 synchronized(txOperationsOnComplete) {
571 if(transactionContext == null) {
572 LOG.debug("Tx {} Adding operation on complete {}", identifier);
574 invokeOperation = false;
575 txOperationsOnComplete.add(operation);
579 if(invokeOperation) {
580 operation.invoke(transactionContext);
584 void enqueueTransactionOperation(final TransactionOperation op) {
586 if (transactionContext != null) {
587 op.invoke(transactionContext);
589 // The shard Tx hasn't been created yet so add the Tx operation to the Tx Future
590 // callback to be executed after the Tx is created.
591 addTxOperationOnComplete(op);
596 * Performs a CreateTransaction try async.
598 private void tryCreateTransaction() {
599 Future<Object> createTxFuture = sendCreateTransaction(primaryShard,
600 new CreateTransaction(identifier.toString(),
601 TransactionProxy.this.transactionType.ordinal(),
602 getTransactionChainId()).toSerializable());
604 createTxFuture.onComplete(this, actorContext.getActorSystem().dispatcher());
608 public void onComplete(Throwable failure, Object response) {
609 if(failure instanceof NoShardLeaderException) {
610 // There's no leader for the shard yet - schedule and try again, unless we're out
611 // of retries. Note: createTxTries is volatile as it may be written by different
612 // threads however not concurrently, therefore decrementing it non-atomically here
614 if(--createTxTries > 0) {
615 LOG.debug("Tx {} Shard {} has no leader yet - scheduling create Tx retry",
616 identifier, shardName);
618 actorContext.getActorSystem().scheduler().scheduleOnce(CREATE_TX_TRY_INTERVAL,
622 tryCreateTransaction();
624 }, actorContext.getActorSystem().dispatcher());
629 // Create the TransactionContext from the response or failure. Store the new
630 // TransactionContext locally until we've completed invoking the
631 // TransactionOperations. This avoids thread timing issues which could cause
632 // out-of-order TransactionOperations. Eg, on a modification operation, if the
633 // TransactionContext is non-null, then we directly call the TransactionContext.
634 // However, at the same time, the code may be executing the cached
635 // TransactionOperations. So to avoid thus timing, we don't publish the
636 // TransactionContext until after we've executed all cached TransactionOperations.
637 TransactionContext localTransactionContext;
638 if(failure != null) {
639 LOG.debug("Tx {} Creating NoOpTransaction because of error: {}", identifier,
640 failure.getMessage());
642 localTransactionContext = new NoOpTransactionContext(failure, identifier, operationLimiter);
643 } else if (response.getClass().equals(CreateTransactionReply.SERIALIZABLE_CLASS)) {
644 localTransactionContext = createValidTransactionContext(
645 CreateTransactionReply.fromSerializable(response));
647 IllegalArgumentException exception = new IllegalArgumentException(String.format(
648 "Invalid reply type %s for CreateTransaction", response.getClass()));
650 localTransactionContext = new NoOpTransactionContext(exception, identifier, operationLimiter);
653 executeTxOperatonsOnComplete(localTransactionContext);
656 private void executeTxOperatonsOnComplete(TransactionContext localTransactionContext) {
658 // Access to txOperationsOnComplete and transactionContext must be protected and atomic
659 // (ie synchronized) with respect to #addTxOperationOnComplete to handle timing
660 // issues and ensure no TransactionOperation is missed and that they are processed
661 // in the order they occurred.
663 // We'll make a local copy of the txOperationsOnComplete list to handle re-entrancy
664 // in case a TransactionOperation results in another transaction operation being
665 // queued (eg a put operation from a client read Future callback that is notified
667 Collection<TransactionOperation> operationsBatch = null;
668 synchronized(txOperationsOnComplete) {
669 if(txOperationsOnComplete.isEmpty()) {
670 // We're done invoking the TransactionOperations so we can now publish the
671 // TransactionContext.
672 transactionContext = localTransactionContext;
676 operationsBatch = new ArrayList<>(txOperationsOnComplete);
677 txOperationsOnComplete.clear();
680 // Invoke TransactionOperations outside the sync block to avoid unnecessary blocking.
681 // A slight down-side is that we need to re-acquire the lock below but this should
683 for(TransactionOperation oper: operationsBatch) {
684 oper.invoke(localTransactionContext);
689 private TransactionContext createValidTransactionContext(CreateTransactionReply reply) {
690 String transactionPath = reply.getTransactionPath();
692 LOG.debug("Tx {} Received transaction actor path {}", identifier, transactionPath);
694 ActorSelection transactionActor = actorContext.actorSelection(transactionPath);
696 if (transactionType == TransactionType.READ_ONLY) {
697 // Add the actor to the remoteTransactionActors list for access by the
698 // cleanup PhantonReference.
699 remoteTransactionActors.add(transactionActor);
701 // Write to the memory barrier volatile to publish the above update to the
702 // remoteTransactionActors list for thread visibility.
703 remoteTransactionActorsMB.set(true);
706 // TxActor is always created where the leader of the shard is.
707 // Check if TxActor is created in the same node
708 boolean isTxActorLocal = actorContext.isPathLocal(transactionPath);
710 return new TransactionContextImpl(transactionPath, transactionActor, identifier,
711 actorContext, schemaContext, isTxActorLocal, reply.getVersion(), operationCompleter);