2 * Copyright (c) 2015 Brocade Communications Systems, Inc. and others. All rights reserved.
3 * Copyright (c) 2015 Cisco Systems, Inc. and others. All rights reserved.
5 * This program and the accompanying materials are made available under the
6 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
7 * and is available at http://www.eclipse.org/legal/epl-v10.html
9 package org.opendaylight.controller.cluster.datastore;
11 import akka.actor.ActorSelection;
12 import akka.dispatch.OnComplete;
13 import akka.pattern.AskTimeoutException;
14 import akka.util.Timeout;
15 import com.google.common.base.Preconditions;
16 import java.util.concurrent.TimeUnit;
17 import org.opendaylight.controller.cluster.datastore.exceptions.NoShardLeaderException;
18 import org.opendaylight.controller.cluster.datastore.exceptions.ShardLeaderNotRespondingException;
19 import org.opendaylight.controller.cluster.datastore.identifiers.TransactionIdentifier;
20 import org.opendaylight.controller.cluster.datastore.messages.CreateTransaction;
21 import org.opendaylight.controller.cluster.datastore.messages.CreateTransactionReply;
22 import org.opendaylight.controller.cluster.datastore.messages.PrimaryShardInfo;
23 import org.opendaylight.controller.cluster.datastore.utils.ActorContext;
24 import org.slf4j.Logger;
25 import org.slf4j.LoggerFactory;
26 import scala.concurrent.Future;
27 import scala.concurrent.duration.FiniteDuration;
30 * Handles creation of TransactionContext instances for remote transactions. This class creates
31 * remote transactions, if necessary, by sending CreateTransaction messages with retries, up to a limit,
32 * if the shard doesn't have a leader yet. This is done by scheduling a retry task after a short delay.
34 * The end result from a completed CreateTransaction message is a TransactionContext that is
35 * used to perform transaction operations. Transaction operations that occur before the
36 * CreateTransaction completes are cache via a TransactionContextWrapper and executed once the
37 * CreateTransaction completes, successfully or not.
39 final class RemoteTransactionContextSupport {
40 private static final Logger LOG = LoggerFactory.getLogger(RemoteTransactionContextSupport.class);
42 private static final long CREATE_TX_TRY_INTERVAL_IN_MS = 1000;
43 private static final long MAX_CREATE_TX_MSG_TIMEOUT_IN_MS = 5000;
45 private final TransactionProxy parent;
46 private final String shardName;
49 * The target primary shard.
51 private volatile ActorSelection primaryShard;
54 * The total timeout for creating a tx on the primary shard.
56 private volatile long totalCreateTxTimeout;
58 private final Timeout createTxMessageTimeout;
60 private final TransactionContextWrapper transactionContextWrapper;
62 RemoteTransactionContextSupport(final TransactionContextWrapper transactionContextWrapper, final TransactionProxy parent,
63 final String shardName) {
64 this.parent = Preconditions.checkNotNull(parent);
65 this.shardName = shardName;
66 this.transactionContextWrapper = transactionContextWrapper;
68 // For the total create tx timeout, use 2 times the election timeout. This should be enough time for
69 // a leader re-election to occur if we happen to hit it in transition.
70 totalCreateTxTimeout = parent.getActorContext().getDatastoreContext().getShardRaftConfig()
71 .getElectionTimeOutInterval().toMillis() * 2;
73 // We'll use the operationTimeout for the the create Tx message timeout so it can be set appropriately
74 // for unit tests but cap it at MAX_CREATE_TX_MSG_TIMEOUT_IN_MS. The operationTimeout could be set
75 // larger than the totalCreateTxTimeout in production which we don't want.
76 long operationTimeout = parent.getActorContext().getOperationTimeout().duration().toMillis();
77 createTxMessageTimeout = new Timeout(Math.min(operationTimeout, MAX_CREATE_TX_MSG_TIMEOUT_IN_MS),
78 TimeUnit.MILLISECONDS);
81 String getShardName() {
85 private TransactionType getTransactionType() {
86 return parent.getType();
89 private ActorContext getActorContext() {
90 return parent.getActorContext();
93 private TransactionIdentifier getIdentifier() {
94 return parent.getIdentifier();
98 * Sets the target primary shard and initiates a CreateTransaction try.
100 void setPrimaryShard(ActorSelection primaryShard, short primaryVersion) {
101 this.primaryShard = primaryShard;
103 if (getTransactionType() == TransactionType.WRITE_ONLY &&
104 getActorContext().getDatastoreContext().isWriteOnlyTransactionOptimizationsEnabled()) {
105 LOG.debug("Tx {} Primary shard {} found - creating WRITE_ONLY transaction context",
106 getIdentifier(), primaryShard);
108 // For write-only Tx's we prepare the transaction modifications directly on the shard actor
109 // to avoid the overhead of creating a separate transaction actor.
110 transactionContextWrapper.executePriorTransactionOperations(createValidTransactionContext(this.primaryShard,
111 this.primaryShard.path().toString(), primaryVersion));
113 tryCreateTransaction();
118 * Performs a CreateTransaction try async.
120 private void tryCreateTransaction() {
121 if(LOG.isDebugEnabled()) {
122 LOG.debug("Tx {} Primary shard {} found - trying create transaction", getIdentifier(), primaryShard);
125 Object serializedCreateMessage = new CreateTransaction(getIdentifier().toString(),
126 getTransactionType().ordinal(), getIdentifier().getChainId()).toSerializable();
128 Future<Object> createTxFuture = getActorContext().executeOperationAsync(primaryShard,
129 serializedCreateMessage, createTxMessageTimeout);
131 createTxFuture.onComplete(new OnComplete<Object>() {
133 public void onComplete(Throwable failure, Object response) {
134 onCreateTransactionComplete(failure, response);
136 }, getActorContext().getClientDispatcher());
139 private void tryFindPrimaryShard() {
140 LOG.debug("Tx {} Retrying findPrimaryShardAsync for shard {}", getIdentifier(), shardName);
142 this.primaryShard = null;
143 Future<PrimaryShardInfo> findPrimaryFuture = getActorContext().findPrimaryShardAsync(shardName);
144 findPrimaryFuture.onComplete(new OnComplete<PrimaryShardInfo>() {
146 public void onComplete(final Throwable failure, final PrimaryShardInfo primaryShardInfo) {
147 onFindPrimaryShardComplete(failure, primaryShardInfo);
149 }, getActorContext().getClientDispatcher());
152 private void onFindPrimaryShardComplete(final Throwable failure, final PrimaryShardInfo primaryShardInfo) {
153 if (failure == null) {
154 this.primaryShard = primaryShardInfo.getPrimaryShardActor();
155 tryCreateTransaction();
157 LOG.debug("Tx {}: Find primary for shard {} failed", getIdentifier(), shardName, failure);
159 onCreateTransactionComplete(failure, null);
163 private void onCreateTransactionComplete(Throwable failure, Object response) {
164 // An AskTimeoutException will occur if the local shard forwards to an unavailable remote leader or
165 // the cached remote leader actor is no longer available.
166 boolean retryCreateTransaction = this.primaryShard != null &&
167 (failure instanceof NoShardLeaderException || failure instanceof AskTimeoutException);
168 if(retryCreateTransaction) {
169 // Schedule a retry unless we're out of retries. Note: totalCreateTxTimeout is volatile as it may
170 // be written by different threads however not concurrently, therefore decrementing it
171 // non-atomically here is ok.
172 if(totalCreateTxTimeout > 0) {
173 long scheduleInterval = CREATE_TX_TRY_INTERVAL_IN_MS;
174 if(failure instanceof AskTimeoutException) {
175 // Since we use the createTxMessageTimeout for the CreateTransaction request and it timed
176 // out, subtract it from the total timeout. Also since the createTxMessageTimeout period
177 // has already elapsed, we can immediately schedule the retry (10 ms is virtually immediate).
178 totalCreateTxTimeout -= createTxMessageTimeout.duration().toMillis();
179 scheduleInterval = 10;
182 totalCreateTxTimeout -= scheduleInterval;
184 LOG.debug("Tx {}: create tx on shard {} failed with exception \"{}\" - scheduling retry in {} ms",
185 getIdentifier(), shardName, failure, scheduleInterval);
187 getActorContext().getActorSystem().scheduler().scheduleOnce(
188 FiniteDuration.create(scheduleInterval, TimeUnit.MILLISECONDS),
192 tryFindPrimaryShard();
194 }, getActorContext().getClientDispatcher());
199 createTransactionContext(failure, response);
202 private void createTransactionContext(Throwable failure, Object response) {
203 // Create the TransactionContext from the response or failure. Store the new
204 // TransactionContext locally until we've completed invoking the
205 // TransactionOperations. This avoids thread timing issues which could cause
206 // out-of-order TransactionOperations. Eg, on a modification operation, if the
207 // TransactionContext is non-null, then we directly call the TransactionContext.
208 // However, at the same time, the code may be executing the cached
209 // TransactionOperations. So to avoid thus timing, we don't publish the
210 // TransactionContext until after we've executed all cached TransactionOperations.
211 TransactionContext localTransactionContext;
212 if(failure != null) {
213 LOG.debug("Tx {} Creating NoOpTransaction because of error", getIdentifier(), failure);
215 Throwable resultingEx = failure;
216 if(failure instanceof AskTimeoutException) {
217 resultingEx = new ShardLeaderNotRespondingException(String.format(
218 "Could not create a %s transaction on shard %s. The shard leader isn't responding.",
219 parent.getType(), shardName), failure);
220 } else if(!(failure instanceof NoShardLeaderException)) {
221 resultingEx = new Exception(String.format(
222 "Error creating %s transaction on shard %s", parent.getType(), shardName), failure);
225 localTransactionContext = new NoOpTransactionContext(resultingEx, getIdentifier());
226 } else if (CreateTransactionReply.SERIALIZABLE_CLASS.equals(response.getClass())) {
227 localTransactionContext = createValidTransactionContext(
228 CreateTransactionReply.fromSerializable(response));
230 IllegalArgumentException exception = new IllegalArgumentException(String.format(
231 "Invalid reply type %s for CreateTransaction", response.getClass()));
233 localTransactionContext = new NoOpTransactionContext(exception, getIdentifier());
236 transactionContextWrapper.executePriorTransactionOperations(localTransactionContext);
239 private TransactionContext createValidTransactionContext(CreateTransactionReply reply) {
240 LOG.debug("Tx {} Received {}", getIdentifier(), reply);
242 return createValidTransactionContext(getActorContext().actorSelection(reply.getTransactionPath()),
243 reply.getTransactionPath(), reply.getVersion());
246 private TransactionContext createValidTransactionContext(ActorSelection transactionActor, String transactionPath,
247 short remoteTransactionVersion) {
248 // TxActor is always created where the leader of the shard is.
249 // Check if TxActor is created in the same node
250 boolean isTxActorLocal = getActorContext().isPathLocal(transactionPath);
251 final TransactionContext ret = new RemoteTransactionContext(transactionContextWrapper.getIdentifier(),
252 transactionActor, getActorContext(), isTxActorLocal, remoteTransactionVersion,
253 transactionContextWrapper.getLimiter());
255 if(parent.getType() == TransactionType.READ_ONLY) {
256 TransactionContextCleanup.track(this, ret);