2 * Copyright (c) 2015 Brocade Communications Systems, Inc. and others. All rights reserved.
3 * Copyright (c) 2015 Cisco Systems, Inc. and others. All rights reserved.
5 * This program and the accompanying materials are made available under the
6 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
7 * and is available at http://www.eclipse.org/legal/epl-v10.html
9 package org.opendaylight.controller.cluster.datastore;
11 import static java.util.Objects.requireNonNull;
13 import akka.actor.ActorSelection;
14 import akka.dispatch.OnComplete;
15 import akka.pattern.AskTimeoutException;
16 import akka.util.Timeout;
17 import java.util.concurrent.TimeUnit;
18 import org.opendaylight.controller.cluster.access.concepts.TransactionIdentifier;
19 import org.opendaylight.controller.cluster.datastore.exceptions.NoShardLeaderException;
20 import org.opendaylight.controller.cluster.datastore.exceptions.ShardLeaderNotRespondingException;
21 import org.opendaylight.controller.cluster.datastore.messages.CreateTransaction;
22 import org.opendaylight.controller.cluster.datastore.messages.CreateTransactionReply;
23 import org.opendaylight.controller.cluster.datastore.messages.PrimaryShardInfo;
24 import org.opendaylight.controller.cluster.datastore.utils.ActorUtils;
25 import org.slf4j.Logger;
26 import org.slf4j.LoggerFactory;
27 import scala.concurrent.Future;
28 import scala.concurrent.duration.FiniteDuration;
31 * Handles creation of TransactionContext instances for remote transactions. This class creates
32 * remote transactions, if necessary, by sending CreateTransaction messages with retries, up to a limit,
33 * if the shard doesn't have a leader yet. This is done by scheduling a retry task after a short delay.
35 * The end result from a completed CreateTransaction message is a TransactionContext that is
36 * used to perform transaction operations. Transaction operations that occur before the
37 * CreateTransaction completes are cache via a TransactionContextWrapper and executed once the
38 * CreateTransaction completes, successfully or not.
40 final class RemoteTransactionContextSupport {
41 private static final Logger LOG = LoggerFactory.getLogger(RemoteTransactionContextSupport.class);
43 private static final long CREATE_TX_TRY_INTERVAL_IN_MS = 1000;
44 private static final long MAX_CREATE_TX_MSG_TIMEOUT_IN_MS = 5000;
46 private final TransactionProxy parent;
47 private final String shardName;
50 * The target primary shard.
52 private volatile PrimaryShardInfo primaryShardInfo;
55 * The total timeout for creating a tx on the primary shard.
57 private volatile long totalCreateTxTimeout;
59 private final Timeout createTxMessageTimeout;
61 private final TransactionContextWrapper transactionContextWrapper;
63 RemoteTransactionContextSupport(final TransactionContextWrapper transactionContextWrapper,
64 final TransactionProxy parent, final String shardName) {
65 this.parent = requireNonNull(parent);
66 this.shardName = shardName;
67 this.transactionContextWrapper = transactionContextWrapper;
69 // For the total create tx timeout, use 2 times the election timeout. This should be enough time for
70 // a leader re-election to occur if we happen to hit it in transition.
71 totalCreateTxTimeout = parent.getActorUtils().getDatastoreContext().getShardRaftConfig()
72 .getElectionTimeOutInterval().toMillis() * 2;
74 // We'll use the operationTimeout for the the create Tx message timeout so it can be set appropriately
75 // for unit tests but cap it at MAX_CREATE_TX_MSG_TIMEOUT_IN_MS. The operationTimeout could be set
76 // larger than the totalCreateTxTimeout in production which we don't want.
77 long operationTimeout = parent.getActorUtils().getOperationTimeout().duration().toMillis();
78 createTxMessageTimeout = new Timeout(Math.min(operationTimeout, MAX_CREATE_TX_MSG_TIMEOUT_IN_MS),
79 TimeUnit.MILLISECONDS);
82 String getShardName() {
86 private TransactionType getTransactionType() {
87 return parent.getType();
90 private ActorUtils getActorUtils() {
91 return parent.getActorUtils();
94 private TransactionIdentifier getIdentifier() {
95 return parent.getIdentifier();
99 * Sets the target primary shard and initiates a CreateTransaction try.
101 void setPrimaryShard(final PrimaryShardInfo newPrimaryShardInfo) {
102 this.primaryShardInfo = newPrimaryShardInfo;
104 if (getTransactionType() == TransactionType.WRITE_ONLY
105 && getActorUtils().getDatastoreContext().isWriteOnlyTransactionOptimizationsEnabled()) {
106 ActorSelection primaryShard = newPrimaryShardInfo.getPrimaryShardActor();
108 LOG.debug("Tx {} Primary shard {} found - creating WRITE_ONLY transaction context",
109 getIdentifier(), primaryShard);
111 // For write-only Tx's we prepare the transaction modifications directly on the shard actor
112 // to avoid the overhead of creating a separate transaction actor.
113 transactionContextWrapper.executePriorTransactionOperations(createValidTransactionContext(
114 primaryShard, String.valueOf(primaryShard.path()), newPrimaryShardInfo.getPrimaryShardVersion()));
116 tryCreateTransaction();
121 Performs a CreateTransaction try async.
123 private void tryCreateTransaction() {
124 LOG.debug("Tx {} Primary shard {} found - trying create transaction", getIdentifier(),
125 primaryShardInfo.getPrimaryShardActor());
127 Object serializedCreateMessage = new CreateTransaction(getIdentifier(), getTransactionType().ordinal(),
128 primaryShardInfo.getPrimaryShardVersion()).toSerializable();
130 Future<Object> createTxFuture = getActorUtils().executeOperationAsync(
131 primaryShardInfo.getPrimaryShardActor(), serializedCreateMessage, createTxMessageTimeout);
133 createTxFuture.onComplete(new OnComplete<Object>() {
135 public void onComplete(final Throwable failure, final Object response) {
136 onCreateTransactionComplete(failure, response);
138 }, getActorUtils().getClientDispatcher());
141 private void tryFindPrimaryShard() {
142 LOG.debug("Tx {} Retrying findPrimaryShardAsync for shard {}", getIdentifier(), shardName);
144 this.primaryShardInfo = null;
145 Future<PrimaryShardInfo> findPrimaryFuture = getActorUtils().findPrimaryShardAsync(shardName);
146 findPrimaryFuture.onComplete(new OnComplete<PrimaryShardInfo>() {
148 public void onComplete(final Throwable failure, final PrimaryShardInfo newPrimaryShardInfo) {
149 onFindPrimaryShardComplete(failure, newPrimaryShardInfo);
151 }, getActorUtils().getClientDispatcher());
154 private void onFindPrimaryShardComplete(final Throwable failure, final PrimaryShardInfo newPrimaryShardInfo) {
155 if (failure == null) {
156 this.primaryShardInfo = newPrimaryShardInfo;
157 tryCreateTransaction();
159 LOG.debug("Tx {}: Find primary for shard {} failed", getIdentifier(), shardName, failure);
161 onCreateTransactionComplete(failure, null);
165 private void onCreateTransactionComplete(final Throwable failure, final Object response) {
166 // An AskTimeoutException will occur if the local shard forwards to an unavailable remote leader or
167 // the cached remote leader actor is no longer available.
168 boolean retryCreateTransaction = primaryShardInfo != null
169 && (failure instanceof NoShardLeaderException || failure instanceof AskTimeoutException);
171 // Schedule a retry unless we're out of retries. Note: totalCreateTxTimeout is volatile as it may
172 // be written by different threads however not concurrently, therefore decrementing it
173 // non-atomically here is ok.
174 if (retryCreateTransaction && totalCreateTxTimeout > 0) {
175 long scheduleInterval = CREATE_TX_TRY_INTERVAL_IN_MS;
176 if (failure instanceof AskTimeoutException) {
177 // Since we use the createTxMessageTimeout for the CreateTransaction request and it timed
178 // out, subtract it from the total timeout. Also since the createTxMessageTimeout period
179 // has already elapsed, we can immediately schedule the retry (10 ms is virtually immediate).
180 totalCreateTxTimeout -= createTxMessageTimeout.duration().toMillis();
181 scheduleInterval = 10;
184 totalCreateTxTimeout -= scheduleInterval;
186 LOG.debug("Tx {}: create tx on shard {} failed with exception \"{}\" - scheduling retry in {} ms",
187 getIdentifier(), shardName, failure, scheduleInterval);
189 getActorUtils().getActorSystem().scheduler().scheduleOnce(
190 FiniteDuration.create(scheduleInterval, TimeUnit.MILLISECONDS),
191 this::tryFindPrimaryShard, getActorUtils().getClientDispatcher());
195 createTransactionContext(failure, response);
198 private void createTransactionContext(final Throwable failure, final Object response) {
199 // Create the TransactionContext from the response or failure. Store the new
200 // TransactionContext locally until we've completed invoking the
201 // TransactionOperations. This avoids thread timing issues which could cause
202 // out-of-order TransactionOperations. Eg, on a modification operation, if the
203 // TransactionContext is non-null, then we directly call the TransactionContext.
204 // However, at the same time, the code may be executing the cached
205 // TransactionOperations. So to avoid thus timing, we don't publish the
206 // TransactionContext until after we've executed all cached TransactionOperations.
207 TransactionContext localTransactionContext;
208 if (failure != null) {
209 LOG.debug("Tx {} Creating NoOpTransaction because of error", getIdentifier(), failure);
211 Throwable resultingEx = failure;
212 if (failure instanceof AskTimeoutException) {
213 resultingEx = new ShardLeaderNotRespondingException(String.format(
214 "Could not create a %s transaction on shard %s. The shard leader isn't responding.",
215 parent.getType(), shardName), failure);
216 } else if (!(failure instanceof NoShardLeaderException)) {
217 resultingEx = new Exception(String.format(
218 "Error creating %s transaction on shard %s", parent.getType(), shardName), failure);
221 localTransactionContext = new NoOpTransactionContext(resultingEx, getIdentifier());
222 } else if (CreateTransactionReply.isSerializedType(response)) {
223 localTransactionContext = createValidTransactionContext(
224 CreateTransactionReply.fromSerializable(response));
226 IllegalArgumentException exception = new IllegalArgumentException(String.format(
227 "Invalid reply type %s for CreateTransaction", response.getClass()));
229 localTransactionContext = new NoOpTransactionContext(exception, getIdentifier());
232 transactionContextWrapper.executePriorTransactionOperations(localTransactionContext);
235 private TransactionContext createValidTransactionContext(final CreateTransactionReply reply) {
236 LOG.debug("Tx {} Received {}", getIdentifier(), reply);
238 return createValidTransactionContext(getActorUtils().actorSelection(reply.getTransactionPath()),
239 reply.getTransactionPath(), primaryShardInfo.getPrimaryShardVersion());
242 private TransactionContext createValidTransactionContext(final ActorSelection transactionActor,
243 final String transactionPath, final short remoteTransactionVersion) {
244 final TransactionContext ret = new RemoteTransactionContext(transactionContextWrapper.getIdentifier(),
245 transactionActor, getActorUtils(), remoteTransactionVersion, transactionContextWrapper.getLimiter());
247 if (parent.getType() == TransactionType.READ_ONLY) {
248 TransactionContextCleanup.track(parent, ret);