2 * Copyright (c) 2014 Brocade Communications Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.controller.cluster.datastore;
10 import akka.actor.ActorRef;
11 import akka.actor.Status;
12 import akka.serialization.Serialization;
13 import com.google.common.annotations.VisibleForTesting;
14 import com.google.common.base.Preconditions;
15 import com.google.common.cache.Cache;
16 import com.google.common.cache.CacheBuilder;
17 import com.google.common.cache.RemovalCause;
18 import com.google.common.cache.RemovalListener;
19 import com.google.common.cache.RemovalNotification;
20 import java.util.LinkedList;
21 import java.util.Queue;
22 import java.util.concurrent.ExecutionException;
23 import java.util.concurrent.TimeUnit;
24 import org.opendaylight.controller.cluster.datastore.compat.BackwardsCompatibleThreePhaseCommitCohort;
25 import org.opendaylight.controller.cluster.datastore.messages.BatchedModifications;
26 import org.opendaylight.controller.cluster.datastore.messages.BatchedModificationsReply;
27 import org.opendaylight.controller.cluster.datastore.messages.CanCommitTransactionReply;
28 import org.opendaylight.controller.cluster.datastore.messages.ForwardedReadyTransaction;
29 import org.opendaylight.controller.cluster.datastore.messages.ReadyTransactionReply;
30 import org.opendaylight.controller.cluster.datastore.modification.Modification;
31 import org.opendaylight.controller.cluster.datastore.modification.MutableCompositeModification;
32 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
33 import org.opendaylight.controller.sal.core.spi.data.DOMStoreThreePhaseCommitCohort;
34 import org.opendaylight.controller.sal.core.spi.data.DOMStoreWriteTransaction;
35 import org.slf4j.Logger;
38 * Coordinates commits for a shard ensuring only one concurrent 3-phase commit.
40 * @author Thomas Pantelis
42 public class ShardCommitCoordinator {
44 // Interface hook for unit tests to replace or decorate the DOMStoreThreePhaseCommitCohorts.
45 public interface CohortDecorator {
46 DOMStoreThreePhaseCommitCohort decorate(String transactionID, DOMStoreThreePhaseCommitCohort actual);
49 private final Cache<String, CohortEntry> cohortCache;
51 private CohortEntry currentCohortEntry;
53 private final DOMTransactionFactory transactionFactory;
55 private final Queue<CohortEntry> queuedCohortEntries;
57 private int queueCapacity;
59 private final Logger log;
61 private final String name;
63 private final RemovalListener<String, CohortEntry> cacheRemovalListener =
64 new RemovalListener<String, CohortEntry>() {
66 public void onRemoval(RemovalNotification<String, CohortEntry> notification) {
67 if(notification.getCause() == RemovalCause.EXPIRED) {
68 log.warn("{}: Transaction {} was timed out of the cache", name, notification.getKey());
73 // This is a hook for unit tests to replace or decorate the DOMStoreThreePhaseCommitCohorts.
74 private CohortDecorator cohortDecorator;
76 private ReadyTransactionReply readyTransactionReply;
78 public ShardCommitCoordinator(DOMTransactionFactory transactionFactory,
79 long cacheExpiryTimeoutInSec, int queueCapacity, ActorRef shardActor, Logger log, String name) {
81 this.queueCapacity = queueCapacity;
84 this.transactionFactory = transactionFactory;
86 cohortCache = CacheBuilder.newBuilder().expireAfterAccess(cacheExpiryTimeoutInSec, TimeUnit.SECONDS).
87 removalListener(cacheRemovalListener).build();
89 // We use a LinkedList here to avoid synchronization overhead with concurrent queue impls
90 // since this should only be accessed on the shard's dispatcher.
91 queuedCohortEntries = new LinkedList<>();
94 public void setQueueCapacity(int queueCapacity) {
95 this.queueCapacity = queueCapacity;
98 private ReadyTransactionReply readyTransactionReply(Shard shard) {
99 if(readyTransactionReply == null) {
100 readyTransactionReply = new ReadyTransactionReply(Serialization.serializedActorPath(shard.self()));
103 return readyTransactionReply;
107 * This method is called to ready a transaction that was prepared by ShardTransaction actor. It caches
108 * the prepared cohort entry for the given transactions ID in preparation for the subsequent 3-phase commit.
110 public void handleForwardedReadyTransaction(ForwardedReadyTransaction ready, ActorRef sender, Shard shard) {
111 log.debug("{}: Readying transaction {}, client version {}", name,
112 ready.getTransactionID(), ready.getTxnClientVersion());
114 CohortEntry cohortEntry = new CohortEntry(ready.getTransactionID(), ready.getCohort(),
115 (MutableCompositeModification) ready.getModification());
116 cohortCache.put(ready.getTransactionID(), cohortEntry);
118 if(ready.getTxnClientVersion() < DataStoreVersions.LITHIUM_VERSION) {
119 // Return our actor path as we'll handle the three phase commit except if the Tx client
120 // version < Helium-1 version which means the Tx was initiated by a base Helium version node.
121 // In that case, the subsequent 3-phase commit messages won't contain the transactionId so to
122 // maintain backwards compatibility, we create a separate cohort actor to provide the compatible behavior.
123 ActorRef replyActorPath = shard.self();
124 if(ready.getTxnClientVersion() < DataStoreVersions.HELIUM_1_VERSION) {
125 log.debug("{}: Creating BackwardsCompatibleThreePhaseCommitCohort", name);
126 replyActorPath = shard.getContext().actorOf(BackwardsCompatibleThreePhaseCommitCohort.props(
127 ready.getTransactionID()));
130 ReadyTransactionReply readyTransactionReply =
131 new ReadyTransactionReply(Serialization.serializedActorPath(replyActorPath),
132 ready.getTxnClientVersion());
133 sender.tell(ready.isReturnSerialized() ? readyTransactionReply.toSerializable() :
134 readyTransactionReply, shard.self());
136 if(ready.isDoImmediateCommit()) {
137 cohortEntry.setDoImmediateCommit(true);
138 cohortEntry.setReplySender(sender);
139 cohortEntry.setShard(shard);
140 handleCanCommit(cohortEntry);
142 // The caller does not want immediate commit - the 3-phase commit will be coordinated by the
143 // front-end so send back a ReadyTransactionReply with our actor path.
144 sender.tell(readyTransactionReply(shard), shard.self());
150 * This method handles a BatchedModifications message for a transaction being prepared directly on the
151 * Shard actor instead of via a ShardTransaction actor. If there's no currently cached
152 * DOMStoreWriteTransaction, one is created. The batched modifications are applied to the write Tx. If
153 * the BatchedModifications is ready to commit then a DOMStoreThreePhaseCommitCohort is created.
155 * @param batched the BatchedModifications
156 * @param shardActor the transaction's shard actor
158 * @throws ExecutionException if an error occurs loading the cache
160 boolean handleBatchedModifications(BatchedModifications batched, ActorRef sender, Shard shard)
161 throws ExecutionException {
162 CohortEntry cohortEntry = cohortCache.getIfPresent(batched.getTransactionID());
163 if(cohortEntry == null) {
164 cohortEntry = new CohortEntry(batched.getTransactionID(),
165 transactionFactory.<DOMStoreWriteTransaction>newTransaction(
166 TransactionProxy.TransactionType.WRITE_ONLY, batched.getTransactionID(),
167 batched.getTransactionChainID()));
168 cohortCache.put(batched.getTransactionID(), cohortEntry);
171 if(log.isDebugEnabled()) {
172 log.debug("{}: Applying {} batched modifications for Tx {}", name,
173 batched.getModifications().size(), batched.getTransactionID());
176 cohortEntry.applyModifications(batched.getModifications());
178 if(batched.isReady()) {
179 if(log.isDebugEnabled()) {
180 log.debug("{}: Readying Tx {}, client version {}", name,
181 batched.getTransactionID(), batched.getVersion());
184 cohortEntry.ready(cohortDecorator, batched.isDoCommitOnReady());
186 if(batched.isDoCommitOnReady()) {
187 cohortEntry.setReplySender(sender);
188 cohortEntry.setShard(shard);
189 handleCanCommit(cohortEntry);
191 sender.tell(readyTransactionReply(shard), shard.self());
194 sender.tell(new BatchedModificationsReply(batched.getModifications().size()), shard.self());
197 return batched.isReady();
200 private void handleCanCommit(CohortEntry cohortEntry) {
201 String transactionID = cohortEntry.getTransactionID();
203 if(log.isDebugEnabled()) {
204 log.debug("{}: Processing canCommit for transaction {} for shard {}",
205 name, transactionID, cohortEntry.getShard().self().path());
208 if(currentCohortEntry != null) {
209 // There's already a Tx commit in progress - attempt to queue this entry to be
210 // committed after the current Tx completes.
211 log.debug("{}: Transaction {} is already in progress - queueing transaction {}",
212 name, currentCohortEntry.getTransactionID(), transactionID);
214 if(queuedCohortEntries.size() < queueCapacity) {
215 queuedCohortEntries.offer(cohortEntry);
217 removeCohortEntry(transactionID);
219 RuntimeException ex = new RuntimeException(
220 String.format("%s: Could not enqueue transaction %s - the maximum commit queue"+
221 " capacity %d has been reached.",
222 name, transactionID, queueCapacity));
223 log.error(ex.getMessage());
224 cohortEntry.getReplySender().tell(new Status.Failure(ex), cohortEntry.getShard().self());
227 // No Tx commit currently in progress - make this the current entry and proceed with
229 cohortEntry.updateLastAccessTime();
230 currentCohortEntry = cohortEntry;
232 doCanCommit(cohortEntry);
237 * This method handles the canCommit phase for a transaction.
239 * @param canCommit the CanCommitTransaction message
240 * @param sender the actor that sent the message
241 * @param shard the transaction's shard actor
243 public void handleCanCommit(String transactionID, final ActorRef sender, final Shard shard) {
244 // Lookup the cohort entry that was cached previously (or should have been) by
245 // transactionReady (via the ForwardedReadyTransaction message).
246 final CohortEntry cohortEntry = cohortCache.getIfPresent(transactionID);
247 if(cohortEntry == null) {
248 // Either canCommit was invoked before ready(shouldn't happen) or a long time passed
249 // between canCommit and ready and the entry was expired from the cache.
250 IllegalStateException ex = new IllegalStateException(
251 String.format("%s: No cohort entry found for transaction %s", name, transactionID));
252 log.error(ex.getMessage());
253 sender.tell(new Status.Failure(ex), shard.self());
257 cohortEntry.setReplySender(sender);
258 cohortEntry.setShard(shard);
260 handleCanCommit(cohortEntry);
263 private void doCanCommit(final CohortEntry cohortEntry) {
265 boolean canCommit = false;
267 // We block on the future here so we don't have to worry about possibly accessing our
268 // state on a different thread outside of our dispatcher. Also, the data store
269 // currently uses a same thread executor anyway.
270 canCommit = cohortEntry.getCohort().canCommit().get();
272 if(cohortEntry.isDoImmediateCommit()) {
274 doCommit(cohortEntry);
276 cohortEntry.getReplySender().tell(new Status.Failure(new TransactionCommitFailedException(
277 "Can Commit failed, no detailed cause available.")), cohortEntry.getShard().self());
280 cohortEntry.getReplySender().tell(
281 canCommit ? CanCommitTransactionReply.YES.toSerializable() :
282 CanCommitTransactionReply.NO.toSerializable(), cohortEntry.getShard().self());
284 } catch (Exception e) {
285 log.debug("{}: An exception occurred during canCommit: {}", name, e);
287 Throwable failure = e;
288 if(e instanceof ExecutionException) {
289 failure = e.getCause();
292 cohortEntry.getReplySender().tell(new Status.Failure(failure), cohortEntry.getShard().self());
295 // Remove the entry from the cache now.
296 currentTransactionComplete(cohortEntry.getTransactionID(), true);
301 private boolean doCommit(CohortEntry cohortEntry) {
302 log.debug("{}: Committing transaction {}", name, cohortEntry.getTransactionID());
304 boolean success = false;
306 // We perform the preCommit phase here atomically with the commit phase. This is an
307 // optimization to eliminate the overhead of an extra preCommit message. We lose front-end
308 // coordination of preCommit across shards in case of failure but preCommit should not
309 // normally fail since we ensure only one concurrent 3-phase commit.
312 // We block on the future here so we don't have to worry about possibly accessing our
313 // state on a different thread outside of our dispatcher. Also, the data store
314 // currently uses a same thread executor anyway.
315 cohortEntry.getCohort().preCommit().get();
317 cohortEntry.getShard().continueCommit(cohortEntry);
319 cohortEntry.updateLastAccessTime();
322 } catch (Exception e) {
323 log.error("{} An exception occurred while preCommitting transaction {}",
324 name, cohortEntry.getTransactionID(), e);
325 cohortEntry.getReplySender().tell(new akka.actor.Status.Failure(e), cohortEntry.getShard().self());
327 currentTransactionComplete(cohortEntry.getTransactionID(), true);
333 boolean handleCommit(final String transactionID, final ActorRef sender, final Shard shard) {
334 // Get the current in-progress cohort entry in the commitCoordinator if it corresponds to
336 final CohortEntry cohortEntry = getCohortEntryIfCurrent(transactionID);
337 if(cohortEntry == null) {
338 // We're not the current Tx - the Tx was likely expired b/c it took too long in
339 // between the canCommit and commit messages.
340 IllegalStateException ex = new IllegalStateException(
341 String.format("%s: Cannot commit transaction %s - it is not the current transaction",
342 name, transactionID));
343 log.error(ex.getMessage());
344 sender.tell(new akka.actor.Status.Failure(ex), shard.self());
348 return doCommit(cohortEntry);
352 * Returns the cohort entry for the Tx commit currently in progress if the given transaction ID
353 * matches the current entry.
355 * @param transactionID the ID of the transaction
356 * @return the current CohortEntry or null if the given transaction ID does not match the
359 public CohortEntry getCohortEntryIfCurrent(String transactionID) {
360 if(isCurrentTransaction(transactionID)) {
361 return currentCohortEntry;
367 public CohortEntry getCurrentCohortEntry() {
368 return currentCohortEntry;
371 public CohortEntry getAndRemoveCohortEntry(String transactionID) {
372 CohortEntry cohortEntry = cohortCache.getIfPresent(transactionID);
373 cohortCache.invalidate(transactionID);
377 public void removeCohortEntry(String transactionID) {
378 cohortCache.invalidate(transactionID);
381 public boolean isCurrentTransaction(String transactionID) {
382 return currentCohortEntry != null &&
383 currentCohortEntry.getTransactionID().equals(transactionID);
387 * This method is called when a transaction is complete, successful or not. If the given
388 * given transaction ID matches the current in-progress transaction, the next cohort entry,
389 * if any, is dequeued and processed.
391 * @param transactionID the ID of the completed transaction
392 * @param removeCohortEntry if true the CohortEntry for the transaction is also removed from
395 public void currentTransactionComplete(String transactionID, boolean removeCohortEntry) {
396 if(removeCohortEntry) {
397 removeCohortEntry(transactionID);
400 if(isCurrentTransaction(transactionID)) {
401 // Dequeue the next cohort entry waiting in the queue.
402 currentCohortEntry = queuedCohortEntries.poll();
403 if(currentCohortEntry != null) {
404 currentCohortEntry.updateLastAccessTime();
405 doCanCommit(currentCohortEntry);
411 void setCohortDecorator(CohortDecorator cohortDecorator) {
412 this.cohortDecorator = cohortDecorator;
416 static class CohortEntry {
417 private final String transactionID;
418 private DOMStoreThreePhaseCommitCohort cohort;
419 private final MutableCompositeModification compositeModification;
420 private final DOMStoreWriteTransaction transaction;
421 private ActorRef replySender;
423 private long lastAccessTime;
424 private boolean doImmediateCommit;
426 CohortEntry(String transactionID, DOMStoreWriteTransaction transaction) {
427 this.compositeModification = new MutableCompositeModification();
428 this.transaction = transaction;
429 this.transactionID = transactionID;
432 CohortEntry(String transactionID, DOMStoreThreePhaseCommitCohort cohort,
433 MutableCompositeModification compositeModification) {
434 this.transactionID = transactionID;
435 this.cohort = cohort;
436 this.compositeModification = compositeModification;
437 this.transaction = null;
440 void updateLastAccessTime() {
441 lastAccessTime = System.currentTimeMillis();
444 long getLastAccessTime() {
445 return lastAccessTime;
448 String getTransactionID() {
449 return transactionID;
452 DOMStoreThreePhaseCommitCohort getCohort() {
456 MutableCompositeModification getModification() {
457 return compositeModification;
460 void applyModifications(Iterable<Modification> modifications) {
461 for(Modification modification: modifications) {
462 compositeModification.addModification(modification);
463 modification.apply(transaction);
467 void ready(CohortDecorator cohortDecorator, boolean doImmediateCommit) {
468 Preconditions.checkState(cohort == null, "cohort was already set");
470 setDoImmediateCommit(doImmediateCommit);
472 cohort = transaction.ready();
474 if(cohortDecorator != null) {
475 // Call the hook for unit tests.
476 cohort = cohortDecorator.decorate(transactionID, cohort);
480 boolean isDoImmediateCommit() {
481 return doImmediateCommit;
484 void setDoImmediateCommit(boolean doImmediateCommit) {
485 this.doImmediateCommit = doImmediateCommit;
488 ActorRef getReplySender() {
492 void setReplySender(ActorRef replySender) {
493 this.replySender = replySender;
500 void setShard(Shard shard) {
504 boolean hasModifications(){
505 return compositeModification.getModifications().size() > 0;