2 * Copyright (c) 2014 Brocade Communications Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.controller.cluster.datastore;
10 import akka.actor.ActorRef;
11 import akka.actor.Status.Failure;
12 import akka.serialization.Serialization;
13 import com.google.common.annotations.VisibleForTesting;
14 import com.google.common.base.Preconditions;
15 import com.google.common.base.Stopwatch;
16 import java.util.ArrayList;
17 import java.util.HashMap;
18 import java.util.Iterator;
19 import java.util.LinkedList;
20 import java.util.List;
22 import java.util.Queue;
23 import java.util.concurrent.ExecutionException;
24 import java.util.concurrent.TimeUnit;
25 import org.opendaylight.controller.cluster.datastore.messages.AbortTransactionReply;
26 import org.opendaylight.controller.cluster.datastore.messages.BatchedModifications;
27 import org.opendaylight.controller.cluster.datastore.messages.BatchedModificationsReply;
28 import org.opendaylight.controller.cluster.datastore.messages.CanCommitTransactionReply;
29 import org.opendaylight.controller.cluster.datastore.messages.ForwardedReadyTransaction;
30 import org.opendaylight.controller.cluster.datastore.messages.ReadyLocalTransaction;
31 import org.opendaylight.controller.cluster.datastore.messages.ReadyTransactionReply;
32 import org.opendaylight.controller.cluster.datastore.modification.Modification;
33 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
34 import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidate;
35 import org.slf4j.Logger;
38 * Coordinates commits for a shard ensuring only one concurrent 3-phase commit.
40 * @author Thomas Pantelis
42 class ShardCommitCoordinator {
44 // Interface hook for unit tests to replace or decorate the DOMStoreThreePhaseCommitCohorts.
45 public interface CohortDecorator {
46 ShardDataTreeCohort decorate(String transactionID, ShardDataTreeCohort actual);
49 private final Map<String, CohortEntry> cohortCache = new HashMap<>();
51 private CohortEntry currentCohortEntry;
53 private final ShardDataTree dataTree;
55 // We use a LinkedList here to avoid synchronization overhead with concurrent queue impls
56 // since this should only be accessed on the shard's dispatcher.
57 private final Queue<CohortEntry> queuedCohortEntries = new LinkedList<>();
59 private int queueCapacity;
61 private final Logger log;
63 private final String name;
65 private final long cacheExpiryTimeoutInMillis;
67 // This is a hook for unit tests to replace or decorate the DOMStoreThreePhaseCommitCohorts.
68 private CohortDecorator cohortDecorator;
70 private ReadyTransactionReply readyTransactionReply;
72 private Runnable runOnPendingTransactionsComplete;
74 ShardCommitCoordinator(ShardDataTree dataTree,
75 long cacheExpiryTimeoutInMillis, int queueCapacity, Logger log, String name) {
77 this.queueCapacity = queueCapacity;
80 this.dataTree = Preconditions.checkNotNull(dataTree);
81 this.cacheExpiryTimeoutInMillis = cacheExpiryTimeoutInMillis;
85 return queuedCohortEntries.size();
88 int getCohortCacheSize() {
89 return cohortCache.size();
92 void setQueueCapacity(int queueCapacity) {
93 this.queueCapacity = queueCapacity;
96 private ReadyTransactionReply readyTransactionReply(Shard shard) {
97 if(readyTransactionReply == null) {
98 readyTransactionReply = new ReadyTransactionReply(Serialization.serializedActorPath(shard.self()));
101 return readyTransactionReply;
104 private boolean queueCohortEntry(CohortEntry cohortEntry, ActorRef sender, Shard shard) {
105 if(queuedCohortEntries.size() < queueCapacity) {
106 queuedCohortEntries.offer(cohortEntry);
108 log.debug("{}: Enqueued transaction {}, queue size {}", name, cohortEntry.getTransactionID(),
109 queuedCohortEntries.size());
113 cohortCache.remove(cohortEntry.getTransactionID());
115 RuntimeException ex = new RuntimeException(
116 String.format("%s: Could not enqueue transaction %s - the maximum commit queue"+
117 " capacity %d has been reached.",
118 name, cohortEntry.getTransactionID(), queueCapacity));
119 log.error(ex.getMessage());
120 sender.tell(new Failure(ex), shard.self());
126 * This method is called to ready a transaction that was prepared by ShardTransaction actor. It caches
127 * the prepared cohort entry for the given transactions ID in preparation for the subsequent 3-phase commit.
129 * @param ready the ForwardedReadyTransaction message to process
130 * @param sender the sender of the message
131 * @param shard the transaction's shard actor
133 void handleForwardedReadyTransaction(ForwardedReadyTransaction ready, ActorRef sender, Shard shard) {
134 log.debug("{}: Readying transaction {}, client version {}", name,
135 ready.getTransactionID(), ready.getTxnClientVersion());
137 ShardDataTreeCohort cohort = ready.getTransaction().ready();
138 CohortEntry cohortEntry = new CohortEntry(ready.getTransactionID(), cohort);
139 cohortCache.put(ready.getTransactionID(), cohortEntry);
141 if(!queueCohortEntry(cohortEntry, sender, shard)) {
145 if(ready.isDoImmediateCommit()) {
146 cohortEntry.setDoImmediateCommit(true);
147 cohortEntry.setReplySender(sender);
148 cohortEntry.setShard(shard);
149 handleCanCommit(cohortEntry);
151 // The caller does not want immediate commit - the 3-phase commit will be coordinated by the
152 // front-end so send back a ReadyTransactionReply with our actor path.
153 sender.tell(readyTransactionReply(shard), shard.self());
158 * This method handles a BatchedModifications message for a transaction being prepared directly on the
159 * Shard actor instead of via a ShardTransaction actor. If there's no currently cached
160 * DOMStoreWriteTransaction, one is created. The batched modifications are applied to the write Tx. If
161 * the BatchedModifications is ready to commit then a DOMStoreThreePhaseCommitCohort is created.
163 * @param batched the BatchedModifications message to process
164 * @param sender the sender of the message
165 * @param shard the transaction's shard actor
167 void handleBatchedModifications(BatchedModifications batched, ActorRef sender, Shard shard) {
168 CohortEntry cohortEntry = cohortCache.get(batched.getTransactionID());
169 if(cohortEntry == null) {
170 cohortEntry = new CohortEntry(batched.getTransactionID(),
171 dataTree.newReadWriteTransaction(batched.getTransactionID(),
172 batched.getTransactionChainID()));
173 cohortCache.put(batched.getTransactionID(), cohortEntry);
176 if(log.isDebugEnabled()) {
177 log.debug("{}: Applying {} batched modifications for Tx {}", name,
178 batched.getModifications().size(), batched.getTransactionID());
181 cohortEntry.applyModifications(batched.getModifications());
183 if(batched.isReady()) {
184 if(cohortEntry.getLastBatchedModificationsException() != null) {
185 cohortCache.remove(cohortEntry.getTransactionID());
186 throw cohortEntry.getLastBatchedModificationsException();
189 if(cohortEntry.getTotalBatchedModificationsReceived() != batched.getTotalMessagesSent()) {
190 cohortCache.remove(cohortEntry.getTransactionID());
191 throw new IllegalStateException(String.format(
192 "The total number of batched messages received %d does not match the number sent %d",
193 cohortEntry.getTotalBatchedModificationsReceived(), batched.getTotalMessagesSent()));
196 if(!queueCohortEntry(cohortEntry, sender, shard)) {
200 if(log.isDebugEnabled()) {
201 log.debug("{}: Readying Tx {}, client version {}", name,
202 batched.getTransactionID(), batched.getVersion());
205 cohortEntry.ready(cohortDecorator, batched.isDoCommitOnReady());
207 if(batched.isDoCommitOnReady()) {
208 cohortEntry.setReplySender(sender);
209 cohortEntry.setShard(shard);
210 handleCanCommit(cohortEntry);
212 sender.tell(readyTransactionReply(shard), shard.self());
215 sender.tell(new BatchedModificationsReply(batched.getModifications().size()), shard.self());
220 * This method handles {@link ReadyLocalTransaction} message. All transaction modifications have
221 * been prepared beforehand by the sender and we just need to drive them through into the dataTree.
223 * @param message the ReadyLocalTransaction message to process
224 * @param sender the sender of the message
225 * @param shard the transaction's shard actor
227 void handleReadyLocalTransaction(ReadyLocalTransaction message, ActorRef sender, Shard shard) {
228 final ShardDataTreeCohort cohort = new SimpleShardDataTreeCohort(dataTree, message.getModification(),
229 message.getTransactionID());
230 final CohortEntry cohortEntry = new CohortEntry(message.getTransactionID(), cohort);
231 cohortCache.put(message.getTransactionID(), cohortEntry);
232 cohortEntry.setDoImmediateCommit(message.isDoCommitOnReady());
234 if(!queueCohortEntry(cohortEntry, sender, shard)) {
238 log.debug("{}: Applying local modifications for Tx {}", name, message.getTransactionID());
240 if (message.isDoCommitOnReady()) {
241 cohortEntry.setReplySender(sender);
242 cohortEntry.setShard(shard);
243 handleCanCommit(cohortEntry);
245 sender.tell(readyTransactionReply(shard), shard.self());
249 private void handleCanCommit(CohortEntry cohortEntry) {
250 String transactionID = cohortEntry.getTransactionID();
252 cohortEntry.updateLastAccessTime();
254 if(currentCohortEntry != null) {
255 // There's already a Tx commit in progress so we can't process this entry yet - but it's in the
256 // queue and will get processed after all prior entries complete.
258 if(log.isDebugEnabled()) {
259 log.debug("{}: Commit for Tx {} already in progress - skipping canCommit for {} for now",
260 name, currentCohortEntry.getTransactionID(), transactionID);
266 // No Tx commit currently in progress - check if this entry is the next one in the queue, If so make
267 // it the current entry and proceed with canCommit.
268 // Purposely checking reference equality here.
269 if(queuedCohortEntries.peek() == cohortEntry) {
270 currentCohortEntry = queuedCohortEntries.poll();
271 doCanCommit(currentCohortEntry);
273 if(log.isDebugEnabled()) {
274 log.debug("{}: Tx {} is the next pending canCommit - skipping {} for now",
275 name, queuedCohortEntries.peek().getTransactionID(), transactionID);
281 * This method handles the canCommit phase for a transaction.
283 * @param transactionID the ID of the transaction to canCommit
284 * @param sender the actor to which to send the response
285 * @param shard the transaction's shard actor
287 void handleCanCommit(String transactionID, final ActorRef sender, final Shard shard) {
288 // Lookup the cohort entry that was cached previously (or should have been) by
289 // transactionReady (via the ForwardedReadyTransaction message).
290 final CohortEntry cohortEntry = cohortCache.get(transactionID);
291 if(cohortEntry == null) {
292 // Either canCommit was invoked before ready(shouldn't happen) or a long time passed
293 // between canCommit and ready and the entry was expired from the cache.
294 IllegalStateException ex = new IllegalStateException(
295 String.format("%s: No cohort entry found for transaction %s", name, transactionID));
296 log.error(ex.getMessage());
297 sender.tell(new Failure(ex), shard.self());
301 cohortEntry.setReplySender(sender);
302 cohortEntry.setShard(shard);
304 handleCanCommit(cohortEntry);
307 private void doCanCommit(final CohortEntry cohortEntry) {
308 boolean canCommit = false;
310 canCommit = cohortEntry.canCommit();
312 log.debug("{}: canCommit for {}: {}", name, cohortEntry.getTransactionID(), canCommit);
314 if(cohortEntry.isDoImmediateCommit()) {
316 doCommit(cohortEntry);
318 cohortEntry.getReplySender().tell(new Failure(new TransactionCommitFailedException(
319 "Can Commit failed, no detailed cause available.")), cohortEntry.getShard().self());
322 cohortEntry.getReplySender().tell(
323 canCommit ? CanCommitTransactionReply.YES.toSerializable() :
324 CanCommitTransactionReply.NO.toSerializable(), cohortEntry.getShard().self());
326 } catch (Exception e) {
327 log.debug("{}: An exception occurred during canCommit", name, e);
329 Throwable failure = e;
330 if(e instanceof ExecutionException) {
331 failure = e.getCause();
334 cohortEntry.getReplySender().tell(new Failure(failure), cohortEntry.getShard().self());
337 // Remove the entry from the cache now.
338 currentTransactionComplete(cohortEntry.getTransactionID(), true);
343 private boolean doCommit(CohortEntry cohortEntry) {
344 log.debug("{}: Committing transaction {}", name, cohortEntry.getTransactionID());
346 boolean success = false;
348 // We perform the preCommit phase here atomically with the commit phase. This is an
349 // optimization to eliminate the overhead of an extra preCommit message. We lose front-end
350 // coordination of preCommit across shards in case of failure but preCommit should not
351 // normally fail since we ensure only one concurrent 3-phase commit.
354 cohortEntry.preCommit();
356 cohortEntry.getShard().continueCommit(cohortEntry);
358 cohortEntry.updateLastAccessTime();
361 } catch (Exception e) {
362 log.error("{} An exception occurred while preCommitting transaction {}",
363 name, cohortEntry.getTransactionID(), e);
364 cohortEntry.getReplySender().tell(new Failure(e), cohortEntry.getShard().self());
366 currentTransactionComplete(cohortEntry.getTransactionID(), true);
373 * This method handles the preCommit and commit phases for a transaction.
375 * @param transactionID the ID of the transaction to commit
376 * @param sender the actor to which to send the response
377 * @param shard the transaction's shard actor
378 * @return true if the transaction was successfully prepared, false otherwise.
380 boolean handleCommit(final String transactionID, final ActorRef sender, final Shard shard) {
381 // Get the current in-progress cohort entry in the commitCoordinator if it corresponds to
383 final CohortEntry cohortEntry = getCohortEntryIfCurrent(transactionID);
384 if(cohortEntry == null) {
385 // We're not the current Tx - the Tx was likely expired b/c it took too long in
386 // between the canCommit and commit messages.
387 IllegalStateException ex = new IllegalStateException(
388 String.format("%s: Cannot commit transaction %s - it is not the current transaction",
389 name, transactionID));
390 log.error(ex.getMessage());
391 sender.tell(new Failure(ex), shard.self());
395 cohortEntry.setReplySender(sender);
396 return doCommit(cohortEntry);
399 void handleAbort(final String transactionID, final ActorRef sender, final Shard shard) {
400 CohortEntry cohortEntry = getCohortEntryIfCurrent(transactionID);
401 if(cohortEntry != null) {
402 // We don't remove the cached cohort entry here (ie pass false) in case the Tx was
403 // aborted during replication in which case we may still commit locally if replication
405 currentTransactionComplete(transactionID, false);
407 cohortEntry = getAndRemoveCohortEntry(transactionID);
410 if(cohortEntry == null) {
414 log.debug("{}: Aborting transaction {}", name, transactionID);
416 final ActorRef self = shard.getSelf();
420 shard.getShardMBean().incrementAbortTransactionsCount();
423 sender.tell(new AbortTransactionReply().toSerializable(), self);
425 } catch (Exception e) {
426 log.error("{}: An exception happened during abort", name, e);
429 sender.tell(new Failure(e), self);
434 void checkForExpiredTransactions(final long timeout, final Shard shard) {
435 CohortEntry cohortEntry = getCurrentCohortEntry();
436 if(cohortEntry != null) {
437 if(cohortEntry.isExpired(timeout)) {
438 log.warn("{}: Current transaction {} has timed out after {} ms - aborting",
439 name, cohortEntry.getTransactionID(), timeout);
441 handleAbort(cohortEntry.getTransactionID(), null, shard);
445 cleanupExpiredCohortEntries();
448 void abortPendingTransactions(final String reason, final Shard shard) {
449 if(currentCohortEntry == null && queuedCohortEntries.isEmpty()) {
453 List<CohortEntry> cohortEntries = new ArrayList<>();
455 if(currentCohortEntry != null) {
456 cohortEntries.add(currentCohortEntry);
457 currentCohortEntry = null;
460 cohortEntries.addAll(queuedCohortEntries);
461 queuedCohortEntries.clear();
463 for(CohortEntry cohortEntry: cohortEntries) {
464 if(cohortEntry.getReplySender() != null) {
465 cohortEntry.getReplySender().tell(new Failure(new RuntimeException(reason)), shard.self());
471 * Returns the cohort entry for the Tx commit currently in progress if the given transaction ID
472 * matches the current entry.
474 * @param transactionID the ID of the transaction
475 * @return the current CohortEntry or null if the given transaction ID does not match the
478 CohortEntry getCohortEntryIfCurrent(String transactionID) {
479 if(isCurrentTransaction(transactionID)) {
480 return currentCohortEntry;
486 CohortEntry getCurrentCohortEntry() {
487 return currentCohortEntry;
490 CohortEntry getAndRemoveCohortEntry(String transactionID) {
491 return cohortCache.remove(transactionID);
494 boolean isCurrentTransaction(String transactionID) {
495 return currentCohortEntry != null &&
496 currentCohortEntry.getTransactionID().equals(transactionID);
500 * This method is called when a transaction is complete, successful or not. If the given
501 * given transaction ID matches the current in-progress transaction, the next cohort entry,
502 * if any, is dequeued and processed.
504 * @param transactionID the ID of the completed transaction
505 * @param removeCohortEntry if true the CohortEntry for the transaction is also removed from
508 void currentTransactionComplete(String transactionID, boolean removeCohortEntry) {
509 if(removeCohortEntry) {
510 cohortCache.remove(transactionID);
513 if(isCurrentTransaction(transactionID)) {
514 currentCohortEntry = null;
516 log.debug("{}: currentTransactionComplete: {}", name, transactionID);
518 maybeProcessNextCohortEntry();
522 private void maybeProcessNextCohortEntry() {
523 // Check if there's a next cohort entry waiting in the queue and if it is ready to commit. Also
524 // clean out expired entries.
525 Iterator<CohortEntry> iter = queuedCohortEntries.iterator();
526 while(iter.hasNext()) {
527 CohortEntry next = iter.next();
528 if(next.isReadyToCommit()) {
529 if(currentCohortEntry == null) {
530 if(log.isDebugEnabled()) {
531 log.debug("{}: Next entry to canCommit {}", name, next);
535 currentCohortEntry = next;
536 currentCohortEntry.updateLastAccessTime();
537 doCanCommit(currentCohortEntry);
541 } else if(next.isExpired(cacheExpiryTimeoutInMillis)) {
542 log.warn("{}: canCommit for transaction {} was not received within {} ms - entry removed from cache",
543 name, next.getTransactionID(), cacheExpiryTimeoutInMillis);
544 } else if(!next.isAborted()) {
549 cohortCache.remove(next.getTransactionID());
552 maybeRunOperationOnPendingTransactionsComplete();
555 void cleanupExpiredCohortEntries() {
556 maybeProcessNextCohortEntry();
559 void setRunOnPendingTransactionsComplete(Runnable operation) {
560 runOnPendingTransactionsComplete = operation;
561 maybeRunOperationOnPendingTransactionsComplete();
564 private void maybeRunOperationOnPendingTransactionsComplete() {
565 if(runOnPendingTransactionsComplete != null && currentCohortEntry == null && queuedCohortEntries.isEmpty()) {
566 log.debug("{}: Pending transactions complete - running operation {}", name, runOnPendingTransactionsComplete);
568 runOnPendingTransactionsComplete.run();
569 runOnPendingTransactionsComplete = null;
574 void setCohortDecorator(CohortDecorator cohortDecorator) {
575 this.cohortDecorator = cohortDecorator;
578 static class CohortEntry {
579 private final String transactionID;
580 private ShardDataTreeCohort cohort;
581 private final ReadWriteShardDataTreeTransaction transaction;
582 private RuntimeException lastBatchedModificationsException;
583 private ActorRef replySender;
585 private boolean doImmediateCommit;
586 private final Stopwatch lastAccessTimer = Stopwatch.createStarted();
587 private int totalBatchedModificationsReceived;
588 private boolean aborted;
590 CohortEntry(String transactionID, ReadWriteShardDataTreeTransaction transaction) {
591 this.transaction = Preconditions.checkNotNull(transaction);
592 this.transactionID = transactionID;
595 CohortEntry(String transactionID, ShardDataTreeCohort cohort) {
596 this.transactionID = transactionID;
597 this.cohort = cohort;
598 this.transaction = null;
601 void updateLastAccessTime() {
602 lastAccessTimer.reset();
603 lastAccessTimer.start();
606 String getTransactionID() {
607 return transactionID;
610 DataTreeCandidate getCandidate() {
611 return cohort.getCandidate();
614 int getTotalBatchedModificationsReceived() {
615 return totalBatchedModificationsReceived;
618 RuntimeException getLastBatchedModificationsException() {
619 return lastBatchedModificationsException;
622 void applyModifications(Iterable<Modification> modifications) {
623 totalBatchedModificationsReceived++;
624 if(lastBatchedModificationsException == null) {
625 for (Modification modification : modifications) {
627 modification.apply(transaction.getSnapshot());
628 } catch (RuntimeException e) {
629 lastBatchedModificationsException = e;
636 boolean canCommit() throws InterruptedException, ExecutionException {
637 // We block on the future here (and also preCommit(), commit(), abort()) so we don't have to worry
638 // about possibly accessing our state on a different thread outside of our dispatcher.
639 // TODO: the ShardDataTreeCohort returns immediate Futures anyway which begs the question - why
640 // bother even returning Futures from ShardDataTreeCohort if we have to treat them synchronously
641 // anyway?. The Futures are really a remnant from when we were using the InMemoryDataBroker.
642 return cohort.canCommit().get();
645 void preCommit() throws InterruptedException, ExecutionException {
646 cohort.preCommit().get();
649 void commit() throws InterruptedException, ExecutionException {
650 cohort.commit().get();
653 void abort() throws InterruptedException, ExecutionException {
655 cohort.abort().get();
658 void ready(CohortDecorator cohortDecorator, boolean doImmediateCommit) {
659 Preconditions.checkState(cohort == null, "cohort was already set");
661 setDoImmediateCommit(doImmediateCommit);
663 cohort = transaction.ready();
665 if(cohortDecorator != null) {
666 // Call the hook for unit tests.
667 cohort = cohortDecorator.decorate(transactionID, cohort);
671 boolean isReadyToCommit() {
672 return replySender != null;
675 boolean isExpired(long expireTimeInMillis) {
676 return lastAccessTimer.elapsed(TimeUnit.MILLISECONDS) >= expireTimeInMillis;
679 boolean isDoImmediateCommit() {
680 return doImmediateCommit;
683 void setDoImmediateCommit(boolean doImmediateCommit) {
684 this.doImmediateCommit = doImmediateCommit;
687 ActorRef getReplySender() {
691 void setReplySender(ActorRef replySender) {
692 this.replySender = replySender;
699 void setShard(Shard shard) {
704 boolean isAborted() {
709 public String toString() {
710 StringBuilder builder = new StringBuilder();
711 builder.append("CohortEntry [transactionID=").append(transactionID).append(", doImmediateCommit=")
712 .append(doImmediateCommit).append("]");
713 return builder.toString();