opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/ShardCommitCoordinator.java

   1 /*
   2  * Copyright (c) 2014 Brocade Communications Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.controller.cluster.datastore;
   9
  10 import akka.actor.ActorRef;
  11 import akka.actor.Status;
  12 import akka.serialization.Serialization;
  13 import com.google.common.annotations.VisibleForTesting;
  14 import com.google.common.base.Preconditions;
  15 import com.google.common.cache.Cache;
  16 import com.google.common.cache.CacheBuilder;
  17 import com.google.common.cache.RemovalCause;
  18 import com.google.common.cache.RemovalListener;
  19 import com.google.common.cache.RemovalNotification;
  20 import java.util.LinkedList;
  21 import java.util.Queue;
  22 import java.util.concurrent.ExecutionException;
  23 import java.util.concurrent.TimeUnit;
  24 import org.opendaylight.controller.cluster.datastore.compat.BackwardsCompatibleThreePhaseCommitCohort;
  25 import org.opendaylight.controller.cluster.datastore.messages.BatchedModifications;
  26 import org.opendaylight.controller.cluster.datastore.messages.BatchedModificationsReply;
  27 import org.opendaylight.controller.cluster.datastore.messages.CanCommitTransactionReply;
  28 import org.opendaylight.controller.cluster.datastore.messages.ForwardedReadyTransaction;
  29 import org.opendaylight.controller.cluster.datastore.messages.ReadyTransactionReply;
  30 import org.opendaylight.controller.cluster.datastore.modification.Modification;
  31 import org.opendaylight.controller.cluster.datastore.modification.MutableCompositeModification;
  32 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  33 import org.opendaylight.controller.sal.core.spi.data.DOMStoreThreePhaseCommitCohort;
  34 import org.opendaylight.controller.sal.core.spi.data.DOMStoreWriteTransaction;
  35 import org.slf4j.Logger;
  36
  37 /**
  38  * Coordinates commits for a shard ensuring only one concurrent 3-phase commit.
  39  *
  40  * @author Thomas Pantelis
  41  */
  42 public class ShardCommitCoordinator {
  43
  44     // Interface hook for unit tests to replace or decorate the DOMStoreThreePhaseCommitCohorts.
  45     public interface CohortDecorator {
  46         DOMStoreThreePhaseCommitCohort decorate(String transactionID, DOMStoreThreePhaseCommitCohort actual);
  47     }
  48
  49     private final Cache<String, CohortEntry> cohortCache;
  50
  51     private CohortEntry currentCohortEntry;
  52
  53     private final DOMTransactionFactory transactionFactory;
  54
  55     private final Queue<CohortEntry> queuedCohortEntries;
  56
  57     private int queueCapacity;
  58
  59     private final Logger log;
  60
  61     private final String name;
  62
  63     private final RemovalListener<String, CohortEntry> cacheRemovalListener =
  64             new RemovalListener<String, CohortEntry>() {
  65                 @Override
  66                 public void onRemoval(RemovalNotification<String, CohortEntry> notification) {
  67                     if(notification.getCause() == RemovalCause.EXPIRED) {
  68                         log.warn("{}: Transaction {} was timed out of the cache", name, notification.getKey());
  69                     }
  70                 }
  71             };
  72
  73     // This is a hook for unit tests to replace or decorate the DOMStoreThreePhaseCommitCohorts.
  74     private CohortDecorator cohortDecorator;
  75
  76     private ReadyTransactionReply readyTransactionReply;
  77
  78     public ShardCommitCoordinator(DOMTransactionFactory transactionFactory,
  79             long cacheExpiryTimeoutInSec, int queueCapacity, ActorRef shardActor, Logger log, String name) {
  80
  81         this.queueCapacity = queueCapacity;
  82         this.log = log;
  83         this.name = name;
  84         this.transactionFactory = transactionFactory;
  85
  86         cohortCache = CacheBuilder.newBuilder().expireAfterAccess(cacheExpiryTimeoutInSec, TimeUnit.SECONDS).
  87                 removalListener(cacheRemovalListener).build();
  88
  89         // We use a LinkedList here to avoid synchronization overhead with concurrent queue impls
  90         // since this should only be accessed on the shard's dispatcher.
  91         queuedCohortEntries = new LinkedList<>();
  92     }
  93
  94     public void setQueueCapacity(int queueCapacity) {
  95         this.queueCapacity = queueCapacity;
  96     }
  97
  98     private ReadyTransactionReply readyTransactionReply(Shard shard) {
  99         if(readyTransactionReply == null) {
 100             readyTransactionReply = new ReadyTransactionReply(Serialization.serializedActorPath(shard.self()));
 101         }
 102
 103         return readyTransactionReply;
 104     }
 105
 106     /**
 107      * This method is called to ready a transaction that was prepared by ShardTransaction actor. It caches
 108      * the prepared cohort entry for the given transactions ID in preparation for the subsequent 3-phase commit.
 109      */
 110     public void handleForwardedReadyTransaction(ForwardedReadyTransaction ready, ActorRef sender, Shard shard) {
 111         log.debug("{}: Readying transaction {}, client version {}", name,
 112                 ready.getTransactionID(), ready.getTxnClientVersion());
 113
 114         CohortEntry cohortEntry = new CohortEntry(ready.getTransactionID(), ready.getCohort(),
 115                 (MutableCompositeModification) ready.getModification());
 116         cohortCache.put(ready.getTransactionID(), cohortEntry);
 117
 118         if(ready.getTxnClientVersion() < DataStoreVersions.LITHIUM_VERSION) {
 119             // Return our actor path as we'll handle the three phase commit except if the Tx client
 120             // version < Helium-1 version which means the Tx was initiated by a base Helium version node.
 121             // In that case, the subsequent 3-phase commit messages won't contain the transactionId so to
 122             // maintain backwards compatibility, we create a separate cohort actor to provide the compatible behavior.
 123             ActorRef replyActorPath = shard.self();
 124             if(ready.getTxnClientVersion() < DataStoreVersions.HELIUM_1_VERSION) {
 125                 log.debug("{}: Creating BackwardsCompatibleThreePhaseCommitCohort", name);
 126                 replyActorPath = shard.getContext().actorOf(BackwardsCompatibleThreePhaseCommitCohort.props(
 127                         ready.getTransactionID()));
 128             }
 129
 130             ReadyTransactionReply readyTransactionReply =
 131                     new ReadyTransactionReply(Serialization.serializedActorPath(replyActorPath),
 132                             ready.getTxnClientVersion());
 133             sender.tell(ready.isReturnSerialized() ? readyTransactionReply.toSerializable() :
 134                 readyTransactionReply, shard.self());
 135         } else {
 136             if(ready.isDoImmediateCommit()) {
 137                 cohortEntry.setDoImmediateCommit(true);
 138                 cohortEntry.setReplySender(sender);
 139                 cohortEntry.setShard(shard);
 140                 handleCanCommit(cohortEntry);
 141             } else {
 142                 // The caller does not want immediate commit - the 3-phase commit will be coordinated by the
 143                 // front-end so send back a ReadyTransactionReply with our actor path.
 144                 sender.tell(readyTransactionReply(shard), shard.self());
 145             }
 146         }
 147     }
 148
 149     /**
 150      * This method handles a BatchedModifications message for a transaction being prepared directly on the
 151      * Shard actor instead of via a ShardTransaction actor. If there's no currently cached
 152      * DOMStoreWriteTransaction, one is created. The batched modifications are applied to the write Tx. If
 153      * the BatchedModifications is ready to commit then a DOMStoreThreePhaseCommitCohort is created.
 154      *
 155      * @param batched the BatchedModifications
 156      * @param shardActor the transaction's shard actor
 157      *
 158      * @throws ExecutionException if an error occurs loading the cache
 159      */
 160     boolean handleBatchedModifications(BatchedModifications batched, ActorRef sender, Shard shard)
 161             throws ExecutionException {
 162         CohortEntry cohortEntry = cohortCache.getIfPresent(batched.getTransactionID());
 163         if(cohortEntry == null) {
 164             cohortEntry = new CohortEntry(batched.getTransactionID(),
 165                     transactionFactory.<DOMStoreWriteTransaction>newTransaction(
 166                         TransactionProxy.TransactionType.WRITE_ONLY, batched.getTransactionID(),
 167                         batched.getTransactionChainID()));
 168             cohortCache.put(batched.getTransactionID(), cohortEntry);
 169         }
 170
 171         if(log.isDebugEnabled()) {
 172             log.debug("{}: Applying {} batched modifications for Tx {}", name,
 173                     batched.getModifications().size(), batched.getTransactionID());
 174         }
 175
 176         cohortEntry.applyModifications(batched.getModifications());
 177
 178         if(batched.isReady()) {
 179             if(log.isDebugEnabled()) {
 180                 log.debug("{}: Readying Tx {}, client version {}", name,
 181                         batched.getTransactionID(), batched.getVersion());
 182             }
 183
 184             cohortEntry.ready(cohortDecorator, batched.isDoCommitOnReady());
 185
 186             if(batched.isDoCommitOnReady()) {
 187                 cohortEntry.setReplySender(sender);
 188                 cohortEntry.setShard(shard);
 189                 handleCanCommit(cohortEntry);
 190             } else {
 191                 sender.tell(readyTransactionReply(shard), shard.self());
 192             }
 193         } else {
 194             sender.tell(new BatchedModificationsReply(batched.getModifications().size()), shard.self());
 195         }
 196
 197         return batched.isReady();
 198     }
 199
 200     private void handleCanCommit(CohortEntry cohortEntry) {
 201         String transactionID = cohortEntry.getTransactionID();
 202
 203         if(log.isDebugEnabled()) {
 204             log.debug("{}: Processing canCommit for transaction {} for shard {}",
 205                     name, transactionID, cohortEntry.getShard().self().path());
 206         }
 207
 208         if(currentCohortEntry != null) {
 209             // There's already a Tx commit in progress - attempt to queue this entry to be
 210             // committed after the current Tx completes.
 211             log.debug("{}: Transaction {} is already in progress - queueing transaction {}",
 212                     name, currentCohortEntry.getTransactionID(), transactionID);
 213
 214             if(queuedCohortEntries.size() < queueCapacity) {
 215                 queuedCohortEntries.offer(cohortEntry);
 216             } else {
 217                 removeCohortEntry(transactionID);
 218
 219                 RuntimeException ex = new RuntimeException(
 220                         String.format("%s: Could not enqueue transaction %s - the maximum commit queue"+
 221                                       " capacity %d has been reached.",
 222                                       name, transactionID, queueCapacity));
 223                 log.error(ex.getMessage());
 224                 cohortEntry.getReplySender().tell(new Status.Failure(ex), cohortEntry.getShard().self());
 225             }
 226         } else {
 227             // No Tx commit currently in progress - make this the current entry and proceed with
 228             // canCommit.
 229             cohortEntry.updateLastAccessTime();
 230             currentCohortEntry = cohortEntry;
 231
 232             doCanCommit(cohortEntry);
 233         }
 234     }
 235
 236     /**
 237      * This method handles the canCommit phase for a transaction.
 238      *
 239      * @param canCommit the CanCommitTransaction message
 240      * @param sender the actor that sent the message
 241      * @param shard the transaction's shard actor
 242      */
 243     public void handleCanCommit(String transactionID, final ActorRef sender, final Shard shard) {
 244         // Lookup the cohort entry that was cached previously (or should have been) by
 245         // transactionReady (via the ForwardedReadyTransaction message).
 246         final CohortEntry cohortEntry = cohortCache.getIfPresent(transactionID);
 247         if(cohortEntry == null) {
 248             // Either canCommit was invoked before ready(shouldn't happen)  or a long time passed
 249             // between canCommit and ready and the entry was expired from the cache.
 250             IllegalStateException ex = new IllegalStateException(
 251                     String.format("%s: No cohort entry found for transaction %s", name, transactionID));
 252             log.error(ex.getMessage());
 253             sender.tell(new Status.Failure(ex), shard.self());
 254             return;
 255         }
 256
 257         cohortEntry.setReplySender(sender);
 258         cohortEntry.setShard(shard);
 259
 260         handleCanCommit(cohortEntry);
 261     }
 262
 263     private void doCanCommit(final CohortEntry cohortEntry) {
 264
 265         boolean canCommit = false;
 266         try {
 267             // We block on the future here so we don't have to worry about possibly accessing our
 268             // state on a different thread outside of our dispatcher. Also, the data store
 269             // currently uses a same thread executor anyway.
 270             canCommit = cohortEntry.getCohort().canCommit().get();
 271
 272             if(cohortEntry.isDoImmediateCommit()) {
 273                 if(canCommit) {
 274                     doCommit(cohortEntry);
 275                 } else {
 276                     cohortEntry.getReplySender().tell(new Status.Failure(new TransactionCommitFailedException(
 277                                 "Can Commit failed, no detailed cause available.")), cohortEntry.getShard().self());
 278                 }
 279             } else {
 280                 cohortEntry.getReplySender().tell(
 281                         canCommit ? CanCommitTransactionReply.YES.toSerializable() :
 282                             CanCommitTransactionReply.NO.toSerializable(), cohortEntry.getShard().self());
 283             }
 284         } catch (Exception e) {
 285             log.debug("{}: An exception occurred during canCommit: {}", name, e);
 286
 287             Throwable failure = e;
 288             if(e instanceof ExecutionException) {
 289                 failure = e.getCause();
 290             }
 291
 292             cohortEntry.getReplySender().tell(new Status.Failure(failure), cohortEntry.getShard().self());
 293         } finally {
 294             if(!canCommit) {
 295                 // Remove the entry from the cache now.
 296                 currentTransactionComplete(cohortEntry.getTransactionID(), true);
 297             }
 298         }
 299     }
 300
 301     private boolean doCommit(CohortEntry cohortEntry) {
 302         log.debug("{}: Committing transaction {}", name, cohortEntry.getTransactionID());
 303
 304         boolean success = false;
 305
 306         // We perform the preCommit phase here atomically with the commit phase. This is an
 307         // optimization to eliminate the overhead of an extra preCommit message. We lose front-end
 308         // coordination of preCommit across shards in case of failure but preCommit should not
 309         // normally fail since we ensure only one concurrent 3-phase commit.
 310
 311         try {
 312             // We block on the future here so we don't have to worry about possibly accessing our
 313             // state on a different thread outside of our dispatcher. Also, the data store
 314             // currently uses a same thread executor anyway.
 315             cohortEntry.getCohort().preCommit().get();
 316
 317             cohortEntry.getShard().continueCommit(cohortEntry);
 318
 319             cohortEntry.updateLastAccessTime();
 320
 321             success = true;
 322         } catch (Exception e) {
 323             log.error("{} An exception occurred while preCommitting transaction {}",
 324                     name, cohortEntry.getTransactionID(), e);
 325             cohortEntry.getReplySender().tell(new akka.actor.Status.Failure(e), cohortEntry.getShard().self());
 326
 327             currentTransactionComplete(cohortEntry.getTransactionID(), true);
 328         }
 329
 330         return success;
 331     }
 332
 333     boolean handleCommit(final String transactionID, final ActorRef sender, final Shard shard) {
 334         // Get the current in-progress cohort entry in the commitCoordinator if it corresponds to
 335         // this transaction.
 336         final CohortEntry cohortEntry = getCohortEntryIfCurrent(transactionID);
 337         if(cohortEntry == null) {
 338             // We're not the current Tx - the Tx was likely expired b/c it took too long in
 339             // between the canCommit and commit messages.
 340             IllegalStateException ex = new IllegalStateException(
 341                     String.format("%s: Cannot commit transaction %s - it is not the current transaction",
 342                             name, transactionID));
 343             log.error(ex.getMessage());
 344             sender.tell(new akka.actor.Status.Failure(ex), shard.self());
 345             return false;
 346         }
 347
 348         return doCommit(cohortEntry);
 349     }
 350
 351     /**
 352      * Returns the cohort entry for the Tx commit currently in progress if the given transaction ID
 353      * matches the current entry.
 354      *
 355      * @param transactionID the ID of the transaction
 356      * @return the current CohortEntry or null if the given transaction ID does not match the
 357      *         current entry.
 358      */
 359     public CohortEntry getCohortEntryIfCurrent(String transactionID) {
 360         if(isCurrentTransaction(transactionID)) {
 361             return currentCohortEntry;
 362         }
 363
 364         return null;
 365     }
 366
 367     public CohortEntry getCurrentCohortEntry() {
 368         return currentCohortEntry;
 369     }
 370
 371     public CohortEntry getAndRemoveCohortEntry(String transactionID) {
 372         CohortEntry cohortEntry = cohortCache.getIfPresent(transactionID);
 373         cohortCache.invalidate(transactionID);
 374         return cohortEntry;
 375     }
 376
 377     public void removeCohortEntry(String transactionID) {
 378         cohortCache.invalidate(transactionID);
 379     }
 380
 381     public boolean isCurrentTransaction(String transactionID) {
 382         return currentCohortEntry != null &&
 383                 currentCohortEntry.getTransactionID().equals(transactionID);
 384     }
 385
 386     /**
 387      * This method is called when a transaction is complete, successful or not. If the given
 388      * given transaction ID matches the current in-progress transaction, the next cohort entry,
 389      * if any, is dequeued and processed.
 390      *
 391      * @param transactionID the ID of the completed transaction
 392      * @param removeCohortEntry if true the CohortEntry for the transaction is also removed from
 393      *        the cache.
 394      */
 395     public void currentTransactionComplete(String transactionID, boolean removeCohortEntry) {
 396         if(removeCohortEntry) {
 397             removeCohortEntry(transactionID);
 398         }
 399
 400         if(isCurrentTransaction(transactionID)) {
 401             // Dequeue the next cohort entry waiting in the queue.
 402             currentCohortEntry = queuedCohortEntries.poll();
 403             if(currentCohortEntry != null) {
 404                 currentCohortEntry.updateLastAccessTime();
 405                 doCanCommit(currentCohortEntry);
 406             }
 407         }
 408     }
 409
 410     @VisibleForTesting
 411     void setCohortDecorator(CohortDecorator cohortDecorator) {
 412         this.cohortDecorator = cohortDecorator;
 413     }
 414
 415
 416     static class CohortEntry {
 417         private final String transactionID;
 418         private DOMStoreThreePhaseCommitCohort cohort;
 419         private final MutableCompositeModification compositeModification;
 420         private final DOMStoreWriteTransaction transaction;
 421         private ActorRef replySender;
 422         private Shard shard;
 423         private long lastAccessTime;
 424         private boolean doImmediateCommit;
 425
 426         CohortEntry(String transactionID, DOMStoreWriteTransaction transaction) {
 427             this.compositeModification = new MutableCompositeModification();
 428             this.transaction = transaction;
 429             this.transactionID = transactionID;
 430         }
 431
 432         CohortEntry(String transactionID, DOMStoreThreePhaseCommitCohort cohort,
 433                 MutableCompositeModification compositeModification) {
 434             this.transactionID = transactionID;
 435             this.cohort = cohort;
 436             this.compositeModification = compositeModification;
 437             this.transaction = null;
 438         }
 439
 440         void updateLastAccessTime() {
 441             lastAccessTime = System.currentTimeMillis();
 442         }
 443
 444         long getLastAccessTime() {
 445             return lastAccessTime;
 446         }
 447
 448         String getTransactionID() {
 449             return transactionID;
 450         }
 451
 452         DOMStoreThreePhaseCommitCohort getCohort() {
 453             return cohort;
 454         }
 455
 456         MutableCompositeModification getModification() {
 457             return compositeModification;
 458         }
 459
 460         void applyModifications(Iterable<Modification> modifications) {
 461             for(Modification modification: modifications) {
 462                 compositeModification.addModification(modification);
 463                 modification.apply(transaction);
 464             }
 465         }
 466
 467         void ready(CohortDecorator cohortDecorator, boolean doImmediateCommit) {
 468             Preconditions.checkState(cohort == null, "cohort was already set");
 469
 470             setDoImmediateCommit(doImmediateCommit);
 471
 472             cohort = transaction.ready();
 473
 474             if(cohortDecorator != null) {
 475                 // Call the hook for unit tests.
 476                 cohort = cohortDecorator.decorate(transactionID, cohort);
 477             }
 478         }
 479
 480         boolean isDoImmediateCommit() {
 481             return doImmediateCommit;
 482         }
 483
 484         void setDoImmediateCommit(boolean doImmediateCommit) {
 485             this.doImmediateCommit = doImmediateCommit;
 486         }
 487
 488         ActorRef getReplySender() {
 489             return replySender;
 490         }
 491
 492         void setReplySender(ActorRef replySender) {
 493             this.replySender = replySender;
 494         }
 495
 496         Shard getShard() {
 497             return shard;
 498         }
 499
 500         void setShard(Shard shard) {
 501             this.shard = shard;
 502         }
 503
 504         boolean hasModifications(){
 505             return compositeModification.getModifications().size() > 0;
 506         }
 507     }
 508 }