X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=opendaylight%2Fmd-sal%2Fsal-distributed-datastore%2Fsrc%2Fmain%2Fjava%2Forg%2Fopendaylight%2Fcontroller%2Fcluster%2Fdatastore%2FShard.java;h=72b31d5598d80aaf7f7961e33c562d72dbbb7bc1;hb=4f64d27f5434ed7bc82de2b6f6a57ba13ccaec4b;hp=5b8a50e18896c0a3e10db64c10b1d6fdffdc41d8;hpb=a51163ead1d60e66eeaf3691adb70b019ce60fb2;p=controller.git diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/Shard.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/Shard.java index 5b8a50e188..72b31d5598 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/Shard.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/Shard.java @@ -12,12 +12,12 @@ import akka.actor.ActorRef; import akka.actor.ActorSelection; import akka.actor.Cancellable; import akka.actor.Props; -import akka.persistence.RecoveryFailure; import akka.serialization.Serialization; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Optional; import com.google.common.base.Preconditions; import java.io.IOException; +import java.util.Collection; import java.util.Collections; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -50,7 +50,6 @@ import org.opendaylight.controller.cluster.datastore.messages.RegisterDataTreeCh import org.opendaylight.controller.cluster.datastore.messages.ShardLeaderStateChanged; import org.opendaylight.controller.cluster.datastore.messages.UpdateSchemaContext; import org.opendaylight.controller.cluster.datastore.modification.Modification; -import org.opendaylight.controller.cluster.datastore.modification.ModificationPayload; import org.opendaylight.controller.cluster.datastore.modification.MutableCompositeModification; import org.opendaylight.controller.cluster.datastore.utils.Dispatchers; import org.opendaylight.controller.cluster.datastore.utils.MessageTracker; @@ -135,7 +134,9 @@ public class Shard extends RaftActor { LOG.info("Shard created : {}, persistent : {}", name, datastoreContext.isPersistent()); - store = new ShardDataTree(builder.getSchemaContext(), builder.getTreeType()); + store = new ShardDataTree(builder.getSchemaContext(), builder.getTreeType(), + new ShardDataTreeChangeListenerPublisherActorProxy(getContext(), name + "-DTCL-publisher"), + new ShardDataChangeListenerPublisherActorProxy(getContext(), name + "-DCL-publisher"), name); shardMBean = ShardMBeanFactory.getShardStatsMBean(name.toString(), datastoreContext.getDataStoreMXBeanType()); @@ -189,33 +190,24 @@ public class Shard extends RaftActor { txCommitTimeoutCheckSchedule.cancel(); } + commitCoordinator.abortPendingTransactions("Transaction aborted due to shutdown.", this); + shardMBean.unregisterMBean(); } @Override - public void onReceiveRecover(final Object message) throws Exception { - if(LOG.isDebugEnabled()) { - LOG.debug("{}: onReceiveRecover: Received message {} from {}", persistenceId(), - message.getClass().toString(), getSender()); - } + protected void handleRecover(final Object message) { + LOG.debug("{}: onReceiveRecover: Received message {} from {}", persistenceId(), message.getClass(), + getSender()); - if (message instanceof RecoveryFailure){ - LOG.error("{}: Recovery failed because of this cause", - persistenceId(), ((RecoveryFailure) message).cause()); - - // Even though recovery failed, we still need to finish our recovery, eg send the - // ActorInitialized message and start the txCommitTimeoutCheckSchedule. - onRecoveryComplete(); - } else { - super.onReceiveRecover(message); - if(LOG.isTraceEnabled()) { - appendEntriesReplyTracker.begin(); - } + super.handleRecover(message); + if (LOG.isTraceEnabled()) { + appendEntriesReplyTracker.begin(); } } @Override - public void onReceiveCommand(final Object message) throws Exception { + protected void handleCommand(final Object message) { MessageTracker.Context context = appendEntriesReplyTracker.received(message); @@ -225,21 +217,21 @@ public class Shard extends RaftActor { } try { - if (CreateTransaction.SERIALIZABLE_CLASS.isInstance(message)) { + if (CreateTransaction.isSerializedType(message)) { handleCreateTransaction(message); - } else if (BatchedModifications.class.isInstance(message)) { + } else if (message instanceof BatchedModifications) { handleBatchedModifications((BatchedModifications)message); } else if (message instanceof ForwardedReadyTransaction) { handleForwardedReadyTransaction((ForwardedReadyTransaction) message); } else if (message instanceof ReadyLocalTransaction) { handleReadyLocalTransaction((ReadyLocalTransaction)message); - } else if (CanCommitTransaction.SERIALIZABLE_CLASS.isInstance(message)) { + } else if (CanCommitTransaction.isSerializedType(message)) { handleCanCommitTransaction(CanCommitTransaction.fromSerializable(message)); - } else if (CommitTransaction.SERIALIZABLE_CLASS.isInstance(message)) { + } else if (CommitTransaction.isSerializedType(message)) { handleCommitTransaction(CommitTransaction.fromSerializable(message)); - } else if (AbortTransaction.SERIALIZABLE_CLASS.isInstance(message)) { + } else if (AbortTransaction.isSerializedType(message)) { handleAbortTransaction(AbortTransaction.fromSerializable(message)); - } else if (CloseTransactionChain.SERIALIZABLE_CLASS.isInstance(message)) { + } else if (CloseTransactionChain.isSerializedType(message)) { closeTransactionChain(CloseTransactionChain.fromSerializable(message)); } else if (message instanceof RegisterChangeListener) { changeSupport.onMessage((RegisterChangeListener) message, isLeader(), hasLeader()); @@ -252,7 +244,7 @@ public class Shard extends RaftActor { setPeerAddress(resolved.getPeerId().toString(), resolved.getPeerAddress()); } else if (message.equals(TX_COMMIT_TIMEOUT_CHECK_MESSAGE)) { - handleTransactionCommitTimeoutCheck(); + commitCoordinator.checkForExpiredTransactions(transactionCommitTimeout, this); } else if(message instanceof DatastoreContext) { onDatastoreContext((DatastoreContext)message); } else if(message instanceof RegisterRoleChangeListener){ @@ -269,7 +261,7 @@ public class Shard extends RaftActor { } else if(ShardTransactionMessageRetrySupport.TIMER_MESSAGE_CLASS.isInstance(message)) { messageRetrySupport.onTimerMessage(message); } else { - super.onReceiveCommand(message); + super.handleCommand(message); } } finally { context.done(); @@ -284,6 +276,10 @@ public class Shard extends RaftActor { return commitCoordinator.getQueueSize(); } + public int getCohortCacheSize() { + return commitCoordinator.getCohortCacheSize(); + } + @Override protected Optional getRoleChangeNotifier() { return roleChangeNotifier; @@ -312,20 +308,6 @@ public class Shard extends RaftActor { updateConfigParams(datastoreContext.getShardRaftConfig()); } - private void handleTransactionCommitTimeoutCheck() { - CohortEntry cohortEntry = commitCoordinator.getCurrentCohortEntry(); - if(cohortEntry != null) { - if(cohortEntry.isExpired(transactionCommitTimeout)) { - LOG.warn("{}: Current transaction {} has timed out after {} ms - aborting", - persistenceId(), cohortEntry.getTransactionID(), transactionCommitTimeout); - - doAbortTransaction(cohortEntry.getTransactionID(), null); - } - } - - commitCoordinator.cleanupExpiredCohortEntries(); - } - private static boolean isEmptyCommit(final DataTreeCandidate candidate) { return ModificationType.UNMODIFIED.equals(candidate.getRootNode().getModificationType()); } @@ -356,7 +338,7 @@ public class Shard extends RaftActor { try { cohortEntry.commit(); - sender.tell(CommitTransactionReply.INSTANCE.toSerializable(), getSelf()); + sender.tell(CommitTransactionReply.instance(cohortEntry.getClientVersion()).toSerializable(), getSelf()); shardMBean.incrementCommittedTransactionCount(); shardMBean.setLastCommittedTransactionTime(System.currentTimeMillis()); @@ -393,7 +375,8 @@ public class Shard extends RaftActor { LOG.error("{}: Failed to re-apply transaction {}", persistenceId(), transactionID, e); } - sender.tell(CommitTransactionReply.INSTANCE.toSerializable(), getSelf()); + sender.tell(CommitTransactionReply.instance(cohortEntry.getClientVersion()).toSerializable(), + getSelf()); } else { // This really shouldn't happen - it likely means that persistence or replication // took so long to complete such that the cohort entry was expired from the cache. @@ -436,21 +419,28 @@ public class Shard extends RaftActor { // the primary/leader shard. However with timing and caching on the front-end, there's a small // window where it could have a stale leader during leadership transitions. // - if(isLeader()) { - failIfIsolatedLeader(getSender()); - + boolean isLeaderActive = isLeaderActive(); + if (isLeader() && isLeaderActive) { handleBatchedModificationsLocal(batched, getSender()); } else { ActorSelection leader = getLeader(); - if(leader != null) { - // TODO: what if this is not the first batch and leadership changed in between batched messages? - // We could check if the commitCoordinator already has a cached entry and forward all the previous - // batched modifications. - LOG.debug("{}: Forwarding BatchedModifications to leader {}", persistenceId(), leader); - leader.forward(batched, getContext()); - } else { + if (!isLeaderActive || leader == null) { messageRetrySupport.addMessageToRetry(batched, getSender(), "Could not commit transaction " + batched.getTransactionID()); + } else { + // If this is not the first batch and leadership changed in between batched messages, + // we need to reconstruct previous BatchedModifications from the transaction + // DataTreeModification, honoring the max batched modification count, and forward all the + // previous BatchedModifications to the new leader. + Collection newModifications = commitCoordinator.createForwardedBatchedModifications( + batched, datastoreContext.getShardBatchedModificationCount()); + + LOG.debug("{}: Forwarding {} BatchedModifications to leader {}", persistenceId(), + newModifications.size(), leader); + + for(BatchedModifications bm: newModifications) { + leader.forward(bm, getContext()); + } } } } @@ -472,9 +462,10 @@ public class Shard extends RaftActor { } private void handleReadyLocalTransaction(final ReadyLocalTransaction message) { - if (isLeader()) { - failIfIsolatedLeader(getSender()); + LOG.debug("{}: handleReadyLocalTransaction for {}", persistenceId(), message.getTransactionID()); + boolean isLeaderActive = isLeaderActive(); + if (isLeader() && isLeaderActive) { try { commitCoordinator.handleReadyLocalTransaction(message, getSender(), this); } catch (Exception e) { @@ -484,34 +475,35 @@ public class Shard extends RaftActor { } } else { ActorSelection leader = getLeader(); - if (leader != null) { + if (!isLeaderActive || leader == null) { + messageRetrySupport.addMessageToRetry(message, getSender(), + "Could not commit transaction " + message.getTransactionID()); + } else { LOG.debug("{}: Forwarding ReadyLocalTransaction to leader {}", persistenceId(), leader); message.setRemoteVersion(getCurrentBehavior().getLeaderPayloadVersion()); leader.forward(message, getContext()); - } else { - messageRetrySupport.addMessageToRetry(message, getSender(), - "Could not commit transaction " + message.getTransactionID()); } } } private void handleForwardedReadyTransaction(ForwardedReadyTransaction forwardedReady) { - if (isLeader()) { - failIfIsolatedLeader(getSender()); + LOG.debug("{}: handleForwardedReadyTransaction for {}", persistenceId(), forwardedReady.getTransactionID()); + boolean isLeaderActive = isLeaderActive(); + if (isLeader() && isLeaderActive) { commitCoordinator.handleForwardedReadyTransaction(forwardedReady, getSender(), this); } else { ActorSelection leader = getLeader(); - if (leader != null) { + if (!isLeaderActive || leader == null) { + messageRetrySupport.addMessageToRetry(forwardedReady, getSender(), + "Could not commit transaction " + forwardedReady.getTransactionID()); + } else { LOG.debug("{}: Forwarding ForwardedReadyTransaction to leader {}", persistenceId(), leader); ReadyLocalTransaction readyLocal = new ReadyLocalTransaction(forwardedReady.getTransactionID(), forwardedReady.getTransaction().getSnapshot(), forwardedReady.isDoImmediateCommit()); readyLocal.setRemoteVersion(getCurrentBehavior().getLeaderPayloadVersion()); leader.forward(readyLocal, getContext()); - } else { - messageRetrySupport.addMessageToRetry(forwardedReady, getSender(), - "Could not commit transaction " + forwardedReady.getTransactionID()); } } } @@ -540,11 +532,10 @@ public class Shard extends RaftActor { } private ActorRef createTypedTransactionActor(int transactionType, - ShardTransactionIdentifier transactionId, String transactionChainId, - short clientVersion ) { + ShardTransactionIdentifier transactionId, String transactionChainId) { return transactionActorFactory.newShardTransaction(TransactionType.fromInt(transactionType), - transactionId, transactionChainId, clientVersion); + transactionId, transactionChainId); } private void createTransaction(CreateTransaction createTransaction) { @@ -555,18 +546,17 @@ public class Shard extends RaftActor { } ActorRef transactionActor = createTransaction(createTransaction.getTransactionType(), - createTransaction.getTransactionId(), createTransaction.getTransactionChainId(), - createTransaction.getVersion()); + createTransaction.getTransactionId(), createTransaction.getTransactionChainId()); getSender().tell(new CreateTransactionReply(Serialization.serializedActorPath(transactionActor), - createTransaction.getTransactionId()).toSerializable(), getSelf()); + createTransaction.getTransactionId(), createTransaction.getVersion()).toSerializable(), getSelf()); } catch (Exception e) { getSender().tell(new akka.actor.Status.Failure(e), getSelf()); } } private ActorRef createTransaction(int transactionType, String remoteTransactionId, - String transactionChainId, short clientVersion) { + String transactionChainId) { ShardTransactionIdentifier transactionId = new ShardTransactionIdentifier(remoteTransactionId); @@ -576,7 +566,7 @@ public class Shard extends RaftActor { } ActorRef transactionActor = createTypedTransactionActor(transactionType, transactionId, - transactionChainId, clientVersion); + transactionChainId); return transactionActor; } @@ -653,12 +643,6 @@ public class Shard extends RaftActor { // Replication consensus reached, proceed to commit finishCommit(clientActor, identifier); } - } else if (data instanceof ModificationPayload) { - try { - applyModificationToState(clientActor, identifier, ((ModificationPayload) data).getModification()); - } catch (ClassNotFoundException | IOException e) { - LOG.error("{}: Error extracting ModificationPayload", persistenceId(), e); - } } else if (data instanceof CompositeModificationPayload) { Object modification = ((CompositeModificationPayload) data).getModification(); @@ -705,6 +689,13 @@ public class Shard extends RaftActor { } store.closeAllTransactionChains(); + + commitCoordinator.abortPendingTransactions( + "The transacton was aborted due to inflight leadership change.", this); + } + + if(hasLeader && !isIsolatedLeader()) { + messageRetrySupport.retryMessages(); } } @@ -712,11 +703,17 @@ public class Shard extends RaftActor { protected void onLeaderChanged(String oldLeader, String newLeader) { shardMBean.incrementLeadershipChangeCount(); - if(hasLeader()) { + if(hasLeader() && !isIsolatedLeader()) { messageRetrySupport.retryMessages(); } } + @Override + protected void pauseLeader(Runnable operation) { + LOG.debug("{}: In pauseLeader, operation: {}", persistenceId(), operation); + commitCoordinator.setRunOnPendingTransactionsComplete(operation); + } + @Override public String persistenceId() { return this.name;