X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=opendaylight%2Fmd-sal%2Fsal-akka-raft%2Fsrc%2Fmain%2Fjava%2Forg%2Fopendaylight%2Fcontroller%2Fcluster%2Fraft%2FRaftActor.java;h=aea1e9fad427a114ec99cd884de7c3c8c80dd7e0;hb=497d356874c8033e0d0f9fccfa7ed6695d6889fc;hp=e9e0d630d092eeffa1833441936c26cfeea40d17;hpb=766df8b58cc1decd7b40e0adbf41d3657fc21f2c;p=controller.git diff --git a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/RaftActor.java b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/RaftActor.java index e9e0d630d0..aea1e9fad4 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/RaftActor.java +++ b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/RaftActor.java @@ -11,8 +11,8 @@ package org.opendaylight.controller.cluster.raft; import akka.actor.ActorRef; import akka.actor.ActorSelection; +import akka.actor.PoisonPill; import akka.japi.Procedure; -import akka.persistence.SnapshotSelectionCriteria; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Objects; import com.google.common.base.Optional; @@ -36,6 +36,7 @@ import org.opendaylight.controller.cluster.notifications.RoleChanged; import org.opendaylight.controller.cluster.raft.base.messages.ApplyJournalEntries; import org.opendaylight.controller.cluster.raft.base.messages.ApplyState; import org.opendaylight.controller.cluster.raft.base.messages.InitiateCaptureSnapshot; +import org.opendaylight.controller.cluster.raft.base.messages.LeaderTransitioning; import org.opendaylight.controller.cluster.raft.base.messages.Replicate; import org.opendaylight.controller.cluster.raft.base.messages.SwitchBehavior; import org.opendaylight.controller.cluster.raft.behaviors.AbstractLeader; @@ -47,6 +48,7 @@ import org.opendaylight.controller.cluster.raft.client.messages.FindLeaderReply; import org.opendaylight.controller.cluster.raft.client.messages.FollowerInfo; import org.opendaylight.controller.cluster.raft.client.messages.GetOnDemandRaftState; import org.opendaylight.controller.cluster.raft.client.messages.OnDemandRaftState; +import org.opendaylight.controller.cluster.raft.client.messages.Shutdown; import org.opendaylight.controller.cluster.raft.protobuff.client.messages.Payload; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -109,7 +111,9 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { */ private final RaftActorContextImpl context; - private final DelegatingPersistentDataProvider delegatingPersistenceProvider = new DelegatingPersistentDataProvider(null); + private final DelegatingPersistentDataProvider delegatingPersistenceProvider; + + private final PersistentDataProvider persistentProvider; private RaftActorRecoverySupport raftRecovery; @@ -119,11 +123,20 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { private final SwitchBehaviorSupplier reusableSwitchBehaviorSupplier = new SwitchBehaviorSupplier(); + private RaftActorServerConfigurationSupport serverConfigurationSupport; + + private RaftActorLeadershipTransferCohort leadershipTransferInProgress; + + private boolean shuttingDown; + public RaftActor(String id, Map peerAddresses, Optional configParams, short payloadVersion) { + persistentProvider = new PersistentDataProvider(this); + delegatingPersistenceProvider = new RaftActorDelegatingPersistentDataProvider(null, persistentProvider); + context = new RaftActorContextImpl(this.getSelf(), - this.getContext(), id, new ElectionTermImpl(delegatingPersistenceProvider, id, LOG), + this.getContext(), id, new ElectionTermImpl(persistentProvider, id, LOG), -1, -1, peerAddresses, (configParams.isPresent() ? configParams.get(): new DefaultConfigParamsImpl()), delegatingPersistenceProvider, LOG); @@ -140,6 +153,7 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { super.preStart(); snapshotSupport = newRaftActorSnapshotMessageSupport(); + serverConfigurationSupport = new RaftActorServerConfigurationSupport(this); } @Override @@ -161,22 +175,20 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { raftRecovery = newRaftActorRecoverySupport(); } - boolean recoveryComplete = raftRecovery.handleRecoveryMessage(message); + boolean recoveryComplete = raftRecovery.handleRecoveryMessage(message, persistentProvider); if(recoveryComplete) { - if(!persistence().isRecoveryApplicable()) { - // Delete all the messages from the akka journal so that we do not end up with consistency issues - // Note I am not using the dataPersistenceProvider and directly using the akka api here - deleteMessages(lastSequenceNr()); - - // Delete all the akka snapshots as they will not be needed - deleteSnapshots(new SnapshotSelectionCriteria(scala.Long.MaxValue(), scala.Long.MaxValue())); - } - onRecoveryComplete(); initializeBehavior(); raftRecovery = null; + + if (context.getReplicatedLog().size() > 0) { + self().tell(new InitiateCaptureSnapshot(), self()); + LOG.info("{}: Snapshot capture initiated after recovery", persistenceId()); + } else { + LOG.info("{}: Snapshot capture NOT initiated after recovery, journal empty", persistenceId()); + } } } @@ -196,7 +208,9 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { @Override public void handleCommand(final Object message) { - if (message instanceof ApplyState){ + if(serverConfigurationSupport.handleMessage(message, getSender())) { + return; + } else if (message instanceof ApplyState){ ApplyState applyState = (ApplyState) message; long elapsedTime = (System.nanoTime() - applyState.getStartTime()); @@ -243,11 +257,90 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { captureSnapshot(); } else if(message instanceof SwitchBehavior){ switchBehavior(((SwitchBehavior) message)); - } else if(!snapshotSupport.handleSnapshotMessage(message)) { + } else if(message instanceof LeaderTransitioning) { + onLeaderTransitioning(); + } else if(message instanceof Shutdown) { + onShutDown(); + } else if(message instanceof Runnable) { + ((Runnable)message).run(); + } else if(!snapshotSupport.handleSnapshotMessage(message, getSender())) { switchBehavior(reusableSwitchBehaviorSupplier.handleMessage(getSender(), message)); } } + private void initiateLeadershipTransfer(final RaftActorLeadershipTransferCohort.OnComplete onComplete) { + LOG.debug("{}: Initiating leader transfer", persistenceId()); + + if(leadershipTransferInProgress == null) { + leadershipTransferInProgress = new RaftActorLeadershipTransferCohort(this, getSender()); + leadershipTransferInProgress.addOnComplete(new RaftActorLeadershipTransferCohort.OnComplete() { + @Override + public void onSuccess(ActorRef raftActorRef, ActorRef replyTo) { + leadershipTransferInProgress = null; + } + + @Override + public void onFailure(ActorRef raftActorRef, ActorRef replyTo) { + leadershipTransferInProgress = null; + } + }); + + leadershipTransferInProgress.addOnComplete(onComplete); + leadershipTransferInProgress.init(); + } else { + LOG.debug("{}: prior leader transfer in progress - adding callback", persistenceId()); + leadershipTransferInProgress.addOnComplete(onComplete); + } + } + + private void onShutDown() { + LOG.debug("{}: onShutDown", persistenceId()); + + if(shuttingDown) { + return; + } + + shuttingDown = true; + if(currentBehavior.state() == RaftState.Leader && context.hasFollowers()) { + initiateLeadershipTransfer(new RaftActorLeadershipTransferCohort.OnComplete() { + @Override + public void onSuccess(ActorRef raftActorRef, ActorRef replyTo) { + LOG.debug("{}: leader transfer succeeded - sending PoisonPill", persistenceId()); + raftActorRef.tell(PoisonPill.getInstance(), raftActorRef); + } + + @Override + public void onFailure(ActorRef raftActorRef, ActorRef replyTo) { + LOG.debug("{}: leader transfer failed - sending PoisonPill", persistenceId()); + raftActorRef.tell(PoisonPill.getInstance(), raftActorRef); + } + }); + } else if(currentBehavior.state() == RaftState.Leader) { + pauseLeader(new TimedRunnable(context.getConfigParams().getElectionTimeOutInterval(), this) { + @Override + protected void doRun() { + self().tell(PoisonPill.getInstance(), self()); + } + + @Override + protected void doCancel() { + self().tell(PoisonPill.getInstance(), self()); + } + }); + } else { + self().tell(PoisonPill.getInstance(), self()); + } + } + + private void onLeaderTransitioning() { + LOG.debug("{}: onLeaderTransitioning", persistenceId()); + Optional roleChangeNotifier = getRoleChangeNotifier(); + if(currentBehavior.state() == RaftState.Follower && roleChangeNotifier.isPresent()) { + roleChangeNotifier.get().tell(newLeaderStateChanged(getId(), null, + currentBehavior.getLeaderPayloadVersion()), getSelf()); + } + } + private void switchBehavior(SwitchBehavior message) { if(!getRaftActorContext().getRaftPolicy().automaticElectionsEnabled()) { RaftState newState = message.getNewState(); @@ -276,6 +369,11 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { private void onGetOnDemandRaftStats() { // Debugging message to retrieve raft stats. + Map peerAddresses = new HashMap<>(); + for(String peerId: context.getPeerIds()) { + peerAddresses.put(peerId, context.getPeerAddress(peerId)); + } + OnDemandRaftState.Builder builder = OnDemandRaftState.builder() .commitIndex(context.getCommitIndex()) .currentTerm(context.getTermInformation().getCurrentTerm()) @@ -291,7 +389,8 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { .snapshotIndex(replicatedLog().getSnapshotIndex()) .snapshotTerm(replicatedLog().getSnapshotTerm()) .votedFor(context.getTermInformation().getVotedFor()) - .peerAddresses(new HashMap<>(context.getPeerAddresses())); + .peerAddresses(peerAddresses) + .customRaftPolicyClassName(context.getConfigParams().getCustomRaftPolicyImplementationClass()); ReplicatedLogEntry lastLogEntry = getLastLogEntry(); if (lastLogEntry != null) { @@ -323,19 +422,23 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { onStateChanged(); } - String oldBehaviorLeaderId = oldBehavior == null ? null : oldBehaviorState.getLeaderId(); + String lastValidLeaderId = oldBehavior == null ? null : oldBehaviorState.getLastValidLeaderId(); String oldBehaviorStateName = oldBehavior == null ? null : oldBehavior.state().name(); // it can happen that the state has not changed but the leader has changed. Optional roleChangeNotifier = getRoleChangeNotifier(); - if(!Objects.equal(oldBehaviorLeaderId, currentBehavior.getLeaderId()) || + if(!Objects.equal(lastValidLeaderId, currentBehavior.getLeaderId()) || oldBehaviorState.getLeaderPayloadVersion() != currentBehavior.getLeaderPayloadVersion()) { if(roleChangeNotifier.isPresent()) { roleChangeNotifier.get().tell(newLeaderStateChanged(getId(), currentBehavior.getLeaderId(), currentBehavior.getLeaderPayloadVersion()), getSelf()); } - onLeaderChanged(oldBehaviorLeaderId, currentBehavior.getLeaderId()); + onLeaderChanged(lastValidLeaderId, currentBehavior.getLeaderId()); + + if(leadershipTransferInProgress != null) { + leadershipTransferInProgress.onNewLeader(currentBehavior.getLeaderId()); + } } if (roleChangeNotifier.isPresent() && @@ -382,7 +485,7 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { replicatedLog().appendAndPersist(replicatedLogEntry, new Procedure() { @Override - public void apply(ReplicatedLogEntry replicatedLogEntry) throws Exception { + public void apply(ReplicatedLogEntry replicatedLogEntry) { if (!hasFollowers()){ // Increment the Commit Index and the Last Applied values raftContext.setCommitIndex(replicatedLogEntry.getIndex()); @@ -433,6 +536,15 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { return context.getId().equals(currentBehavior.getLeaderId()); } + protected boolean isLeaderActive() { + return currentBehavior.state() != RaftState.IsolatedLeader && !shuttingDown && + !isLeadershipTransferInProgress(); + } + + private boolean isLeadershipTransferInProgress() { + return leadershipTransferInProgress != null && leadershipTransferInProgress.isTransferring(); + } + /** * Derived actor can call getLeader if they need a reference to the Leader. * This would be useful for example in forwarding a request to an actor @@ -483,7 +595,32 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { } protected void updateConfigParams(ConfigParams configParams) { + + // obtain the RaftPolicy for oldConfigParams and the updated one. + String oldRaftPolicy = context.getConfigParams(). + getCustomRaftPolicyImplementationClass(); + String newRaftPolicy = configParams. + getCustomRaftPolicyImplementationClass(); + + LOG.debug("{}: RaftPolicy used with prev.config {}, RaftPolicy used with newConfig {}", persistenceId(), + oldRaftPolicy, newRaftPolicy); context.setConfigParams(configParams); + if (!Objects.equal(oldRaftPolicy, newRaftPolicy)) { + // The RaftPolicy was modified. If the current behavior is Follower then re-initialize to Follower + // but transfer the previous leaderId so it doesn't immediately try to schedule an election. This + // avoids potential disruption. Otherwise, switch to Follower normally. + RaftActorBehavior behavior = currentBehavior.getDelegate(); + if(behavior instanceof Follower) { + String previousLeaderId = ((Follower)behavior).getLeaderId(); + short previousLeaderPayloadVersion = behavior.getLeaderPayloadVersion(); + + LOG.debug("{}: Re-initializing to Follower with previous leaderId {}", persistenceId(), previousLeaderId); + + changeCurrentBehavior(new Follower(context, previousLeaderId, previousLeaderPayloadVersion)); + } else { + initializeBehavior(); + } + } } public final DataPersistenceProvider persistence() { @@ -589,6 +726,21 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { */ protected abstract Optional getRoleChangeNotifier(); + /** + * This method is called prior to operations such as leadership transfer and actor shutdown when the leader + * must pause or stop its duties. This method allows derived classes to gracefully pause or finish current + * work prior to performing the operation. On completion of any work, the run method must be called on the + * given Runnable to proceed with the given operation. Important: the run method must be called on + * this actor's thread dispatcher as as it modifies internal state. + *

+ * The default implementation immediately runs the operation. + * + * @param operation the operation to run + */ + protected void pauseLeader(Runnable operation) { + operation.run(); + } + protected void onLeaderChanged(String oldLeader, String newLeader){}; private String getLeaderAddress(){ @@ -670,21 +822,25 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { private static class BehaviorStateHolder { private RaftActorBehavior behavior; - private String leaderId; + private String lastValidLeaderId; private short leaderPayloadVersion; void init(RaftActorBehavior behavior) { this.behavior = behavior; - this.leaderId = behavior != null ? behavior.getLeaderId() : null; this.leaderPayloadVersion = behavior != null ? behavior.getLeaderPayloadVersion() : -1; + + String behaviorLeaderId = behavior != null ? behavior.getLeaderId() : null; + if(behaviorLeaderId != null) { + this.lastValidLeaderId = behaviorLeaderId; + } } RaftActorBehavior getBehavior() { return behavior; } - String getLeaderId() { - return leaderId; + String getLastValidLeaderId() { + return lastValidLeaderId; } short getLeaderPayloadVersion() {