import akka.actor.ActorRef;
import akka.actor.ActorSelection;
+import akka.actor.PoisonPill;
import akka.japi.Procedure;
-import akka.persistence.SnapshotSelectionCriteria;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Objects;
import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableMap;
+import com.google.common.base.Supplier;
import com.google.common.collect.Lists;
import java.io.Serializable;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.opendaylight.controller.cluster.notifications.RoleChanged;
import org.opendaylight.controller.cluster.raft.base.messages.ApplyJournalEntries;
import org.opendaylight.controller.cluster.raft.base.messages.ApplyState;
+import org.opendaylight.controller.cluster.raft.base.messages.InitiateCaptureSnapshot;
+import org.opendaylight.controller.cluster.raft.base.messages.LeaderTransitioning;
import org.opendaylight.controller.cluster.raft.base.messages.Replicate;
+import org.opendaylight.controller.cluster.raft.base.messages.SwitchBehavior;
import org.opendaylight.controller.cluster.raft.behaviors.AbstractLeader;
import org.opendaylight.controller.cluster.raft.behaviors.DelegatingRaftActorBehavior;
import org.opendaylight.controller.cluster.raft.behaviors.Follower;
import org.opendaylight.controller.cluster.raft.client.messages.FollowerInfo;
import org.opendaylight.controller.cluster.raft.client.messages.GetOnDemandRaftState;
import org.opendaylight.controller.cluster.raft.client.messages.OnDemandRaftState;
+import org.opendaylight.controller.cluster.raft.client.messages.Shutdown;
import org.opendaylight.controller.cluster.raft.protobuff.client.messages.Payload;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
*/
private final RaftActorContextImpl context;
- private final DelegatingPersistentDataProvider delegatingPersistenceProvider = new DelegatingPersistentDataProvider(null);
+ private final DelegatingPersistentDataProvider delegatingPersistenceProvider;
+
+ private final PersistentDataProvider persistentProvider;
private RaftActorRecoverySupport raftRecovery;
private final BehaviorStateHolder reusableBehaviorStateHolder = new BehaviorStateHolder();
- public RaftActor(String id, Map<String, String> peerAddresses) {
- this(id, peerAddresses, Optional.<ConfigParams>absent());
- }
+ private final SwitchBehaviorSupplier reusableSwitchBehaviorSupplier = new SwitchBehaviorSupplier();
+
+ private RaftActorServerConfigurationSupport serverConfigurationSupport;
+
+ private RaftActorLeadershipTransferCohort leadershipTransferInProgress;
+
+ private boolean shuttingDown;
public RaftActor(String id, Map<String, String> peerAddresses,
- Optional<ConfigParams> configParams) {
+ Optional<ConfigParams> configParams, short payloadVersion) {
+
+ persistentProvider = new PersistentDataProvider(this);
+ delegatingPersistenceProvider = new RaftActorDelegatingPersistentDataProvider(null, persistentProvider);
context = new RaftActorContextImpl(this.getSelf(),
- this.getContext(), id, new ElectionTermImpl(delegatingPersistenceProvider, id, LOG),
+ this.getContext(), id, new ElectionTermImpl(persistentProvider, id, LOG),
-1, -1, peerAddresses,
- (configParams.isPresent() ? configParams.get(): new DefaultConfigParamsImpl()), LOG);
+ (configParams.isPresent() ? configParams.get(): new DefaultConfigParamsImpl()),
+ delegatingPersistenceProvider, LOG);
- context.setReplicatedLog(ReplicatedLogImpl.newInstance(context, delegatingPersistenceProvider, currentBehavior));
+ context.setPayloadVersion(payloadVersion);
+ context.setReplicatedLog(ReplicatedLogImpl.newInstance(context, currentBehavior));
}
@Override
context.getConfigParams().getJournalRecoveryLogBatchSize());
super.preStart();
+
+ snapshotSupport = newRaftActorSnapshotMessageSupport();
+ serverConfigurationSupport = new RaftActorServerConfigurationSupport(this);
}
@Override
raftRecovery = newRaftActorRecoverySupport();
}
- boolean recoveryComplete = raftRecovery.handleRecoveryMessage(message);
+ boolean recoveryComplete = raftRecovery.handleRecoveryMessage(message, persistentProvider);
if(recoveryComplete) {
- if(!persistence().isRecoveryApplicable()) {
- // Delete all the messages from the akka journal so that we do not end up with consistency issues
- // Note I am not using the dataPersistenceProvider and directly using the akka api here
- deleteMessages(lastSequenceNr());
-
- // Delete all the akka snapshots as they will not be needed
- deleteSnapshots(new SnapshotSelectionCriteria(scala.Long.MaxValue(), scala.Long.MaxValue()));
- }
-
onRecoveryComplete();
initializeBehavior();
raftRecovery = null;
+
+ if (context.getReplicatedLog().size() > 0) {
+ self().tell(new InitiateCaptureSnapshot(), self());
+ LOG.info("{}: Snapshot capture initiated after recovery", persistenceId());
+ } else {
+ LOG.info("{}: Snapshot capture NOT initiated after recovery, journal empty", persistenceId());
+ }
}
}
protected RaftActorRecoverySupport newRaftActorRecoverySupport() {
- return new RaftActorRecoverySupport(delegatingPersistenceProvider, context, currentBehavior,
- getRaftActorRecoveryCohort());
+ return new RaftActorRecoverySupport(context, currentBehavior, getRaftActorRecoveryCohort());
}
protected void initializeBehavior(){
}
@Override
- public void handleCommand(Object message) {
- if(snapshotSupport == null) {
- snapshotSupport = newRaftActorSnapshotMessageSupport();
- }
-
- boolean handled = snapshotSupport.handleSnapshotMessage(message);
- if(handled) {
+ public void handleCommand(final Object message) {
+ if(serverConfigurationSupport.handleMessage(message, getSender())) {
return;
- }
-
- if (message instanceof ApplyState){
+ } else if (message instanceof ApplyState){
ApplyState applyState = (ApplyState) message;
long elapsedTime = (System.nanoTime() - applyState.getStartTime());
applyState(applyState.getClientActor(), applyState.getIdentifier(),
applyState.getReplicatedLogEntry().getData());
+ if (!hasFollowers()) {
+ // for single node, the capture should happen after the apply state
+ // as we delete messages from the persistent journal which have made it to the snapshot
+ // capturing the snapshot before applying makes the persistent journal and snapshot out of sync
+ // and recovery shows data missing
+ context.getReplicatedLog().captureSnapshotIfReady(applyState.getReplicatedLogEntry());
+
+ context.getSnapshotManager().trimLog(context.getLastApplied(), currentBehavior);
+ }
+
} else if (message instanceof ApplyJournalEntries){
ApplyJournalEntries applyEntries = (ApplyJournalEntries) message;
if(LOG.isDebugEnabled()) {
);
} else if(message instanceof GetOnDemandRaftState) {
onGetOnDemandRaftStats();
+ } else if(message instanceof InitiateCaptureSnapshot) {
+ captureSnapshot();
+ } else if(message instanceof SwitchBehavior){
+ switchBehavior(((SwitchBehavior) message));
+ } else if(message instanceof LeaderTransitioning) {
+ onLeaderTransitioning();
+ } else if(message instanceof Shutdown) {
+ onShutDown();
+ } else if(message instanceof Runnable) {
+ ((Runnable)message).run();
+ } else if(!snapshotSupport.handleSnapshotMessage(message, getSender())) {
+ switchBehavior(reusableSwitchBehaviorSupplier.handleMessage(getSender(), message));
+ }
+ }
+
+ private void initiateLeadershipTransfer(final RaftActorLeadershipTransferCohort.OnComplete onComplete) {
+ LOG.debug("{}: Initiating leader transfer", persistenceId());
+
+ if(leadershipTransferInProgress == null) {
+ leadershipTransferInProgress = new RaftActorLeadershipTransferCohort(this, getSender());
+ leadershipTransferInProgress.addOnComplete(new RaftActorLeadershipTransferCohort.OnComplete() {
+ @Override
+ public void onSuccess(ActorRef raftActorRef, ActorRef replyTo) {
+ leadershipTransferInProgress = null;
+ }
+
+ @Override
+ public void onFailure(ActorRef raftActorRef, ActorRef replyTo) {
+ leadershipTransferInProgress = null;
+ }
+ });
+
+ leadershipTransferInProgress.addOnComplete(onComplete);
+ leadershipTransferInProgress.init();
+ } else {
+ LOG.debug("{}: prior leader transfer in progress - adding callback", persistenceId());
+ leadershipTransferInProgress.addOnComplete(onComplete);
+ }
+ }
+
+ private void onShutDown() {
+ LOG.debug("{}: onShutDown", persistenceId());
+
+ if(shuttingDown) {
+ return;
+ }
+
+ shuttingDown = true;
+ if(currentBehavior.state() == RaftState.Leader && context.hasFollowers()) {
+ initiateLeadershipTransfer(new RaftActorLeadershipTransferCohort.OnComplete() {
+ @Override
+ public void onSuccess(ActorRef raftActorRef, ActorRef replyTo) {
+ LOG.debug("{}: leader transfer succeeded - sending PoisonPill", persistenceId());
+ raftActorRef.tell(PoisonPill.getInstance(), raftActorRef);
+ }
+
+ @Override
+ public void onFailure(ActorRef raftActorRef, ActorRef replyTo) {
+ LOG.debug("{}: leader transfer failed - sending PoisonPill", persistenceId());
+ raftActorRef.tell(PoisonPill.getInstance(), raftActorRef);
+ }
+ });
+ } else if(currentBehavior.state() == RaftState.Leader) {
+ pauseLeader(new TimedRunnable(context.getConfigParams().getElectionTimeOutInterval(), this) {
+ @Override
+ protected void doRun() {
+ self().tell(PoisonPill.getInstance(), self());
+ }
+
+ @Override
+ protected void doCancel() {
+ self().tell(PoisonPill.getInstance(), self());
+ }
+ });
} else {
- reusableBehaviorStateHolder.init(getCurrentBehavior());
+ self().tell(PoisonPill.getInstance(), self());
+ }
+ }
- setCurrentBehavior(currentBehavior.handleMessage(getSender(), message));
+ private void onLeaderTransitioning() {
+ LOG.debug("{}: onLeaderTransitioning", persistenceId());
+ Optional<ActorRef> roleChangeNotifier = getRoleChangeNotifier();
+ if(currentBehavior.state() == RaftState.Follower && roleChangeNotifier.isPresent()) {
+ roleChangeNotifier.get().tell(newLeaderStateChanged(getId(), null,
+ currentBehavior.getLeaderPayloadVersion()), getSelf());
+ }
+ }
- handleBehaviorChange(reusableBehaviorStateHolder, getCurrentBehavior());
+ private void switchBehavior(SwitchBehavior message) {
+ if(!getRaftActorContext().getRaftPolicy().automaticElectionsEnabled()) {
+ RaftState newState = message.getNewState();
+ if( newState == RaftState.Leader || newState == RaftState.Follower) {
+ switchBehavior(reusableSwitchBehaviorSupplier.handleMessage(getSender(), message));
+ getRaftActorContext().getTermInformation().updateAndPersist(message.getNewTerm(), "");
+ } else {
+ LOG.warn("Switching to behavior : {} - not supported", newState);
+ }
}
}
+ private void switchBehavior(Supplier<RaftActorBehavior> supplier){
+ reusableBehaviorStateHolder.init(getCurrentBehavior());
+
+ setCurrentBehavior(supplier.get());
+
+ handleBehaviorChange(reusableBehaviorStateHolder, getCurrentBehavior());
+ }
+
protected RaftActorSnapshotMessageSupport newRaftActorSnapshotMessageSupport() {
- return new RaftActorSnapshotMessageSupport(delegatingPersistenceProvider, context,
- currentBehavior, getRaftActorSnapshotCohort());
+ return new RaftActorSnapshotMessageSupport(context, currentBehavior,
+ getRaftActorSnapshotCohort());
}
private void onGetOnDemandRaftStats() {
// Debugging message to retrieve raft stats.
+ Map<String, String> peerAddresses = new HashMap<>();
+ for(String peerId: context.getPeerIds()) {
+ peerAddresses.put(peerId, context.getPeerAddress(peerId));
+ }
+
OnDemandRaftState.Builder builder = OnDemandRaftState.builder()
.commitIndex(context.getCommitIndex())
.currentTerm(context.getTermInformation().getCurrentTerm())
.snapshotIndex(replicatedLog().getSnapshotIndex())
.snapshotTerm(replicatedLog().getSnapshotTerm())
.votedFor(context.getTermInformation().getVotedFor())
- .peerAddresses(ImmutableMap.copyOf(context.getPeerAddresses()));
+ .peerAddresses(peerAddresses)
+ .customRaftPolicyClassName(context.getConfigParams().getCustomRaftPolicyImplementationClass());
ReplicatedLogEntry lastLogEntry = getLastLogEntry();
if (lastLogEntry != null) {
onStateChanged();
}
- String oldBehaviorLeaderId = oldBehavior == null ? null : oldBehaviorState.getLeaderId();
+ String lastValidLeaderId = oldBehavior == null ? null : oldBehaviorState.getLastValidLeaderId();
String oldBehaviorStateName = oldBehavior == null ? null : oldBehavior.state().name();
// it can happen that the state has not changed but the leader has changed.
Optional<ActorRef> roleChangeNotifier = getRoleChangeNotifier();
- if(!Objects.equal(oldBehaviorLeaderId, currentBehavior.getLeaderId())) {
+ if(!Objects.equal(lastValidLeaderId, currentBehavior.getLeaderId()) ||
+ oldBehaviorState.getLeaderPayloadVersion() != currentBehavior.getLeaderPayloadVersion()) {
if(roleChangeNotifier.isPresent()) {
- roleChangeNotifier.get().tell(new LeaderStateChanged(getId(), currentBehavior.getLeaderId()), getSelf());
+ roleChangeNotifier.get().tell(newLeaderStateChanged(getId(), currentBehavior.getLeaderId(),
+ currentBehavior.getLeaderPayloadVersion()), getSelf());
}
- onLeaderChanged(oldBehaviorLeaderId, currentBehavior.getLeaderId());
+ onLeaderChanged(lastValidLeaderId, currentBehavior.getLeaderId());
+
+ if(leadershipTransferInProgress != null) {
+ leadershipTransferInProgress.onNewLeader(currentBehavior.getLeaderId());
+ }
}
if (roleChangeNotifier.isPresent() &&
}
}
+ protected LeaderStateChanged newLeaderStateChanged(String memberId, String leaderId, short leaderPayloadVersion) {
+ return new LeaderStateChanged(memberId, leaderId, leaderPayloadVersion);
+ }
+
+ @Override
+ public long snapshotSequenceNr() {
+ // When we do a snapshot capture, we also capture and save the sequence-number of the persistent journal,
+ // so that we can delete the persistent journal based on the saved sequence-number
+ // However , when akka replays the journal during recovery, it replays it from the sequence number when the snapshot
+ // was saved and not the number we saved.
+ // We would want to override it , by asking akka to use the last-sequence number known to us.
+ return context.getSnapshotManager().getLastSequenceNumber();
+ }
+
/**
* When a derived RaftActor needs to persist something it must call
* persistData.
replicatedLog().appendAndPersist(replicatedLogEntry, new Procedure<ReplicatedLogEntry>() {
@Override
- public void apply(ReplicatedLogEntry replicatedLogEntry) throws Exception {
- if(!hasFollowers()){
+ public void apply(ReplicatedLogEntry replicatedLogEntry) {
+ if (!hasFollowers()){
// Increment the Commit Index and the Last Applied values
raftContext.setCommitIndex(replicatedLogEntry.getIndex());
raftContext.setLastApplied(replicatedLogEntry.getIndex());
- // Apply the state immediately
- applyState(clientActor, identifier, data);
+ // Apply the state immediately.
+ self().tell(new ApplyState(clientActor, identifier, replicatedLogEntry), self());
// Send a ApplyJournalEntries message so that we write the fact that we applied
// the state to durable storage
self().tell(new ApplyJournalEntries(replicatedLogEntry.getIndex()), self());
- context.getSnapshotManager().trimLog(context.getLastApplied(), currentBehavior);
-
} else if (clientActor != null) {
+ context.getReplicatedLog().captureSnapshotIfReady(replicatedLogEntry);
+
// Send message for replication
currentBehavior.handleMessage(getSelf(),
new Replicate(clientActor, identifier, replicatedLogEntry));
return context.getId().equals(currentBehavior.getLeaderId());
}
+ protected boolean isLeaderActive() {
+ return currentBehavior.state() != RaftState.IsolatedLeader && !shuttingDown &&
+ !isLeadershipTransferInProgress();
+ }
+
+ private boolean isLeadershipTransferInProgress() {
+ return leadershipTransferInProgress != null && leadershipTransferInProgress.isTransferring();
+ }
+
/**
* Derived actor can call getLeader if they need a reference to the Leader.
* This would be useful for example in forwarding a request to an actor
}
protected void updateConfigParams(ConfigParams configParams) {
+
+ // obtain the RaftPolicy for oldConfigParams and the updated one.
+ String oldRaftPolicy = context.getConfigParams().
+ getCustomRaftPolicyImplementationClass();
+ String newRaftPolicy = configParams.
+ getCustomRaftPolicyImplementationClass();
+
+ LOG.debug("{}: RaftPolicy used with prev.config {}, RaftPolicy used with newConfig {}", persistenceId(),
+ oldRaftPolicy, newRaftPolicy);
context.setConfigParams(configParams);
+ if (!Objects.equal(oldRaftPolicy, newRaftPolicy)) {
+ // The RaftPolicy was modified. If the current behavior is Follower then re-initialize to Follower
+ // but transfer the previous leaderId so it doesn't immediately try to schedule an election. This
+ // avoids potential disruption. Otherwise, switch to Follower normally.
+ RaftActorBehavior behavior = currentBehavior.getDelegate();
+ if(behavior instanceof Follower) {
+ String previousLeaderId = ((Follower)behavior).getLeaderId();
+
+ LOG.debug("{}: Re-initializing to Follower with previous leaderId {}", persistenceId(), previousLeaderId);
+
+ changeCurrentBehavior(new Follower(context, previousLeaderId));
+ } else {
+ initializeBehavior();
+ }
+ }
}
public final DataPersistenceProvider persistence() {
*/
protected abstract Optional<ActorRef> getRoleChangeNotifier();
+ /**
+ * This method is called prior to operations such as leadership transfer and actor shutdown when the leader
+ * must pause or stop its duties. This method allows derived classes to gracefully pause or finish current
+ * work prior to performing the operation. On completion of any work, the run method must be called on the
+ * given Runnable to proceed with the given operation. <b>Important:</b> the run method must be called on
+ * this actor's thread dispatcher as as it modifies internal state.
+ * <p>
+ * The default implementation immediately runs the operation.
+ *
+ * @param operation the operation to run
+ */
+ protected void pauseLeader(Runnable operation) {
+ operation.run();
+ }
+
protected void onLeaderChanged(String oldLeader, String newLeader){};
private String getLeaderAddress(){
return getRaftActorContext().hasFollowers();
}
+ private void captureSnapshot() {
+ SnapshotManager snapshotManager = context.getSnapshotManager();
+
+ if(!snapshotManager.isCapturing()) {
+ LOG.debug("Take a snapshot of current state. lastReplicatedLog is {} and replicatedToAllIndex is {}",
+ replicatedLog().last(), currentBehavior.getReplicatedToAllIndex());
+
+ snapshotManager.capture(replicatedLog().last(), currentBehavior.getReplicatedToAllIndex());
+ }
+ }
+
/**
- * @deprecated Deprecated in favor of {@link org.opendaylight.controller.cluster.raft.base.messages.DeleteEntriesTest}
+ * @deprecated Deprecated in favor of {@link org.opendaylight.controller.cluster.raft.base.messages.DeleteEntries}
* whose type for fromIndex is long instead of int. This class was kept for backwards
* compatibility with Helium.
*/
+ // Suppressing this warning as we can't set serialVersionUID to maintain backwards compatibility.
+ @SuppressWarnings("serial")
@Deprecated
static class DeleteEntries implements Serializable {
- private static final long serialVersionUID = 1L;
private final int fromIndex;
public DeleteEntries(int fromIndex) {
}
}
+ /**
+ * @deprecated Deprecated in favor of non-inner class {@link org.opendaylight.controller.cluster.raft.base.messages.UpdateElectionTerm}
+ * which has serialVersionUID set. This class was kept for backwards compatibility with Helium.
+ */
+ // Suppressing this warning as we can't set serialVersionUID to maintain backwards compatibility.
+ @SuppressWarnings("serial")
+ @Deprecated
static class UpdateElectionTerm implements Serializable {
- private static final long serialVersionUID = 1L;
private final long currentTerm;
private final String votedFor;
private static class BehaviorStateHolder {
private RaftActorBehavior behavior;
- private String leaderId;
+ private String lastValidLeaderId;
+ private short leaderPayloadVersion;
void init(RaftActorBehavior behavior) {
this.behavior = behavior;
- this.leaderId = behavior != null ? behavior.getLeaderId() : null;
+ this.leaderPayloadVersion = behavior != null ? behavior.getLeaderPayloadVersion() : -1;
+
+ String behaviorLeaderId = behavior != null ? behavior.getLeaderId() : null;
+ if(behaviorLeaderId != null) {
+ this.lastValidLeaderId = behaviorLeaderId;
+ }
}
RaftActorBehavior getBehavior() {
return behavior;
}
- String getLeaderId() {
- return leaderId;
+ String getLastValidLeaderId() {
+ return lastValidLeaderId;
+ }
+
+ short getLeaderPayloadVersion() {
+ return leaderPayloadVersion;
+ }
+ }
+
+ private class SwitchBehaviorSupplier implements Supplier<RaftActorBehavior> {
+ private Object message;
+ private ActorRef sender;
+
+ public SwitchBehaviorSupplier handleMessage(ActorRef sender, Object message){
+ this.sender = sender;
+ this.message = message;
+ return this;
+ }
+
+ @Override
+ public RaftActorBehavior get() {
+ if(this.message instanceof SwitchBehavior){
+ return ((SwitchBehavior) message).getNewState().createBehavior(getRaftActorContext());
+ }
+ return currentBehavior.handleMessage(sender, message);
}
}
}