X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=opendaylight%2Fmd-sal%2Fsal-akka-raft%2Fsrc%2Fmain%2Fjava%2Forg%2Fopendaylight%2Fcontroller%2Fcluster%2Fraft%2FRaftActor.java;h=c256c822a420e3a22b5a351778d58a88e73a9e8d;hb=12fa2670527dfe66df922dd2feb2001eec3f60ba;hp=dc65869f6df051b01f277676ebb810cff47e419b;hpb=80c5ff995321bb55b248d01a50bdc7204fd398d4;p=controller.git diff --git a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/RaftActor.java b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/RaftActor.java index dc65869f6d..c256c822a4 100644 --- a/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/RaftActor.java +++ b/opendaylight/md-sal/sal-akka-raft/src/main/java/org/opendaylight/controller/cluster/raft/RaftActor.java @@ -113,6 +113,8 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { private int currentRecoveryBatchCount; + + public RaftActor(String id, Map peerAddresses) { this(id, peerAddresses, Optional.absent()); } @@ -170,9 +172,7 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { onRecoveryComplete(); - RaftActorBehavior oldBehavior = currentBehavior; - currentBehavior = new Follower(context); - handleBehaviorChange(oldBehavior, currentBehavior); + initializeBehavior(); } } } @@ -199,12 +199,12 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { timer.start(); // Apply the snapshot to the actors state - applyRecoverySnapshot(ByteString.copyFrom(snapshot.getState())); + applyRecoverySnapshot(snapshot.getState()); timer.stop(); LOG.info("Recovery snapshot applied for {} in {}: snapshotIndex={}, snapshotTerm={}, journal-size=" + replicatedLog.size(), persistenceId(), timer.toString(), - replicatedLog.snapshotIndex, replicatedLog.snapshotTerm); + replicatedLog.getSnapshotIndex(), replicatedLog.getSnapshotTerm()); } private void onRecoveredJournalLogEntry(ReplicatedLogEntry logEntry) { @@ -268,11 +268,19 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { "Persistence Id = " + persistenceId() + " Last index in log={}, snapshotIndex={}, snapshotTerm={}, " + "journal-size={}", - replicatedLog.lastIndex(), replicatedLog.snapshotIndex, - replicatedLog.snapshotTerm, replicatedLog.size()); + replicatedLog.lastIndex(), replicatedLog.getSnapshotIndex(), + replicatedLog.getSnapshotTerm(), replicatedLog.size()); + + initializeBehavior(); + } + protected void initializeBehavior(){ + changeCurrentBehavior(new Follower(context)); + } + + protected void changeCurrentBehavior(RaftActorBehavior newBehavior){ RaftActorBehavior oldBehavior = currentBehavior; - currentBehavior = new Follower(context); + currentBehavior = newBehavior; handleBehaviorChange(oldBehavior, currentBehavior); } @@ -309,7 +317,8 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { snapshot.getLastAppliedTerm() ); } - applySnapshot(ByteString.copyFrom(snapshot.getState())); + + applySnapshot(snapshot.getState()); //clears the followers log, sets the snapshot index to ensure adjusted-index works replicatedLog = new ReplicatedLogImpl(snapshot); @@ -346,17 +355,14 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { } else if (message instanceof CaptureSnapshot) { LOG.info("CaptureSnapshot received by actor"); - CaptureSnapshot cs = (CaptureSnapshot)message; - captureSnapshot = cs; - createSnapshot(); - } else if (message instanceof CaptureSnapshotReply){ - LOG.info("CaptureSnapshotReply received by actor"); - CaptureSnapshotReply csr = (CaptureSnapshotReply) message; + if(captureSnapshot == null) { + captureSnapshot = (CaptureSnapshot)message; + createSnapshot(); + } - ByteString stateInBytes = csr.getSnapshot(); - LOG.info("CaptureSnapshotReply stateInBytes size:{}", stateInBytes.size()); - handleCaptureSnapshotReply(stateInBytes); + } else if (message instanceof CaptureSnapshotReply){ + handleCaptureSnapshotReply(((CaptureSnapshotReply) message).getSnapshot()); } else { if (!(message instanceof AppendEntriesMessages.AppendEntries) @@ -400,8 +406,8 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { * @param identifier * @param data */ - protected void persistData(ActorRef clientActor, String identifier, - Payload data) { + protected void persistData(final ActorRef clientActor, final String identifier, + final Payload data) { ReplicatedLogEntry replicatedLogEntry = new ReplicatedLogImplEntry( context.getReplicatedLog().lastIndex() + 1, @@ -411,9 +417,42 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { LOG.debug("Persist data {}", replicatedLogEntry); } + final RaftActorContext raftContext = getRaftActorContext(); + replicatedLog - .appendAndPersist(clientActor, identifier, replicatedLogEntry); - } + .appendAndPersist(replicatedLogEntry, new Procedure() { + @Override + public void apply(ReplicatedLogEntry replicatedLogEntry) throws Exception { + if(!hasFollowers()){ + // Increment the Commit Index and the Last Applied values + raftContext.setCommitIndex(replicatedLogEntry.getIndex()); + raftContext.setLastApplied(replicatedLogEntry.getIndex()); + + // Apply the state immediately + applyState(clientActor, identifier, data); + + // Send a ApplyLogEntries message so that we write the fact that we applied + // the state to durable storage + self().tell(new ApplyLogEntries((int) replicatedLogEntry.getIndex()), self()); + + // Check if the "real" snapshot capture has been initiated. If no then do the fake snapshot + if(!hasSnapshotCaptureInitiated){ + raftContext.getReplicatedLog().snapshotPreCommit(raftContext.getLastApplied(), + raftContext.getTermInformation().getCurrentTerm()); + raftContext.getReplicatedLog().snapshotCommit(); + } else { + LOG.debug("Skipping fake snapshotting for {} because real snapshotting is in progress", getId()); + } + } else if (clientActor != null) { + // Send message for replication + currentBehavior.handleMessage(getSelf(), + new Replicate(clientActor, identifier, + replicatedLogEntry) + ); + } + + } + }); } protected String getId() { return context.getId(); @@ -542,7 +581,7 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { * * @param snapshot A snapshot of the state of the actor */ - protected abstract void applyRecoverySnapshot(ByteString snapshot); + protected abstract void applyRecoverySnapshot(byte[] snapshotBytes); /** * This method is called during recovery at the end of a batch to apply the current batched @@ -571,9 +610,9 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { * operations when the derived actor is out of sync with it's peers * and the only way to bring it in sync is by applying a snapshot * - * @param snapshot A snapshot of the state of the actor + * @param snapshotBytes A snapshot of the state of the actor */ - protected abstract void applySnapshot(ByteString snapshot); + protected abstract void applySnapshot(byte[] snapshotBytes); /** * This method will be called by the RaftActor when the state of the @@ -620,11 +659,13 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { return peerAddress; } - private void handleCaptureSnapshotReply(ByteString stateInBytes) { + private void handleCaptureSnapshotReply(byte[] snapshotBytes) { + LOG.info("CaptureSnapshotReply received by actor: snapshot size {}", snapshotBytes.length); + // create a snapshot object from the state provided and save it // when snapshot is saved async, SaveSnapshotSuccess is raised. - Snapshot sn = Snapshot.create(stateInBytes.toByteArray(), + Snapshot sn = Snapshot.create(snapshotBytes, context.getReplicatedLog().getFrom(captureSnapshot.getLastAppliedIndex() + 1), captureSnapshot.getLastIndex(), captureSnapshot.getLastTerm(), captureSnapshot.getLastAppliedIndex(), captureSnapshot.getLastAppliedTerm()); @@ -646,15 +687,23 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { if (isLeader() && captureSnapshot.isInstallSnapshotInitiated()) { // this would be call straight to the leader and won't initiate in serialization - currentBehavior.handleMessage(getSelf(), new SendInstallSnapshot(stateInBytes)); + currentBehavior.handleMessage(getSelf(), new SendInstallSnapshot( + ByteString.copyFrom(snapshotBytes))); } captureSnapshot = null; hasSnapshotCaptureInitiated = false; } + protected boolean hasFollowers(){ + return getRaftActorContext().getPeerAddresses().keySet().size() > 0; + } + private class ReplicatedLogImpl extends AbstractReplicatedLogImpl { + private static final int DATA_SIZE_DIVIDER = 5; + private long dataSizeSinceLastSnapshot = 0; + public ReplicatedLogImpl(Snapshot snapshot) { super(snapshot.getLastAppliedIndex(), snapshot.getLastAppliedTerm(), snapshot.getUnAppliedEntries()); @@ -689,7 +738,7 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { @Override public void appendAndPersist( final ReplicatedLogEntry replicatedLogEntry) { - appendAndPersist(null, null, replicatedLogEntry); + appendAndPersist(replicatedLogEntry, null); } @Override @@ -697,9 +746,9 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { return dataSize; } - public void appendAndPersist(final ActorRef clientActor, - final String identifier, - final ReplicatedLogEntry replicatedLogEntry) { + public void appendAndPersist( + final ReplicatedLogEntry replicatedLogEntry, + final Procedure callback) { if(LOG.isDebugEnabled()) { LOG.debug("Append log entry and persist {} ", replicatedLogEntry); @@ -717,22 +766,48 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { new Procedure() { @Override public void apply(ReplicatedLogEntry evt) throws Exception { - dataSize += replicatedLogEntry.size(); + int logEntrySize = replicatedLogEntry.size(); + + dataSize += logEntrySize; + long dataSizeForCheck = dataSize; + + dataSizeSinceLastSnapshot += logEntrySize; + long journalSize = lastIndex()+1; + + if(!hasFollowers()) { + // When we do not have followers we do not maintain an in-memory log + // due to this the journalSize will never become anything close to the + // snapshot batch count. In fact will mostly be 1. + // Similarly since the journal's dataSize depends on the entries in the + // journal the journal's dataSize will never reach a value close to the + // memory threshold. + // By maintaining the dataSize outside the journal we are tracking essentially + // what we have written to the disk however since we no longer are in + // need of doing a snapshot just for the sake of freeing up memory we adjust + // the real size of data by the DATA_SIZE_DIVIDER so that we do not snapshot as often + // as if we were maintaining a real snapshot + dataSizeForCheck = dataSizeSinceLastSnapshot / DATA_SIZE_DIVIDER; + } long dataThreshold = Runtime.getRuntime().totalMemory() * getRaftActorContext().getConfigParams().getSnapshotDataThresholdPercentage() / 100; // when a snaphsot is being taken, captureSnapshot != null if (hasSnapshotCaptureInitiated == false && - ( journal.size() % context.getConfigParams().getSnapshotBatchCount() == 0 || - dataSize > dataThreshold)) { + ( journalSize % context.getConfigParams().getSnapshotBatchCount() == 0 || + dataSizeForCheck > dataThreshold)) { + + dataSizeSinceLastSnapshot = 0; LOG.info("Initiating Snapshot Capture.."); long lastAppliedIndex = -1; long lastAppliedTerm = -1; ReplicatedLogEntry lastAppliedEntry = get(context.getLastApplied()); - if (lastAppliedEntry != null) { + if (!hasFollowers()) { + lastAppliedIndex = replicatedLogEntry.getIndex(); + lastAppliedTerm = replicatedLogEntry.getTerm(); + } else if (lastAppliedEntry != null) { lastAppliedIndex = lastAppliedEntry.getIndex(); lastAppliedTerm = lastAppliedEntry.getTerm(); } @@ -751,12 +826,8 @@ public abstract class RaftActor extends AbstractUntypedPersistentActor { null); hasSnapshotCaptureInitiated = true; } - // Send message for replication - if (clientActor != null) { - currentBehavior.handleMessage(getSelf(), - new Replicate(clientActor, identifier, - replicatedLogEntry) - ); + if(callback != null){ + callback.apply(replicatedLogEntry); } } }