From d4fa6758d6b94aad894854c0fe6fcd82e7bbefd6 Mon Sep 17 00:00:00 2001 From: Robert Varga Date: Thu, 24 Mar 2016 22:07:49 +0100 Subject: [PATCH] BUG-5414 introduce EOS inJeopardy flag The inJeopardy flag is used to indicate that the leader has lost quorum, e.g. if cannot reach majority of followers or the follower has lost connection to the leader (and has initiated new elections). While EOS is in jeopardy, any reported entity state may not reflect cluster-wide consensus, but rather represents the latest intended state as seen by this node. Change-Id: I18df5a11ebbef6607fb0a0754ba0f09bc52f19ba Signed-off-by: Robert Varga --- .../api/clustering/EntityOwnershipChange.java | 19 ++++++- .../EntityOwnershipListenerSupport.java | 15 +++++- .../entityownership/EntityOwnershipShard.java | 54 ++++++++++++++++++- 3 files changed, 84 insertions(+), 4 deletions(-) diff --git a/opendaylight/md-sal/sal-common-api/src/main/java/org/opendaylight/controller/md/sal/common/api/clustering/EntityOwnershipChange.java b/opendaylight/md-sal/sal-common-api/src/main/java/org/opendaylight/controller/md/sal/common/api/clustering/EntityOwnershipChange.java index b9d67f1759..b8145be6c5 100644 --- a/opendaylight/md-sal/sal-common-api/src/main/java/org/opendaylight/controller/md/sal/common/api/clustering/EntityOwnershipChange.java +++ b/opendaylight/md-sal/sal-common-api/src/main/java/org/opendaylight/controller/md/sal/common/api/clustering/EntityOwnershipChange.java @@ -20,12 +20,19 @@ public class EntityOwnershipChange { private final boolean wasOwner; private final boolean isOwner; private final boolean hasOwner; + private final boolean inJeopardy; public EntityOwnershipChange(@Nonnull Entity entity, boolean wasOwner, boolean isOwner, boolean hasOwner) { + this(entity, wasOwner, isOwner, hasOwner, false); + } + + public EntityOwnershipChange(@Nonnull Entity entity, boolean wasOwner, boolean isOwner, boolean hasOwner, + boolean inJeopardy) { this.entity = Preconditions.checkNotNull(entity, "entity can't be null"); this.wasOwner = wasOwner; this.isOwner = isOwner; this.hasOwner = hasOwner; + this.inJeopardy = inJeopardy; } /** @@ -61,9 +68,19 @@ public class EntityOwnershipChange { return hasOwner; } + /** + * Returns the current jeopardy state. When in a jeopardy state, the values from other methods may potentially + * be out of date. + * + * @return true if the local node is in a jeopardy state. If false, the reported information is accurate. + */ + public boolean inJeopardy() { + return inJeopardy; + } + @Override public String toString() { return "EntityOwnershipChanged [entity=" + entity + ", wasOwner=" + wasOwner + ", isOwner=" + isOwner - + ", hasOwner=" + hasOwner + "]"; + + ", hasOwner=" + hasOwner + ", inJeopardy=" + inJeopardy + "]"; } } diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipListenerSupport.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipListenerSupport.java index dd83e65f1d..c0fe55f12d 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipListenerSupport.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipListenerSupport.java @@ -37,6 +37,7 @@ class EntityOwnershipListenerSupport { private final Map listenerActorMap = new IdentityHashMap<>(); private final Set entitiesWithCandidateSet = new HashSet<>(); private final Multimap entityTypeListenerMap = HashMultimap.create(); + private volatile boolean inJeopardy = false; EntityOwnershipListenerSupport(ActorContext actorContext, String logId) { this.actorContext = actorContext; @@ -47,6 +48,18 @@ class EntityOwnershipListenerSupport { return logId; } + /** + * Set the in-jeopardy flag and indicate its previous state. + * + * @param inJeopardy new value of the in-jeopardy flag + * @return Previous value of the flag. + */ + boolean setInJeopardy(final boolean inJeopardy) { + final boolean wasInJeopardy = this.inJeopardy; + this.inJeopardy = inJeopardy; + return wasInJeopardy; + } + boolean hasCandidateForEntity(Entity entity) { return entitiesWithCandidateSet.contains(entity); } @@ -89,7 +102,7 @@ class EntityOwnershipListenerSupport { private void notifyListeners(Entity entity, boolean wasOwner, boolean isOwner, boolean hasOwner, Collection listeners) { - EntityOwnershipChange changed = new EntityOwnershipChange(entity, wasOwner, isOwner, hasOwner); + EntityOwnershipChange changed = new EntityOwnershipChange(entity, wasOwner, isOwner, hasOwner, inJeopardy); for(EntityOwnershipListener listener: listeners) { ActorRef listenerActor = listenerActorFor(listener); diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShard.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShard.java index 2909d0ae0c..3d618b8da2 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShard.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShard.java @@ -55,6 +55,7 @@ import org.opendaylight.controller.cluster.datastore.messages.SuccessReply; import org.opendaylight.controller.cluster.datastore.modification.DeleteModification; import org.opendaylight.controller.cluster.datastore.modification.MergeModification; import org.opendaylight.controller.cluster.datastore.modification.WriteModification; +import org.opendaylight.controller.cluster.raft.RaftState; import org.opendaylight.controller.md.sal.common.api.clustering.Entity; import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier; import org.opendaylight.yangtools.yang.data.api.schema.DataContainerChild; @@ -253,13 +254,62 @@ class EntityOwnershipShard extends Shard { return getLeader() != null && !isIsolatedLeader(); } + /** + * Determine if we are in jeopardy based on observed RAFT state. + */ + private static boolean inJeopardy(final RaftState state) { + switch (state) { + case Candidate: + case Follower: + case Leader: + return false; + case IsolatedLeader: + return true; + } + throw new IllegalStateException("Unsupported RAFT state " + state); + } + + private void notifyAllListeners() { + searchForEntities(new EntityWalker() { + @Override + public void onEntity(MapEntryNode entityTypeNode, MapEntryNode entityNode) { + Optional> possibleType = entityTypeNode.getChild(ENTITY_TYPE_NODE_ID); + if (possibleType.isPresent()) { + final boolean hasOwner; + final boolean isOwner; + + Optional> possibleOwner = entityNode.getChild(ENTITY_OWNER_NODE_ID); + if (possibleOwner.isPresent()) { + isOwner = localMemberName.equals(possibleOwner.get().getValue().toString()); + hasOwner = true; + } else { + isOwner = false; + hasOwner = false; + } + + Entity entity = new Entity(possibleType.get().getValue().toString(), + (YangInstanceIdentifier) entityNode.getChild(ENTITY_ID_NODE_ID).get().getValue()); + + listenerSupport.notifyEntityOwnershipListeners(entity, isOwner, isOwner, hasOwner); + } + } + }); + } + @Override protected void onStateChanged() { super.onStateChanged(); boolean isLeader = isLeader(); - if(LOG.isDebugEnabled()) { - LOG.debug("{}: onStateChanged: isLeader: {}, hasLeader: {}", persistenceId(), isLeader, hasLeader()); + LOG.debug("{}: onStateChanged: isLeader: {}, hasLeader: {}", persistenceId(), isLeader, hasLeader()); + + // Examine current RAFT state to see if we are in jeopardy, potentially notifying all listeners + final boolean inJeopardy = inJeopardy(getRaftState()); + final boolean wasInJeopardy = listenerSupport.setInJeopardy(inJeopardy); + if (inJeopardy != wasInJeopardy) { + LOG.debug("{}: {} jeopardy state, notifying all listeners", persistenceId(), + inJeopardy ? "entered" : "left"); + notifyAllListeners(); } commitCoordinator.onStateChanged(this, isLeader); -- 2.36.6