BUG-5414 introduce EOS inJeopardy flag

The inJeopardy flag is used to indicate that the leader has lost quorum,
e.g. if cannot reach majority of followers or the follower has lost connection
to the leader (and has initiated new elections).

While EOS is in jeopardy, any reported entity state may not reflect cluster-wide
consensus, but rather represents the latest intended state as seen by this node.

Change-Id: I18df5a11ebbef6607fb0a0754ba0f09bc52f19ba
Signed-off-by: Robert Varga <rovarga@cisco.com>

diff --git a/opendaylight/md-sal/sal-common-api/src/main/java/org/opendaylight/controller/md/sal/common/api/clustering/EntityOwnershipChange.java b/opendaylight/md-sal/sal-common-api/src/main/java/org/opendaylight/controller/md/sal/common/api/clustering/EntityOwnershipChange.java
index b9d67f175923fc6eca65544ef6d1e160adf228d9..b8145be6c5f866961acc2c45e095ac4c6e77ae2f 100644 (file)
--- a/opendaylight/md-sal/sal-common-api/src/main/java/org/opendaylight/controller/md/sal/common/api/clustering/EntityOwnershipChange.java
+++ b/opendaylight/md-sal/sal-common-api/src/main/java/org/opendaylight/controller/md/sal/common/api/clustering/EntityOwnershipChange.java
@@ -20,12 +20,19 @@ public class EntityOwnershipChange {
     private final boolean wasOwner;
     private final boolean isOwner;
     private final boolean hasOwner;
+    private final boolean inJeopardy;
 
     public EntityOwnershipChange(@Nonnull Entity entity, boolean wasOwner, boolean isOwner, boolean hasOwner) {
+        this(entity, wasOwner, isOwner, hasOwner, false);
+    }
+
+    public EntityOwnershipChange(@Nonnull Entity entity, boolean wasOwner, boolean isOwner, boolean hasOwner,
+            boolean inJeopardy) {
         this.entity = Preconditions.checkNotNull(entity, "entity can't be null");
         this.wasOwner = wasOwner;
         this.isOwner = isOwner;
         this.hasOwner = hasOwner;
+        this.inJeopardy = inJeopardy;
     }
 
     /**
@@ -61,9 +68,19 @@ public class EntityOwnershipChange {
         return hasOwner;
     }
 
+    /**
+     * Returns the current jeopardy state. When in a jeopardy state, the values from other methods may potentially
+     * be out of date.
+     *
+     * @return true if the local node is in a jeopardy state. If false, the reported information is accurate.
+     */
+    public boolean inJeopardy() {
+        return inJeopardy;
+    }
+
     @Override
     public String toString() {
         return "EntityOwnershipChanged [entity=" + entity + ", wasOwner=" + wasOwner + ", isOwner=" + isOwner
-                + ", hasOwner=" + hasOwner + "]";
+                + ", hasOwner=" + hasOwner + ", inJeopardy=" + inJeopardy + "]";
     }
 }

diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipListenerSupport.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipListenerSupport.java
index dd83e65f1d169a74175ee65fc8eeaf823a012e66..c0fe55f12d14ee092a8d34ac6cf41882523fb929 100644 (file)
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipListenerSupport.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipListenerSupport.java
@@ -37,6 +37,7 @@ class EntityOwnershipListenerSupport {
     private final Map<EntityOwnershipListener, ListenerActorRefEntry> listenerActorMap = new IdentityHashMap<>();
     private final Set<Entity> entitiesWithCandidateSet = new HashSet<>();
     private final Multimap<String, EntityOwnershipListener> entityTypeListenerMap = HashMultimap.create();
+    private volatile boolean inJeopardy = false;
 
     EntityOwnershipListenerSupport(ActorContext actorContext, String logId) {
         this.actorContext = actorContext;
@@ -47,6 +48,18 @@ class EntityOwnershipListenerSupport {
         return logId;
     }
 
+    /**
+     * Set the in-jeopardy flag and indicate its previous state.
+     *
+     * @param inJeopardy new value of the in-jeopardy flag
+     * @return Previous value of the flag.
+     */
+    boolean setInJeopardy(final boolean inJeopardy) {
+        final boolean wasInJeopardy = this.inJeopardy;
+        this.inJeopardy = inJeopardy;
+        return wasInJeopardy;
+    }
+
     boolean hasCandidateForEntity(Entity entity) {
         return entitiesWithCandidateSet.contains(entity);
     }
@@ -89,7 +102,7 @@ class EntityOwnershipListenerSupport {
 
     private void notifyListeners(Entity entity, boolean wasOwner, boolean isOwner, boolean hasOwner,
             Collection<EntityOwnershipListener> listeners) {
-        EntityOwnershipChange changed = new EntityOwnershipChange(entity, wasOwner, isOwner, hasOwner);
+        EntityOwnershipChange changed = new EntityOwnershipChange(entity, wasOwner, isOwner, hasOwner, inJeopardy);
         for(EntityOwnershipListener listener: listeners) {
             ActorRef listenerActor = listenerActorFor(listener);
 

diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShard.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShard.java
index 2909d0ae0c0a3ab80a8af0d4fdd500dc1e4a6a0d..3d618b8da2d994fb4368d4736f84766fa5e6581a 100644 (file)
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShard.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShard.java
@@ -55,6 +55,7 @@ import org.opendaylight.controller.cluster.datastore.messages.SuccessReply;
 import org.opendaylight.controller.cluster.datastore.modification.DeleteModification;
 import org.opendaylight.controller.cluster.datastore.modification.MergeModification;
 import org.opendaylight.controller.cluster.datastore.modification.WriteModification;
+import org.opendaylight.controller.cluster.raft.RaftState;
 import org.opendaylight.controller.md.sal.common.api.clustering.Entity;
 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
 import org.opendaylight.yangtools.yang.data.api.schema.DataContainerChild;
@@ -253,13 +254,62 @@ class EntityOwnershipShard extends Shard {
         return getLeader() != null && !isIsolatedLeader();
     }
 
+    /**
+     * Determine if we are in jeopardy based on observed RAFT state.
+     */
+    private static boolean inJeopardy(final RaftState state) {
+        switch (state) {
+            case Candidate:
+            case Follower:
+            case Leader:
+                return false;
+            case IsolatedLeader:
+                return true;
+        }
+        throw new IllegalStateException("Unsupported RAFT state " + state);
+    }
+
+    private void notifyAllListeners() {
+        searchForEntities(new EntityWalker() {
+            @Override
+            public void onEntity(MapEntryNode entityTypeNode, MapEntryNode entityNode) {
+                Optional<DataContainerChild<?, ?>> possibleType = entityTypeNode.getChild(ENTITY_TYPE_NODE_ID);
+                if (possibleType.isPresent()) {
+                    final boolean hasOwner;
+                    final boolean isOwner;
+
+                    Optional<DataContainerChild<?, ?>> possibleOwner = entityNode.getChild(ENTITY_OWNER_NODE_ID);
+                    if (possibleOwner.isPresent()) {
+                        isOwner = localMemberName.equals(possibleOwner.get().getValue().toString());
+                        hasOwner = true;
+                    } else {
+                        isOwner = false;
+                        hasOwner = false;
+                    }
+
+                    Entity entity = new Entity(possibleType.get().getValue().toString(),
+                        (YangInstanceIdentifier) entityNode.getChild(ENTITY_ID_NODE_ID).get().getValue());
+
+                    listenerSupport.notifyEntityOwnershipListeners(entity, isOwner, isOwner, hasOwner);
+                }
+            }
+        });
+    }
+
     @Override
     protected void onStateChanged() {
         super.onStateChanged();
 
         boolean isLeader = isLeader();
-        if(LOG.isDebugEnabled()) {
-            LOG.debug("{}: onStateChanged: isLeader: {}, hasLeader: {}", persistenceId(), isLeader, hasLeader());
+        LOG.debug("{}: onStateChanged: isLeader: {}, hasLeader: {}", persistenceId(), isLeader, hasLeader());
+
+        // Examine current RAFT state to see if we are in jeopardy, potentially notifying all listeners
+        final boolean inJeopardy = inJeopardy(getRaftState());
+        final boolean wasInJeopardy = listenerSupport.setInJeopardy(inJeopardy);
+        if (inJeopardy != wasInJeopardy) {
+            LOG.debug("{}: {} jeopardy state, notifying all listeners", persistenceId(),
+                inJeopardy ? "entered" : "left");
+            notifyAllListeners();
         }
 
         commitCoordinator.onStateChanged(this, isLeader);