BUG-5414 introduce EOS inJeopardy flag

The inJeopardy flag is used to indicate that the leader has lost quorum,
e.g. if cannot reach majority of followers or the follower has lost connection
to the leader (and has initiated new elections).

While EOS is in jeopardy, any reported entity state may not reflect cluster-wide
consensus, but rather represents the latest intended state as seen by this node.

Change-Id: I18df5a11ebbef6607fb0a0754ba0f09bc52f19ba
Signed-off-by: Robert Varga <rovarga@cisco.com>
(cherry picked from commit d4fa6758d6b94aad894854c0fe6fcd82e7bbefd6)

diff --git a/opendaylight/md-sal/sal-common-api/src/main/java/org/opendaylight/controller/md/sal/common/api/clustering/EntityOwnershipChange.java b/opendaylight/md-sal/sal-common-api/src/main/java/org/opendaylight/controller/md/sal/common/api/clustering/EntityOwnershipChange.java
index b9d67f175923fc6eca65544ef6d1e160adf228d9..b8145be6c5f866961acc2c45e095ac4c6e77ae2f 100644 (file)
--- a/opendaylight/md-sal/sal-common-api/src/main/java/org/opendaylight/controller/md/sal/common/api/clustering/EntityOwnershipChange.java
+++ b/opendaylight/md-sal/sal-common-api/src/main/java/org/opendaylight/controller/md/sal/common/api/clustering/EntityOwnershipChange.java
@@ -20,12 +20,19 @@ public class EntityOwnershipChange {
     private final boolean wasOwner;
     private final boolean isOwner;
     private final boolean hasOwner;
+    private final boolean inJeopardy;
 
     public EntityOwnershipChange(@Nonnull Entity entity, boolean wasOwner, boolean isOwner, boolean hasOwner) {
+        this(entity, wasOwner, isOwner, hasOwner, false);
+    }
+
+    public EntityOwnershipChange(@Nonnull Entity entity, boolean wasOwner, boolean isOwner, boolean hasOwner,
+            boolean inJeopardy) {
         this.entity = Preconditions.checkNotNull(entity, "entity can't be null");
         this.wasOwner = wasOwner;
         this.isOwner = isOwner;
         this.hasOwner = hasOwner;
+        this.inJeopardy = inJeopardy;
     }
 
     /**
@@ -61,9 +68,19 @@ public class EntityOwnershipChange {
         return hasOwner;
     }
 
+    /**
+     * Returns the current jeopardy state. When in a jeopardy state, the values from other methods may potentially
+     * be out of date.
+     *
+     * @return true if the local node is in a jeopardy state. If false, the reported information is accurate.
+     */
+    public boolean inJeopardy() {
+        return inJeopardy;
+    }
+
     @Override
     public String toString() {
         return "EntityOwnershipChanged [entity=" + entity + ", wasOwner=" + wasOwner + ", isOwner=" + isOwner
-                + ", hasOwner=" + hasOwner + "]";
+                + ", hasOwner=" + hasOwner + ", inJeopardy=" + inJeopardy + "]";
     }
 }

diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipListenerSupport.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipListenerSupport.java
index e944325c69229ade6828dca233043c307b43f6eb..7165d9d7de618afd99367f2e3a006b642ca550e6 100644 (file)
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipListenerSupport.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipListenerSupport.java
@@ -37,6 +37,7 @@ class EntityOwnershipListenerSupport {
     private final Map<EntityOwnershipListener, ListenerActorRefEntry> listenerActorMap = new IdentityHashMap<>();
     private final Set<Entity> entitiesWithCandidateSet = new HashSet<>();
     private final Multimap<String, EntityOwnershipListener> entityTypeListenerMap = HashMultimap.create();
+    private volatile boolean inJeopardy = false;
 
     EntityOwnershipListenerSupport(ActorContext actorContext, String logId) {
         this.actorContext = actorContext;
@@ -47,6 +48,18 @@ class EntityOwnershipListenerSupport {
         return logId;
     }
 
+    /**
+     * Set the in-jeopardy flag and indicate its previous state.
+     *
+     * @param inJeopardy new value of the in-jeopardy flag
+     * @return Previous value of the flag.
+     */
+    boolean setInJeopardy(final boolean inJeopardy) {
+        final boolean wasInJeopardy = this.inJeopardy;
+        this.inJeopardy = inJeopardy;
+        return wasInJeopardy;
+    }
+
     boolean hasCandidateForEntity(Entity entity) {
         return entitiesWithCandidateSet.contains(entity);
     }
@@ -90,7 +103,7 @@ class EntityOwnershipListenerSupport {
 
     private void notifyListeners(Entity entity, boolean wasOwner, boolean isOwner, boolean hasOwner,
             Collection<EntityOwnershipListener> listeners) {
-        EntityOwnershipChange changed = new EntityOwnershipChange(entity, wasOwner, isOwner, hasOwner);
+        EntityOwnershipChange changed = new EntityOwnershipChange(entity, wasOwner, isOwner, hasOwner, inJeopardy);
         for(EntityOwnershipListener listener: listeners) {
             ActorRef listenerActor = listenerActorFor(listener);
 

diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShard.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShard.java
index 3a66924edca7baaa926f5bee0fb54f8d8df89ee3..749f6ef189b58084653327aeaa5a0e1d53ca5d24 100644 (file)
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShard.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShard.java
@@ -55,6 +55,7 @@ import org.opendaylight.controller.cluster.datastore.messages.SuccessReply;
 import org.opendaylight.controller.cluster.datastore.modification.DeleteModification;
 import org.opendaylight.controller.cluster.datastore.modification.MergeModification;
 import org.opendaylight.controller.cluster.datastore.modification.WriteModification;
+import org.opendaylight.controller.cluster.raft.RaftState;
 import org.opendaylight.controller.md.sal.common.api.clustering.Entity;
 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
 import org.opendaylight.yangtools.yang.data.api.schema.DataContainerChild;
@@ -253,13 +254,62 @@ class EntityOwnershipShard extends Shard {
         return getLeader() != null && !isIsolatedLeader();
     }
 
+    /**
+     * Determine if we are in jeopardy based on observed RAFT state.
+     */
+    private static boolean inJeopardy(final RaftState state) {
+        switch (state) {
+            case Candidate:
+            case Follower:
+            case Leader:
+                return false;
+            case IsolatedLeader:
+                return true;
+        }
+        throw new IllegalStateException("Unsupported RAFT state " + state);
+    }
+
+    private void notifyAllListeners() {
+        searchForEntities(new EntityWalker() {
+            @Override
+            public void onEntity(MapEntryNode entityTypeNode, MapEntryNode entityNode) {
+                Optional<DataContainerChild<?, ?>> possibleType = entityTypeNode.getChild(ENTITY_TYPE_NODE_ID);
+                if (possibleType.isPresent()) {
+                    final boolean hasOwner;
+                    final boolean isOwner;
+
+                    Optional<DataContainerChild<?, ?>> possibleOwner = entityNode.getChild(ENTITY_OWNER_NODE_ID);
+                    if (possibleOwner.isPresent()) {
+                        isOwner = localMemberName.equals(possibleOwner.get().getValue().toString());
+                        hasOwner = true;
+                    } else {
+                        isOwner = false;
+                        hasOwner = false;
+                    }
+
+                    Entity entity = new Entity(possibleType.get().getValue().toString(),
+                        (YangInstanceIdentifier) entityNode.getChild(ENTITY_ID_NODE_ID).get().getValue());
+
+                    listenerSupport.notifyEntityOwnershipListeners(entity, isOwner, isOwner, hasOwner);
+                }
+            }
+        });
+    }
+
     @Override
     protected void onStateChanged() {
         super.onStateChanged();
 
         boolean isLeader = isLeader();
-        if(LOG.isDebugEnabled()) {
-            LOG.debug("{}: onStateChanged: isLeader: {}, hasLeader: {}", persistenceId(), isLeader, hasLeader());
+        LOG.debug("{}: onStateChanged: isLeader: {}, hasLeader: {}", persistenceId(), isLeader, hasLeader());
+
+        // Examine current RAFT state to see if we are in jeopardy, potentially notifying all listeners
+        final boolean inJeopardy = inJeopardy(getRaftState());
+        final boolean wasInJeopardy = listenerSupport.setInJeopardy(inJeopardy);
+        if (inJeopardy != wasInJeopardy) {
+            LOG.debug("{}: {} jeopardy state, notifying all listeners", persistenceId(),
+                inJeopardy ? "entered" : "left");
+            notifyAllListeners();
         }
 
         commitCoordinator.onStateChanged(this, isLeader);