Bug 5504: Handle IllegalStateException from commit

Tries to re-apply the transaction if the "store tree and candidate base
differ" IllegalStateException occurs.

Change-Id: If2ef81d88fbd756edd54842d1afb7cd62043de05
Signed-off-by: Tom Pantelis <tpanteli@brocade.com>

diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/Shard.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/Shard.java
index af3177ea03a23d059893fdad376ae6ddca9f111e..1bd06146dec85490cd6d3820d7aef69cfc60e08d 100644 (file)
--- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/Shard.java
+++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/Shard.java
@@ -347,7 +347,22 @@ public class Shard extends RaftActor {
         LOG.debug("{}: Finishing commit for transaction {}", persistenceId(), cohortEntry.getTransactionID());
 
         try {
-            cohortEntry.commit();
+            try {
+                cohortEntry.commit();
+            } catch(IllegalStateException e) {
+                // We may get a "store tree and candidate base differ" IllegalStateException from commit under
+                // certain edge case scenarios so we'll try to re-apply the candidate from scratch as a last
+                // resort. Eg, we're a follower and a tx payload is replicated but the leader goes down before
+                // applying it to the state. We then become the leader and a second tx is pre-committed and
+                // replicated. When consensus occurs, this will cause the first tx to be applied as a foreign
+                // candidate via applyState prior to the second tx. Since the second tx has already been
+                // pre-committed, when the it gets here to commit it will get an IllegalStateException.
+
+                LOG.debug("{}: commit failed for transaction {} - retrying as foreign candidate", persistenceId(),
+                        transactionID, e);
+
+                store.applyForeignCandidate(transactionID, cohortEntry.getCandidate());
+            }
 
             sender.tell(CommitTransactionReply.INSTANCE.toSerializable(), getSelf());