From: Tom Pantelis Date: Wed, 27 Jan 2016 00:33:04 +0000 (-0500) Subject: Bug 4992: Removed old leader's candidates on leader change X-Git-Tag: release/boron~418 X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?p=controller.git;a=commitdiff_plain;h=207129172cb981630f955170cb67efceba02df85 Bug 4992: Removed old leader's candidates on leader change Modified onLeaderChanged to call removeCandidateFromEntities same as onPeerDown. Change-Id: I9b56e64254485fa0de4fdc1b7f4f6ddf100338af Signed-off-by: Tom Pantelis --- diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShard.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShard.java index 30fafbbb10..74cc6717dc 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShard.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/datastore/entityownership/EntityOwnershipShard.java @@ -7,7 +7,6 @@ */ package org.opendaylight.controller.cluster.datastore.entityownership; -import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.CANDIDATE_NAME_NODE_ID; import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.CANDIDATE_NODE_ID; import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_ID_NODE_ID; import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.ENTITY_ID_QNAME; @@ -269,7 +268,7 @@ class EntityOwnershipShard extends Shard { LOG.debug("{}: oldLeaderMemberName: {}", persistenceId(), oldLeaderMemberName); if(downPeerMemberNames.contains(oldLeaderMemberName)) { - selectNewOwnerForEntitiesOwnedBy(oldLeaderMemberName); + removeCandidateFromEntities(oldLeaderMemberName); } } else { // The leader changed - notify the coordinator to check if pending modifications need to be sent. @@ -359,32 +358,6 @@ class EntityOwnershipShard extends Shard { commitCoordinator.onStateChanged(this, isLeader()); } - private void selectNewOwnerForEntitiesOwnedBy(String owner) { - final BatchedModifications modifications = commitCoordinator.newBatchedModifications(); - searchForEntitiesOwnedBy(owner, new EntityWalker() { - @Override - public void onEntity(MapEntryNode entityTypeNode, MapEntryNode entityNode) { - - YangInstanceIdentifier entityPath = YangInstanceIdentifier.builder(ENTITY_TYPES_PATH). - node(entityTypeNode.getIdentifier()).node(ENTITY_NODE_ID).node(entityNode.getIdentifier()). - node(ENTITY_OWNER_NODE_ID).build(); - - String entityType = EntityOwnersModel.entityTypeFromEntityPath(entityPath); - - Object newOwner = newOwner(getCandidateNames(entityNode), - entityOwnershipStatistics.byEntityType(entityType), - getEntityOwnerElectionStrategy(entityPath)); - - LOG.debug("{}: Found entity {}, writing new owner {}", persistenceId(), entityPath, newOwner); - - modifications.addModification(new WriteModification(entityPath, - ImmutableNodes.leafNode(ENTITY_OWNER_NODE_ID, newOwner))); - } - }); - - commitCoordinator.commitModifications(modifications, this); - } - private void removeCandidateFromEntities(final String owner) { final BatchedModifications modifications = commitCoordinator.newBatchedModifications(); searchForEntities(new EntityWalker() { @@ -451,16 +424,6 @@ class EntityOwnershipShard extends Shard { } } - private static Collection getCandidateNames(MapEntryNode entity) { - Collection candidates = ((MapNode) entity.getChild(CANDIDATE_NODE_ID).get()).getValue(); - Collection candidateNames = new ArrayList<>(candidates.size()); - for(MapEntryNode candidate: candidates) { - candidateNames.add(candidate.getChild(CANDIDATE_NAME_NODE_ID).get().getValue().toString()); - } - - return candidateNames; - } - private void writeNewOwner(YangInstanceIdentifier entityPath, String newOwner) { LOG.debug("{}: Writing new owner {} for entity {}", persistenceId(), newOwner, entityPath); diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/MemberNode.java b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/MemberNode.java index 0c1a6a2760..5ae53317f0 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/MemberNode.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/MemberNode.java @@ -100,6 +100,28 @@ public class MemberNode { fail("Member(s) " + otherMembersSet + " are not Up"); } + public void waitForMemberDown(String member) { + Stopwatch sw = Stopwatch.createStarted(); + while(sw.elapsed(TimeUnit.SECONDS) <= 10) { + CurrentClusterState state = Cluster.get(kit.getSystem()).state(); + for(Member m: state.getUnreachable()) { + if(member.equals(m.getRoles().iterator().next())) { + return; + } + } + + for(Member m: state.getMembers()) { + if(m.status() != MemberStatus.up() && member.equals(m.getRoles().iterator().next())) { + return; + } + } + + Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS); + } + + fail("Member " + member + " is now down"); + } + public void cleanup() { if(!cleanedUp) { cleanedUp = true; diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/DistributedEntityOwnershipIntegrationTest.java b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/DistributedEntityOwnershipIntegrationTest.java index 25a6dc439a..bad8647abf 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/DistributedEntityOwnershipIntegrationTest.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/DistributedEntityOwnershipIntegrationTest.java @@ -22,6 +22,7 @@ import static org.opendaylight.controller.cluster.datastore.entityownership.Abst import static org.opendaylight.controller.cluster.datastore.entityownership.DistributedEntityOwnershipService.ENTITY_OWNERSHIP_SHARD_NAME; import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.CANDIDATE_NAME_NODE_ID; import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.entityPath; +import akka.actor.ActorRef; import akka.actor.Status.Failure; import akka.actor.Status.Success; import akka.cluster.Cluster; @@ -45,11 +46,14 @@ import org.mockito.Mockito; import org.mockito.MockitoAnnotations; import org.opendaylight.controller.cluster.datastore.DatastoreContext; import org.opendaylight.controller.cluster.datastore.DistributedDataStore; +import org.opendaylight.controller.cluster.datastore.IntegrationTestKit; import org.opendaylight.controller.cluster.datastore.MemberNode; import org.opendaylight.controller.cluster.datastore.MemberNode.RaftStateVerifier; import org.opendaylight.controller.cluster.datastore.entityownership.selectionstrategy.EntityOwnerSelectionStrategyConfig; import org.opendaylight.controller.cluster.datastore.messages.AddShardReplica; +import org.opendaylight.controller.cluster.raft.RaftState; import org.opendaylight.controller.cluster.raft.client.messages.OnDemandRaftState; +import org.opendaylight.controller.cluster.raft.policy.DisableElectionsRaftPolicy; import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal; import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore; import org.opendaylight.controller.md.cluster.datastore.model.SchemaContextHelper; @@ -259,6 +263,85 @@ public class DistributedEntityOwnershipIntegrationTest { verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, false, false)); } + @Test + public void testLeaderCandidatesRemovedAfterShutdown() throws Exception { + followerDatastoreContextBuilder.shardElectionTimeoutFactor(5). + customRaftPolicyImplementation(DisableElectionsRaftPolicy.class.getName()); + + String name = "test"; + MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name ). + moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false). + datastoreContextBuilder(leaderDatastoreContextBuilder).build(); + + MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name ). + moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false). + datastoreContextBuilder(followerDatastoreContextBuilder).build(); + + MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name ). + moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false). + datastoreContextBuilder(followerDatastoreContextBuilder).build(); + + DistributedDataStore leaderDistributedDataStore = leaderNode.configDataStore(); + + leaderDistributedDataStore.waitTillReady(); + follower1Node.configDataStore().waitTillReady(); + follower2Node.configDataStore().waitTillReady(); + + EntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore); + EntityOwnershipService follower1EntityOwnershipService = newOwnershipService(follower1Node.configDataStore()); + EntityOwnershipService follower2EntityOwnershipService = newOwnershipService(follower2Node.configDataStore()); + + leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME); + + // Register follower1 candidate for entity1 and verify it becomes owner + + follower1EntityOwnershipService.registerCandidate(ENTITY1); + verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2"); + + // Register leader candidate for entity1 + + leaderEntityOwnershipService.registerCandidate(ENTITY1); + verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-2", "member-1"); + verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2"); + + // Register leader candidate for entity2 and verify it becomes owner + + leaderEntityOwnershipService.registerCandidate(ENTITY2); + verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1"); + + // Register follower2 candidate for entity2 + + follower2EntityOwnershipService.registerCandidate(ENTITY2); + verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-1", "member-3"); + verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1"); + + // Shutdown the leader and verify its removed from the candidate list + + leaderNode.cleanup(); + follower1Node.waitForMemberDown("member-1"); + + // Re-enable elections on folower1 so it becomes the leader + + ActorRef follower1Shard = IntegrationTestKit.findLocalShard(follower1Node.configDataStore(). + getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME); + follower1Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build()). + customRaftPolicyImplementation(null).build(), ActorRef.noSender()); + + MemberNode.verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME, new RaftStateVerifier() { + @Override + public void verify(OnDemandRaftState raftState) { + assertEquals("Raft state", RaftState.Leader.toString(), raftState.getRaftState()); + } + }); + + // Verify the prior leader's candidates are removed + + verifyCandidates(follower1Node.configDataStore(), ENTITY1, "member-2"); + verifyCandidates(follower1Node.configDataStore(), ENTITY2, "member-3"); + verifyOwner(follower1Node.configDataStore(), ENTITY1, "member-2"); + verifyOwner(follower1Node.configDataStore(), ENTITY2, "member-3"); + } + /** * Reproduces bug 4554 * @@ -417,7 +500,7 @@ public class DistributedEntityOwnershipIntegrationTest { private static void verifyCandidates(DistributedDataStore dataStore, Entity entity, String... expCandidates) throws Exception { AssertionError lastError = null; Stopwatch sw = Stopwatch.createStarted(); - while(sw.elapsed(TimeUnit.MILLISECONDS) <= 5000) { + while(sw.elapsed(TimeUnit.MILLISECONDS) <= 10000) { Optional> possible = dataStore.newReadOnlyTransaction().read( entityPath(entity.getType(), entity.getId()).node(Candidate.QNAME)).get(5, TimeUnit.SECONDS); try {