opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/DistributedEntityOwnershipIntegrationTest.java

   1 /*
   2  * Copyright (c) 2015 Brocade Communications Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.controller.cluster.datastore.entityownership;
   9
  10 import static org.junit.Assert.assertEquals;
  11 import static org.junit.Assert.assertNull;
  12 import static org.junit.Assert.assertThat;
  13 import static org.junit.Assert.assertTrue;
  14 import static org.mockito.AdditionalMatchers.or;
  15 import static org.mockito.Mockito.doNothing;
  16 import static org.mockito.Mockito.never;
  17 import static org.mockito.Mockito.reset;
  18 import static org.mockito.Mockito.timeout;
  19 import static org.mockito.Mockito.verify;
  20 import static org.opendaylight.controller.cluster.datastore.MemberNode.verifyRaftState;
  21 import static org.opendaylight.controller.cluster.datastore.entityownership.AbstractEntityOwnershipTest.ownershipChange;
  22 import static org.opendaylight.controller.cluster.datastore.entityownership.DistributedEntityOwnershipService.ENTITY_OWNERSHIP_SHARD_NAME;
  23 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.CANDIDATE_NAME_NODE_ID;
  24 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.entityPath;
  25
  26 import akka.actor.ActorRef;
  27 import akka.actor.Status.Failure;
  28 import akka.actor.Status.Success;
  29 import akka.cluster.Cluster;
  30 import akka.testkit.JavaTestKit;
  31 import com.google.common.base.Optional;
  32 import com.google.common.base.Stopwatch;
  33 import com.google.common.collect.Iterables;
  34 import com.google.common.collect.Lists;
  35 import com.google.common.util.concurrent.Uninterruptibles;
  36 import java.util.ArrayList;
  37 import java.util.Arrays;
  38 import java.util.Collection;
  39 import java.util.List;
  40 import java.util.concurrent.TimeUnit;
  41 import org.junit.After;
  42 import org.junit.Before;
  43 import org.junit.Test;
  44 import org.mockito.ArgumentCaptor;
  45 import org.mockito.Mock;
  46 import org.mockito.Mockito;
  47 import org.mockito.MockitoAnnotations;
  48 import org.mockito.exceptions.base.MockitoException;
  49 import org.opendaylight.controller.cluster.datastore.AbstractDataStore;
  50 import org.opendaylight.controller.cluster.datastore.DatastoreContext;
  51 import org.opendaylight.controller.cluster.datastore.IntegrationTestKit;
  52 import org.opendaylight.controller.cluster.datastore.MemberNode;
  53 import org.opendaylight.controller.cluster.datastore.entityownership.selectionstrategy.EntityOwnerSelectionStrategyConfig;
  54 import org.opendaylight.controller.cluster.datastore.messages.AddShardReplica;
  55 import org.opendaylight.controller.cluster.raft.RaftState;
  56 import org.opendaylight.controller.cluster.raft.policy.DisableElectionsRaftPolicy;
  57 import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal;
  58 import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore;
  59 import org.opendaylight.controller.md.cluster.datastore.model.SchemaContextHelper;
  60 import org.opendaylight.mdsal.eos.common.api.EntityOwnershipState;
  61 import org.opendaylight.mdsal.eos.dom.api.DOMEntity;
  62 import org.opendaylight.mdsal.eos.dom.api.DOMEntityOwnershipCandidateRegistration;
  63 import org.opendaylight.mdsal.eos.dom.api.DOMEntityOwnershipChange;
  64 import org.opendaylight.mdsal.eos.dom.api.DOMEntityOwnershipListener;
  65 import org.opendaylight.mdsal.eos.dom.api.DOMEntityOwnershipService;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.clustering.entity.owners.rev150804.entity.owners.entity.type.entity.Candidate;
  67 import org.opendaylight.yangtools.yang.data.api.schema.MapEntryNode;
  68 import org.opendaylight.yangtools.yang.data.api.schema.MapNode;
  69 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
  70 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
  71
  72 /**
  73  * End-to-end integration tests for the entity ownership functionality.
  74  *
  75  * @author Thomas Pantelis
  76  */
  77 public class DistributedEntityOwnershipIntegrationTest {
  78     private static final String MODULE_SHARDS_CONFIG = "module-shards-default.conf";
  79     private static final String MODULE_SHARDS_5_NODE_CONFIG = "module-shards-default-5-node.conf";
  80     private static final String MODULE_SHARDS_MEMBER_1_CONFIG = "module-shards-default-member-1.conf";
  81     private static final String ENTITY_TYPE1 = "entityType1";
  82     private static final String ENTITY_TYPE2 = "entityType2";
  83     private static final DOMEntity ENTITY1 = new DOMEntity(ENTITY_TYPE1, "entity1");
  84     private static final DOMEntity ENTITY1_2 = new DOMEntity(ENTITY_TYPE2, "entity1");
  85     private static final DOMEntity ENTITY2 = new DOMEntity(ENTITY_TYPE1, "entity2");
  86     private static final DOMEntity ENTITY3 = new DOMEntity(ENTITY_TYPE1, "entity3");
  87     private static final DOMEntity ENTITY4 = new DOMEntity(ENTITY_TYPE1, "entity4");
  88     private static final SchemaContext SCHEMA_CONTEXT = SchemaContextHelper.entityOwners();
  89
  90     private final DatastoreContext.Builder leaderDatastoreContextBuilder =
  91             DatastoreContext.newBuilder().shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(5)
  92                     .shardIsolatedLeaderCheckIntervalInMillis(1000000);
  93
  94     private final DatastoreContext.Builder followerDatastoreContextBuilder =
  95             DatastoreContext.newBuilder().shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(10000);
  96
  97     private final List<MemberNode> memberNodes = new ArrayList<>();
  98
  99     @Mock
 100     private DOMEntityOwnershipListener leaderMockListener;
 101
 102     @Mock
 103     private DOMEntityOwnershipListener leaderMockListener2;
 104
 105     @Mock
 106     private DOMEntityOwnershipListener follower1MockListener;
 107
 108     @Mock
 109     private DOMEntityOwnershipListener follower2MockListener;
 110
 111     @Before
 112     public void setUp() {
 113         MockitoAnnotations.initMocks(this);
 114         InMemoryJournal.clear();
 115         InMemorySnapshotStore.clear();
 116     }
 117
 118     @After
 119     public void tearDown() {
 120         for (MemberNode m : Lists.reverse(memberNodes)) {
 121             m.cleanup();
 122         }
 123         memberNodes.clear();
 124     }
 125
 126     private static DistributedEntityOwnershipService newOwnershipService(final AbstractDataStore datastore) {
 127         return DistributedEntityOwnershipService.start(datastore.getActorContext(),
 128                 EntityOwnerSelectionStrategyConfig.newBuilder().build());
 129     }
 130
 131     @Test
 132     public void testFunctionalityWithThreeNodes() throws Exception {
 133         String name = "testFunctionalityWithThreeNodes";
 134         MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
 135                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 136                 .datastoreContextBuilder(leaderDatastoreContextBuilder).build();
 137
 138         MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
 139                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 140                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
 141
 142         MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name)
 143                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 144                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
 145
 146         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
 147
 148         leaderDistributedDataStore.waitTillReady();
 149         follower1Node.configDataStore().waitTillReady();
 150         follower2Node.configDataStore().waitTillReady();
 151
 152         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
 153         final DOMEntityOwnershipService follower1EntityOwnershipService =
 154                 newOwnershipService(follower1Node.configDataStore());
 155         final DOMEntityOwnershipService follower2EntityOwnershipService =
 156                 newOwnershipService(follower2Node.configDataStore());
 157
 158         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
 159
 160         leaderEntityOwnershipService.registerListener(ENTITY_TYPE1, leaderMockListener);
 161         leaderEntityOwnershipService.registerListener(ENTITY_TYPE2, leaderMockListener2);
 162         follower1EntityOwnershipService.registerListener(ENTITY_TYPE1, follower1MockListener);
 163
 164         // Register leader candidate for entity1 and verify it becomes owner
 165
 166         leaderEntityOwnershipService.registerCandidate(ENTITY1);
 167         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, true, true));
 168         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
 169         reset(leaderMockListener, follower1MockListener);
 170
 171         verifyGetOwnershipState(leaderEntityOwnershipService, ENTITY1, EntityOwnershipState.IS_OWNER);
 172         verifyGetOwnershipState(follower1EntityOwnershipService, ENTITY1, EntityOwnershipState.OWNED_BY_OTHER);
 173
 174         // Register leader candidate for entity1_2 (same id, different type) and verify it becomes owner
 175
 176         leaderEntityOwnershipService.registerCandidate(ENTITY1_2);
 177         verify(leaderMockListener2, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1_2, false, true, true));
 178         Uninterruptibles.sleepUninterruptibly(300, TimeUnit.MILLISECONDS);
 179         verify(leaderMockListener, never()).ownershipChanged(ownershipChange(ENTITY1_2));
 180         reset(leaderMockListener2);
 181
 182         // Register follower1 candidate for entity1 and verify it gets added but doesn't become owner
 183
 184         follower1EntityOwnershipService.registerCandidate(ENTITY1);
 185         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-1", "member-2");
 186         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-1");
 187         verifyOwner(follower2Node.configDataStore(), ENTITY1, "member-1");
 188         Uninterruptibles.sleepUninterruptibly(300, TimeUnit.MILLISECONDS);
 189         verify(leaderMockListener, never()).ownershipChanged(ownershipChange(ENTITY1));
 190         verify(follower1MockListener, never()).ownershipChanged(ownershipChange(ENTITY1));
 191
 192         // Register follower1 candidate for entity2 and verify it becomes owner
 193
 194         final DOMEntityOwnershipCandidateRegistration follower1Entity2Reg =
 195                 follower1EntityOwnershipService.registerCandidate(ENTITY2);
 196         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, true, true));
 197         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, false, true));
 198         verifyOwner(follower2Node.configDataStore(), ENTITY2, "member-2");
 199         reset(leaderMockListener, follower1MockListener);
 200
 201         // Register follower2 candidate for entity2 and verify it gets added but doesn't become owner
 202
 203         follower2EntityOwnershipService.registerListener(ENTITY_TYPE1, follower2MockListener);
 204         verify(follower2MockListener, timeout(5000).times(2)).ownershipChanged(or(
 205                 ownershipChange(ENTITY1, false, false, true), ownershipChange(ENTITY2, false, false, true)));
 206
 207         follower2EntityOwnershipService.registerCandidate(ENTITY2);
 208         verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-2", "member-3");
 209         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-2");
 210
 211         // Unregister follower1 candidate for entity2 and verify follower2 becomes owner
 212
 213         follower1Entity2Reg.close();
 214         verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-3");
 215         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-3");
 216         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, true, false, true));
 217         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, false, true));
 218
 219         // Depending on timing, follower2MockListener could get ownershipChanged with "false, false, true" if
 220         // if the original ownership change with "member-2 is replicated to follower2 after the listener is
 221         // registered.
 222         Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
 223         verify(follower2MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, true, true));
 224
 225         // Register follower1 candidate for entity3 and verify it becomes owner
 226
 227         follower1EntityOwnershipService.registerCandidate(ENTITY3);
 228         verifyOwner(leaderDistributedDataStore, ENTITY3, "member-2");
 229         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY3, false, true, true));
 230         verify(follower2MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY3, false, false, true));
 231         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY3, false, false, true));
 232
 233         // Register follower2 candidate for entity4 and verify it becomes owner
 234
 235         follower2EntityOwnershipService.registerCandidate(ENTITY4);
 236         verifyOwner(leaderDistributedDataStore, ENTITY4, "member-3");
 237         verify(follower2MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY4, false, true, true));
 238         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY4, false, false, true));
 239         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY4, false, false, true));
 240         reset(follower1MockListener, follower2MockListener);
 241
 242         // Register follower1 candidate for entity4 and verify it gets added but doesn't become owner
 243
 244         follower1EntityOwnershipService.registerCandidate(ENTITY4);
 245         verifyCandidates(leaderDistributedDataStore, ENTITY4, "member-3", "member-2");
 246         verifyOwner(leaderDistributedDataStore, ENTITY4, "member-3");
 247
 248         // Shutdown follower2 and verify it's owned entities (entity 4) get re-assigned
 249
 250         reset(leaderMockListener, follower1MockListener);
 251         follower2Node.cleanup();
 252
 253         verify(follower1MockListener, timeout(15000)).ownershipChanged(ownershipChange(ENTITY4, false, true, true));
 254         verify(leaderMockListener, timeout(15000)).ownershipChanged(ownershipChange(ENTITY4, false, false, true));
 255
 256         // Register leader candidate for entity2 and verify it becomes owner
 257
 258         DOMEntityOwnershipCandidateRegistration leaderEntity2Reg =
 259                 leaderEntityOwnershipService.registerCandidate(ENTITY2);
 260         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
 261         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, true, true));
 262
 263         // Unregister leader candidate for entity2 and verify the owner is cleared
 264
 265         leaderEntity2Reg.close();
 266         verifyOwner(leaderDistributedDataStore, ENTITY2, "");
 267         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, true, false, false));
 268         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, false, false));
 269     }
 270
 271     @Test
 272     public void testLeaderEntityOwnersReassignedAfterShutdown() throws Exception {
 273         followerDatastoreContextBuilder.shardElectionTimeoutFactor(5)
 274                     .customRaftPolicyImplementation(DisableElectionsRaftPolicy.class.getName());
 275
 276         String name = "testLeaderEntityOwnersReassignedAfterShutdown";
 277         MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
 278                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 279                 .datastoreContextBuilder(leaderDatastoreContextBuilder).build();
 280
 281         MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
 282                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 283                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
 284
 285         MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name)
 286                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 287                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
 288
 289         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
 290
 291         leaderDistributedDataStore.waitTillReady();
 292         follower1Node.configDataStore().waitTillReady();
 293         follower2Node.configDataStore().waitTillReady();
 294
 295         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
 296         final DOMEntityOwnershipService follower1EntityOwnershipService =
 297                 newOwnershipService(follower1Node.configDataStore());
 298         final DOMEntityOwnershipService follower2EntityOwnershipService =
 299                 newOwnershipService(follower2Node.configDataStore());
 300
 301         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
 302
 303         // Register follower1 candidate for entity1 and verify it becomes owner
 304
 305         follower1EntityOwnershipService.registerCandidate(ENTITY1);
 306         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
 307
 308         // Register leader candidate for entity1
 309
 310         leaderEntityOwnershipService.registerCandidate(ENTITY1);
 311         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-2", "member-1");
 312         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
 313
 314         // Register leader candidate for entity2 and verify it becomes owner
 315
 316         leaderEntityOwnershipService.registerCandidate(ENTITY2);
 317         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
 318
 319         // Register follower2 candidate for entity2
 320
 321         follower2EntityOwnershipService.registerCandidate(ENTITY2);
 322         verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-1", "member-3");
 323         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
 324
 325         // Shutdown the leader and verify its removed from the candidate list
 326
 327         leaderNode.cleanup();
 328         follower1Node.waitForMemberDown("member-1");
 329
 330         // Re-enable elections on follower1 so it becomes the leader
 331
 332         ActorRef follower1Shard = IntegrationTestKit.findLocalShard(follower1Node.configDataStore().getActorContext(),
 333                 ENTITY_OWNERSHIP_SHARD_NAME);
 334         follower1Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build())
 335                 .customRaftPolicyImplementation(null).build(), ActorRef.noSender());
 336
 337         MemberNode.verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
 338             raftState -> assertEquals("Raft state", RaftState.Leader.toString(), raftState.getRaftState()));
 339
 340         // Verify the prior leader's entity owners are re-assigned.
 341
 342         verifyCandidates(follower1Node.configDataStore(), ENTITY1, "member-2", "member-1");
 343         verifyCandidates(follower1Node.configDataStore(), ENTITY2, "member-1", "member-3");
 344         verifyOwner(follower1Node.configDataStore(), ENTITY1, "member-2");
 345         verifyOwner(follower1Node.configDataStore(), ENTITY2, "member-3");
 346     }
 347
 348     @Test
 349     public void testLeaderAndFollowerEntityOwnersReassignedAfterShutdown() throws Exception {
 350         followerDatastoreContextBuilder.shardElectionTimeoutFactor(5)
 351                 .customRaftPolicyImplementation(DisableElectionsRaftPolicy.class.getName());
 352
 353         String name = "testLeaderAndFollowerEntityOwnersReassignedAfterShutdown";
 354         final MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1")
 355                 .useAkkaArtery(false).testName(name)
 356                 .moduleShardsConfig(MODULE_SHARDS_5_NODE_CONFIG).schemaContext(SCHEMA_CONTEXT)
 357                 .createOperDatastore(false).datastoreContextBuilder(leaderDatastoreContextBuilder).build();
 358
 359         final MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2")
 360                 .useAkkaArtery(false).testName(name)
 361                 .moduleShardsConfig(MODULE_SHARDS_5_NODE_CONFIG).schemaContext(SCHEMA_CONTEXT)
 362                 .createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
 363
 364         final MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3")
 365                 .useAkkaArtery(false).testName(name)
 366                 .moduleShardsConfig(MODULE_SHARDS_5_NODE_CONFIG).schemaContext(SCHEMA_CONTEXT)
 367                 .createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
 368
 369         final MemberNode follower3Node = MemberNode.builder(memberNodes).akkaConfig("Member4")
 370                 .useAkkaArtery(false).testName(name)
 371                 .moduleShardsConfig(MODULE_SHARDS_5_NODE_CONFIG).schemaContext(SCHEMA_CONTEXT)
 372                 .createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
 373
 374         final MemberNode follower4Node = MemberNode.builder(memberNodes).akkaConfig("Member5")
 375                 .useAkkaArtery(false).testName(name)
 376                 .moduleShardsConfig(MODULE_SHARDS_5_NODE_CONFIG).schemaContext(SCHEMA_CONTEXT)
 377                 .createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
 378
 379         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
 380
 381         leaderDistributedDataStore.waitTillReady();
 382         follower1Node.configDataStore().waitTillReady();
 383         follower2Node.configDataStore().waitTillReady();
 384
 385         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
 386         final DOMEntityOwnershipService follower1EntityOwnershipService =
 387                 newOwnershipService(follower1Node.configDataStore());
 388         final DOMEntityOwnershipService follower2EntityOwnershipService =
 389                 newOwnershipService(follower2Node.configDataStore());
 390         final DOMEntityOwnershipService follower3EntityOwnershipService =
 391                 newOwnershipService(follower3Node.configDataStore());
 392         newOwnershipService(follower4Node.configDataStore());
 393
 394         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
 395
 396         // Register follower1 candidate for entity1 and verify it becomes owner
 397
 398         follower1EntityOwnershipService.registerCandidate(ENTITY1);
 399         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
 400
 401         // Register leader candidate for entity1
 402
 403         leaderEntityOwnershipService.registerCandidate(ENTITY1);
 404         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-2", "member-1");
 405         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
 406
 407         // Register leader candidate for entity2 and verify it becomes owner
 408
 409         leaderEntityOwnershipService.registerCandidate(ENTITY2);
 410         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
 411
 412         // Register follower2 candidate for entity2
 413
 414         follower2EntityOwnershipService.registerCandidate(ENTITY2);
 415         verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-1", "member-3");
 416         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
 417
 418         // Register follower3 as a candidate for entity2 as well
 419
 420         follower3EntityOwnershipService.registerCandidate(ENTITY2);
 421         verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-1", "member-3", "member-4");
 422         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
 423
 424         // Shutdown the leader and follower3
 425
 426         leaderNode.cleanup();
 427         follower3Node.cleanup();
 428
 429         follower1Node.waitForMemberDown("member-1");
 430         follower1Node.waitForMemberDown("member-4");
 431
 432         // Re-enable elections on follower1 so it becomes the leader
 433
 434         ActorRef follower1Shard = IntegrationTestKit.findLocalShard(follower1Node.configDataStore().getActorContext(),
 435                 ENTITY_OWNERSHIP_SHARD_NAME);
 436         follower1Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build())
 437                 .customRaftPolicyImplementation(null).build(), ActorRef.noSender());
 438
 439         MemberNode.verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
 440             raftState -> assertEquals("Raft state", RaftState.Leader.toString(), raftState.getRaftState()));
 441
 442         // Verify the prior leader's and follower3 entity owners are re-assigned.
 443
 444         verifyCandidates(follower1Node.configDataStore(), ENTITY1, "member-2", "member-1");
 445         verifyCandidates(follower1Node.configDataStore(), ENTITY2, "member-1", "member-3", "member-4");
 446         verifyOwner(follower1Node.configDataStore(), ENTITY1, "member-2");
 447         verifyOwner(follower1Node.configDataStore(), ENTITY2, "member-3");
 448     }
 449
 450     /**
 451      * Reproduces bug <a href="https://bugs.opendaylight.org/show_bug.cgi?id=4554">4554</a>.
 452      */
 453     @Test
 454     public void testCloseCandidateRegistrationInQuickSuccession() throws Exception {
 455         String name = "testCloseCandidateRegistrationInQuickSuccession";
 456         MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
 457                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 458                 .datastoreContextBuilder(leaderDatastoreContextBuilder).build();
 459
 460         MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
 461                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 462                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
 463
 464         MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name)
 465                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 466                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
 467
 468         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
 469
 470         leaderDistributedDataStore.waitTillReady();
 471         follower1Node.configDataStore().waitTillReady();
 472         follower2Node.configDataStore().waitTillReady();
 473
 474         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
 475         final DOMEntityOwnershipService follower1EntityOwnershipService =
 476                 newOwnershipService(follower1Node.configDataStore());
 477         final DOMEntityOwnershipService follower2EntityOwnershipService =
 478                 newOwnershipService(follower2Node.configDataStore());
 479
 480         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
 481
 482         leaderEntityOwnershipService.registerListener(ENTITY_TYPE1, leaderMockListener);
 483         follower1EntityOwnershipService.registerListener(ENTITY_TYPE1, follower1MockListener);
 484         follower2EntityOwnershipService.registerListener(ENTITY_TYPE1, follower2MockListener);
 485
 486         final DOMEntityOwnershipCandidateRegistration candidate1 =
 487                 leaderEntityOwnershipService.registerCandidate(ENTITY1);
 488         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, true, true));
 489
 490         final DOMEntityOwnershipCandidateRegistration candidate2 =
 491                 follower1EntityOwnershipService.registerCandidate(ENTITY1);
 492         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
 493
 494         final DOMEntityOwnershipCandidateRegistration candidate3 =
 495                 follower2EntityOwnershipService.registerCandidate(ENTITY1);
 496         verify(follower2MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
 497
 498         Mockito.reset(leaderMockListener, follower1MockListener, follower2MockListener);
 499
 500         ArgumentCaptor<DOMEntityOwnershipChange> leaderChangeCaptor =
 501                 ArgumentCaptor.forClass(DOMEntityOwnershipChange.class);
 502         ArgumentCaptor<DOMEntityOwnershipChange> follower1ChangeCaptor =
 503                 ArgumentCaptor.forClass(DOMEntityOwnershipChange.class);
 504         ArgumentCaptor<DOMEntityOwnershipChange> follower2ChangeCaptor =
 505                 ArgumentCaptor.forClass(DOMEntityOwnershipChange.class);
 506         doNothing().when(leaderMockListener).ownershipChanged(leaderChangeCaptor.capture());
 507         doNothing().when(follower1MockListener).ownershipChanged(follower1ChangeCaptor.capture());
 508         doNothing().when(follower2MockListener).ownershipChanged(follower2ChangeCaptor.capture());
 509
 510         candidate1.close();
 511         candidate2.close();
 512         candidate3.close();
 513
 514         boolean passed = false;
 515         for (int i = 0; i < 100; i++) {
 516             Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
 517             final Optional<EntityOwnershipState> leaderState = leaderEntityOwnershipService.getOwnershipState(ENTITY1);
 518             final Optional<EntityOwnershipState> follower1State =
 519                     follower1EntityOwnershipService.getOwnershipState(ENTITY1);
 520             final Optional<EntityOwnershipState> follower2State =
 521                     follower2EntityOwnershipService.getOwnershipState(ENTITY1);
 522             final Optional<DOMEntityOwnershipChange> leaderChange = getValueSafely(leaderChangeCaptor);
 523             final Optional<DOMEntityOwnershipChange> follower1Change = getValueSafely(follower1ChangeCaptor);
 524             final Optional<DOMEntityOwnershipChange> follower2Change = getValueSafely(follower2ChangeCaptor);
 525             if (!leaderState.isPresent() || leaderState.get() == EntityOwnershipState.NO_OWNER
 526                     && follower1State.isPresent() && follower1State.get() == EntityOwnershipState.NO_OWNER
 527                     && follower2State.isPresent() && follower2State.get() == EntityOwnershipState.NO_OWNER
 528                     && leaderChange.isPresent() && !leaderChange.get().getState().hasOwner()
 529                     && follower1Change.isPresent() && !follower1Change.get().getState().hasOwner()
 530                     && follower2Change.isPresent() && !follower2Change.get().getState().hasOwner()) {
 531                 passed = true;
 532                 break;
 533             }
 534         }
 535
 536         assertTrue("No ownership change message was sent with hasOwner=false", passed);
 537     }
 538
 539     private static Optional<DOMEntityOwnershipChange> getValueSafely(ArgumentCaptor<DOMEntityOwnershipChange> captor) {
 540         try {
 541             return Optional.fromNullable(captor.getValue());
 542         } catch (MockitoException e) {
 543             // No value was captured
 544             return Optional.absent();
 545         }
 546     }
 547
 548     /**
 549      * Tests bootstrapping the entity-ownership shard when there's no shards initially configured for local
 550      * member. The entity-ownership shard is initially created as inactive (ie remains a follower), requiring
 551      * an AddShardReplica request to join it to an existing leader.
 552      */
 553     @Test
 554     public void testEntityOwnershipShardBootstrapping() throws Exception {
 555         String name = "testEntityOwnershipShardBootstrapping";
 556         String moduleShardsConfig = MODULE_SHARDS_MEMBER_1_CONFIG;
 557         MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
 558                 .moduleShardsConfig(moduleShardsConfig).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 559                 .datastoreContextBuilder(leaderDatastoreContextBuilder).build();
 560
 561         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
 562         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
 563
 564         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
 565
 566         MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
 567                 .moduleShardsConfig(moduleShardsConfig).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 568                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
 569
 570         AbstractDataStore follower1DistributedDataStore = follower1Node.configDataStore();
 571         follower1DistributedDataStore.waitTillReady();
 572
 573         leaderNode.waitForMembersUp("member-2");
 574         follower1Node.waitForMembersUp("member-1");
 575
 576         DOMEntityOwnershipService follower1EntityOwnershipService = newOwnershipService(follower1DistributedDataStore);
 577
 578         leaderEntityOwnershipService.registerListener(ENTITY_TYPE1, leaderMockListener);
 579
 580         // Register a candidate for follower1 - should get queued since follower1 has no leader
 581         final DOMEntityOwnershipCandidateRegistration candidateReg =
 582                 follower1EntityOwnershipService.registerCandidate(ENTITY1);
 583         Uninterruptibles.sleepUninterruptibly(300, TimeUnit.MILLISECONDS);
 584         verify(leaderMockListener, never()).ownershipChanged(ownershipChange(ENTITY1));
 585
 586         // Add replica in follower1
 587         AddShardReplica addReplica = new AddShardReplica(ENTITY_OWNERSHIP_SHARD_NAME);
 588         follower1DistributedDataStore.getActorContext().getShardManager().tell(addReplica,
 589                 follower1Node.kit().getRef());
 590         Object reply = follower1Node.kit().expectMsgAnyClassOf(JavaTestKit.duration("5 sec"),
 591                 Success.class, Failure.class);
 592         if (reply instanceof Failure) {
 593             throw new AssertionError("AddShardReplica failed", ((Failure)reply).cause());
 594         }
 595
 596         // The queued candidate registration should proceed
 597         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
 598         reset(leaderMockListener);
 599
 600         candidateReg.close();
 601         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, false));
 602         reset(leaderMockListener);
 603
 604         // Restart follower1 and verify the entity ownership shard is re-instated by registering.
 605         Cluster.get(leaderNode.kit().getSystem()).down(Cluster.get(follower1Node.kit().getSystem()).selfAddress());
 606         follower1Node.cleanup();
 607
 608         follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
 609                 .moduleShardsConfig(moduleShardsConfig).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 610                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
 611         follower1EntityOwnershipService = newOwnershipService(follower1Node.configDataStore());
 612
 613         follower1EntityOwnershipService.registerCandidate(ENTITY1);
 614         verify(leaderMockListener, timeout(20000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
 615
 616         verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME, raftState -> {
 617             assertNull("Custom RaftPolicy class name", raftState.getCustomRaftPolicyClassName());
 618             assertEquals("Peer count", 1, raftState.getPeerAddresses().keySet().size());
 619             assertThat("Peer Id", Iterables.<String>getLast(raftState.getPeerAddresses().keySet()),
 620                     org.hamcrest.CoreMatchers.containsString("member-1"));
 621         });
 622     }
 623
 624     @Test
 625     public void testOwnerSelectedOnRapidUnregisteringAndRegisteringOfCandidates() throws Exception {
 626         String name = "testOwnerSelectedOnRapidUnregisteringAndRegisteringOfCandidates";
 627         MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
 628                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 629                 .datastoreContextBuilder(leaderDatastoreContextBuilder).build();
 630
 631         MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
 632                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 633                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
 634
 635         MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name)
 636                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 637                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
 638
 639         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
 640
 641         leaderDistributedDataStore.waitTillReady();
 642         follower1Node.configDataStore().waitTillReady();
 643         follower2Node.configDataStore().waitTillReady();
 644
 645         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
 646         final DOMEntityOwnershipService follower1EntityOwnershipService =
 647                 newOwnershipService(follower1Node.configDataStore());
 648         newOwnershipService(follower2Node.configDataStore());
 649
 650         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
 651
 652         // Register leader candidate for entity1 and verify it becomes owner
 653
 654         DOMEntityOwnershipCandidateRegistration leaderEntity1Reg =
 655                 leaderEntityOwnershipService.registerCandidate(ENTITY1);
 656
 657         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-1");
 658         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-1");
 659
 660         leaderEntity1Reg.close();
 661         follower1EntityOwnershipService.registerCandidate(ENTITY1);
 662
 663         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-2");
 664         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
 665     }
 666
 667     @Test
 668     public void testOwnerSelectedOnRapidRegisteringAndUnregisteringOfCandidates() throws Exception {
 669         String name = "testOwnerSelectedOnRapidRegisteringAndUnregisteringOfCandidates";
 670         MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
 671                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 672                 .datastoreContextBuilder(leaderDatastoreContextBuilder).build();
 673
 674         MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
 675                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 676                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
 677
 678         MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name)
 679                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 680                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
 681
 682         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
 683
 684         leaderDistributedDataStore.waitTillReady();
 685         follower1Node.configDataStore().waitTillReady();
 686         follower2Node.configDataStore().waitTillReady();
 687
 688         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
 689         final DOMEntityOwnershipService follower1EntityOwnershipService =
 690                 newOwnershipService(follower1Node.configDataStore());
 691         newOwnershipService(follower2Node.configDataStore());
 692
 693         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
 694
 695         // Register leader candidate for entity1 and verify it becomes owner
 696
 697         final DOMEntityOwnershipCandidateRegistration leaderEntity1Reg =
 698                 leaderEntityOwnershipService.registerCandidate(ENTITY1);
 699
 700         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-1");
 701         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-1");
 702
 703         follower1EntityOwnershipService.registerCandidate(ENTITY1);
 704         leaderEntity1Reg.close();
 705
 706         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-2");
 707         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
 708     }
 709
 710     private static void verifyGetOwnershipState(final DOMEntityOwnershipService service, final DOMEntity entity,
 711             final EntityOwnershipState expState) {
 712         Optional<EntityOwnershipState> state = service.getOwnershipState(entity);
 713         assertEquals("getOwnershipState present", true, state.isPresent());
 714         assertEquals("EntityOwnershipState", expState, state.get());
 715     }
 716
 717     private static void verifyCandidates(final AbstractDataStore dataStore, final DOMEntity entity,
 718             final String... expCandidates) throws Exception {
 719         AssertionError lastError = null;
 720         Stopwatch sw = Stopwatch.createStarted();
 721         while (sw.elapsed(TimeUnit.MILLISECONDS) <= 10000) {
 722             Optional<NormalizedNode<?, ?>> possible = dataStore.newReadOnlyTransaction()
 723                     .read(entityPath(entity.getType(), entity.getIdentifier()).node(Candidate.QNAME))
 724                     .get(5, TimeUnit.SECONDS);
 725             try {
 726                 assertEquals("Candidates not found for " + entity, true, possible.isPresent());
 727                 Collection<String> actual = new ArrayList<>();
 728                 for (MapEntryNode candidate: ((MapNode)possible.get()).getValue()) {
 729                     actual.add(candidate.getChild(CANDIDATE_NAME_NODE_ID).get().getValue().toString());
 730                 }
 731
 732                 assertEquals("Candidates for " + entity, Arrays.asList(expCandidates), actual);
 733                 return;
 734             } catch (AssertionError e) {
 735                 lastError = e;
 736                 Uninterruptibles.sleepUninterruptibly(300, TimeUnit.MILLISECONDS);
 737             }
 738         }
 739
 740         throw lastError;
 741     }
 742
 743     @SuppressWarnings("checkstyle:IllegalCatch")
 744     private static void verifyOwner(final AbstractDataStore dataStore, final DOMEntity entity,
 745             final String expOwner) {
 746         AbstractEntityOwnershipTest.verifyOwner(expOwner, entity.getType(), entity.getIdentifier(), path -> {
 747             try {
 748                 return dataStore.newReadOnlyTransaction().read(path).get(5, TimeUnit.SECONDS).get();
 749             } catch (Exception e) {
 750                 return null;
 751             }
 752         });
 753     }
 754 }