opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/entityownership/DistributedEntityOwnershipIntegrationTest.java

   1 /*
   2  * Copyright (c) 2015 Brocade Communications Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.controller.cluster.datastore.entityownership;
   9
  10 import static org.junit.Assert.assertEquals;
  11 import static org.junit.Assert.assertNull;
  12 import static org.junit.Assert.assertThat;
  13 import static org.junit.Assert.assertTrue;
  14 import static org.mockito.Mockito.doNothing;
  15 import static org.mockito.Mockito.never;
  16 import static org.mockito.Mockito.reset;
  17 import static org.mockito.Mockito.timeout;
  18 import static org.mockito.Mockito.verify;
  19 import static org.opendaylight.controller.cluster.datastore.MemberNode.verifyRaftState;
  20 import static org.opendaylight.controller.cluster.datastore.entityownership.AbstractEntityOwnershipTest.ownershipChange;
  21 import static org.opendaylight.controller.cluster.datastore.entityownership.DistributedEntityOwnershipService.ENTITY_OWNERSHIP_SHARD_NAME;
  22 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.CANDIDATE_NAME_NODE_ID;
  23 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.entityPath;
  24
  25 import akka.actor.ActorRef;
  26 import akka.actor.Status.Failure;
  27 import akka.actor.Status.Success;
  28 import akka.cluster.Cluster;
  29 import akka.testkit.JavaTestKit;
  30 import com.google.common.base.Optional;
  31 import com.google.common.base.Stopwatch;
  32 import com.google.common.collect.Iterables;
  33 import com.google.common.collect.Lists;
  34 import com.google.common.util.concurrent.Uninterruptibles;
  35 import java.util.ArrayList;
  36 import java.util.Arrays;
  37 import java.util.Collection;
  38 import java.util.List;
  39 import java.util.concurrent.TimeUnit;
  40 import org.junit.After;
  41 import org.junit.Before;
  42 import org.junit.Test;
  43 import org.mockito.ArgumentCaptor;
  44 import org.mockito.Mock;
  45 import org.mockito.Mockito;
  46 import org.mockito.MockitoAnnotations;
  47 import org.mockito.exceptions.base.MockitoException;
  48 import org.opendaylight.controller.cluster.datastore.AbstractDataStore;
  49 import org.opendaylight.controller.cluster.datastore.DatastoreContext;
  50 import org.opendaylight.controller.cluster.datastore.IntegrationTestKit;
  51 import org.opendaylight.controller.cluster.datastore.MemberNode;
  52 import org.opendaylight.controller.cluster.datastore.entityownership.selectionstrategy.EntityOwnerSelectionStrategyConfig;
  53 import org.opendaylight.controller.cluster.datastore.messages.AddShardReplica;
  54 import org.opendaylight.controller.cluster.raft.RaftState;
  55 import org.opendaylight.controller.cluster.raft.policy.DisableElectionsRaftPolicy;
  56 import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal;
  57 import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore;
  58 import org.opendaylight.controller.md.cluster.datastore.model.SchemaContextHelper;
  59 import org.opendaylight.mdsal.eos.common.api.CandidateAlreadyRegisteredException;
  60 import org.opendaylight.mdsal.eos.common.api.EntityOwnershipState;
  61 import org.opendaylight.mdsal.eos.dom.api.DOMEntity;
  62 import org.opendaylight.mdsal.eos.dom.api.DOMEntityOwnershipCandidateRegistration;
  63 import org.opendaylight.mdsal.eos.dom.api.DOMEntityOwnershipChange;
  64 import org.opendaylight.mdsal.eos.dom.api.DOMEntityOwnershipListener;
  65 import org.opendaylight.mdsal.eos.dom.api.DOMEntityOwnershipService;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.clustering.entity.owners.rev150804.entity.owners.entity.type.entity.Candidate;
  67 import org.opendaylight.yangtools.yang.data.api.schema.MapEntryNode;
  68 import org.opendaylight.yangtools.yang.data.api.schema.MapNode;
  69 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
  70 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
  71
  72 /**
  73  * End-to-end integration tests for the entity ownership functionality.
  74  *
  75  * @author Thomas Pantelis
  76  */
  77 public class DistributedEntityOwnershipIntegrationTest {
  78     private static final String MODULE_SHARDS_CONFIG = "module-shards-default.conf";
  79     private static final String MODULE_SHARDS_5_NODE_CONFIG = "module-shards-default-5-node.conf";
  80     private static final String MODULE_SHARDS_MEMBER_1_CONFIG = "module-shards-default-member-1.conf";
  81     private static final String ENTITY_TYPE1 = "entityType1";
  82     private static final String ENTITY_TYPE2 = "entityType2";
  83     private static final DOMEntity ENTITY1 = new DOMEntity(ENTITY_TYPE1, "entity1");
  84     private static final DOMEntity ENTITY1_2 = new DOMEntity(ENTITY_TYPE2, "entity1");
  85     private static final DOMEntity ENTITY2 = new DOMEntity(ENTITY_TYPE1, "entity2");
  86     private static final DOMEntity ENTITY3 = new DOMEntity(ENTITY_TYPE1, "entity3");
  87     private static final DOMEntity ENTITY4 = new DOMEntity(ENTITY_TYPE1, "entity4");
  88     private static final SchemaContext SCHEMA_CONTEXT = SchemaContextHelper.entityOwners();
  89
  90     private final DatastoreContext.Builder leaderDatastoreContextBuilder =
  91             DatastoreContext.newBuilder().shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(5)
  92                     .shardIsolatedLeaderCheckIntervalInMillis(1000000);
  93
  94     private final DatastoreContext.Builder followerDatastoreContextBuilder =
  95             DatastoreContext.newBuilder().shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(10000);
  96
  97     private final List<MemberNode> memberNodes = new ArrayList<>();
  98
  99     @Mock
 100     private DOMEntityOwnershipListener leaderMockListener;
 101
 102     @Mock
 103     private DOMEntityOwnershipListener leaderMockListener2;
 104
 105     @Mock
 106     private DOMEntityOwnershipListener follower1MockListener;
 107
 108     @Mock
 109     private DOMEntityOwnershipListener follower2MockListener;
 110
 111     @Before
 112     public void setUp() {
 113         MockitoAnnotations.initMocks(this);
 114         InMemoryJournal.clear();
 115         InMemorySnapshotStore.clear();
 116     }
 117
 118     @After
 119     public void tearDown() {
 120         for (MemberNode m : Lists.reverse(memberNodes)) {
 121             m.cleanup();
 122         }
 123         memberNodes.clear();
 124     }
 125
 126     private static DistributedEntityOwnershipService newOwnershipService(final AbstractDataStore datastore) {
 127         return DistributedEntityOwnershipService.start(datastore.getActorContext(),
 128                 EntityOwnerSelectionStrategyConfig.newBuilder().build());
 129     }
 130
 131     @Test
 132     public void testFunctionalityWithThreeNodes() throws Exception {
 133         String name = "test";
 134         MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
 135                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 136                 .datastoreContextBuilder(leaderDatastoreContextBuilder).build();
 137
 138         MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
 139                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 140                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
 141
 142         MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name)
 143                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 144                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
 145
 146         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
 147
 148         leaderDistributedDataStore.waitTillReady();
 149         follower1Node.configDataStore().waitTillReady();
 150         follower2Node.configDataStore().waitTillReady();
 151
 152         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
 153         final DOMEntityOwnershipService follower1EntityOwnershipService =
 154                 newOwnershipService(follower1Node.configDataStore());
 155         final DOMEntityOwnershipService follower2EntityOwnershipService =
 156                 newOwnershipService(follower2Node.configDataStore());
 157
 158         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
 159
 160         leaderEntityOwnershipService.registerListener(ENTITY_TYPE1, leaderMockListener);
 161         leaderEntityOwnershipService.registerListener(ENTITY_TYPE2, leaderMockListener2);
 162         follower1EntityOwnershipService.registerListener(ENTITY_TYPE1, follower1MockListener);
 163
 164         // Register leader candidate for entity1 and verify it becomes owner
 165
 166         leaderEntityOwnershipService.registerCandidate(ENTITY1);
 167         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, true, true));
 168         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
 169         reset(leaderMockListener, follower1MockListener);
 170
 171         verifyGetOwnershipState(leaderEntityOwnershipService, ENTITY1, EntityOwnershipState.IS_OWNER);
 172         verifyGetOwnershipState(follower1EntityOwnershipService, ENTITY1, EntityOwnershipState.OWNED_BY_OTHER);
 173
 174         // Register leader candidate for entity1_2 (same id, different type) and verify it becomes owner
 175
 176         leaderEntityOwnershipService.registerCandidate(ENTITY1_2);
 177         verify(leaderMockListener2, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1_2, false, true, true));
 178         Uninterruptibles.sleepUninterruptibly(300, TimeUnit.MILLISECONDS);
 179         verify(leaderMockListener, never()).ownershipChanged(ownershipChange(ENTITY1_2));
 180         reset(leaderMockListener2);
 181
 182         // Register follower1 candidate for entity1 and verify it gets added but doesn't become owner
 183
 184         follower1EntityOwnershipService.registerCandidate(ENTITY1);
 185         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-1", "member-2");
 186         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-1");
 187         Uninterruptibles.sleepUninterruptibly(300, TimeUnit.MILLISECONDS);
 188         verify(leaderMockListener, never()).ownershipChanged(ownershipChange(ENTITY1));
 189         verify(follower1MockListener, never()).ownershipChanged(ownershipChange(ENTITY1));
 190
 191         // Register follower1 candidate for entity2 and verify it becomes owner
 192
 193         final DOMEntityOwnershipCandidateRegistration follower1Entity2Reg =
 194                 follower1EntityOwnershipService.registerCandidate(ENTITY2);
 195         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, true, true));
 196         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, false, true));
 197         reset(leaderMockListener, follower1MockListener);
 198
 199         // Register follower2 candidate for entity2 and verify it gets added but doesn't become owner
 200
 201         follower2EntityOwnershipService.registerListener(ENTITY_TYPE1, follower2MockListener);
 202         verify(follower2MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, false, true));
 203         verify(follower2MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
 204
 205         follower2EntityOwnershipService.registerCandidate(ENTITY2);
 206         verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-2", "member-3");
 207         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-2");
 208
 209         // Unregister follower1 candidate for entity2 and verify follower2 becomes owner
 210
 211         follower1Entity2Reg.close();
 212         verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-3");
 213         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-3");
 214         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, true, false, true));
 215         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, false, true));
 216
 217         // Depending on timing, follower2MockListener could get ownershipChanged with "false, false, true" if
 218         // if the original ownership change with "member-2 is replicated to follower2 after the listener is
 219         // registered.
 220         Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
 221         verify(follower2MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, true, true));
 222
 223         // Register follower1 candidate for entity3 and verify it becomes owner
 224
 225         follower1EntityOwnershipService.registerCandidate(ENTITY3);
 226         verifyOwner(leaderDistributedDataStore, ENTITY3, "member-2");
 227         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY3, false, true, true));
 228         verify(follower2MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY3, false, false, true));
 229         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY3, false, false, true));
 230
 231         // Register follower2 candidate for entity4 and verify it becomes owner
 232
 233         follower2EntityOwnershipService.registerCandidate(ENTITY4);
 234         verifyOwner(leaderDistributedDataStore, ENTITY4, "member-3");
 235         verify(follower2MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY4, false, true, true));
 236         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY4, false, false, true));
 237         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY4, false, false, true));
 238         reset(follower1MockListener, follower2MockListener);
 239
 240         // Register follower1 candidate for entity4 and verify it gets added but doesn't become owner
 241
 242         follower1EntityOwnershipService.registerCandidate(ENTITY4);
 243         verifyCandidates(leaderDistributedDataStore, ENTITY4, "member-3", "member-2");
 244         verifyOwner(leaderDistributedDataStore, ENTITY4, "member-3");
 245
 246         // Shutdown follower2 and verify it's owned entities (entity 4) get re-assigned
 247
 248         reset(leaderMockListener, follower1MockListener);
 249         follower2Node.cleanup();
 250
 251         verify(follower1MockListener, timeout(15000)).ownershipChanged(ownershipChange(ENTITY4, false, true, true));
 252         verify(leaderMockListener, timeout(15000)).ownershipChanged(ownershipChange(ENTITY4, false, false, true));
 253
 254         // Register leader candidate for entity2 and verify it becomes owner
 255
 256         DOMEntityOwnershipCandidateRegistration leaderEntity2Reg =
 257                 leaderEntityOwnershipService.registerCandidate(ENTITY2);
 258         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
 259         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, true, true));
 260
 261         // Unregister leader candidate for entity2 and verify the owner is cleared
 262
 263         leaderEntity2Reg.close();
 264         verifyOwner(leaderDistributedDataStore, ENTITY2, "");
 265         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, true, false, false));
 266         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, false, false));
 267     }
 268
 269     @Test
 270     public void testLeaderEntityOwnersReassignedAfterShutdown() throws Exception {
 271         followerDatastoreContextBuilder.shardElectionTimeoutFactor(5)
 272                     .customRaftPolicyImplementation(DisableElectionsRaftPolicy.class.getName());
 273
 274         String name = "test";
 275         MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
 276                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 277                 .datastoreContextBuilder(leaderDatastoreContextBuilder).build();
 278
 279         MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
 280                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 281                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
 282
 283         MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name)
 284                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 285                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
 286
 287         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
 288
 289         leaderDistributedDataStore.waitTillReady();
 290         follower1Node.configDataStore().waitTillReady();
 291         follower2Node.configDataStore().waitTillReady();
 292
 293         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
 294         final DOMEntityOwnershipService follower1EntityOwnershipService =
 295                 newOwnershipService(follower1Node.configDataStore());
 296         final DOMEntityOwnershipService follower2EntityOwnershipService =
 297                 newOwnershipService(follower2Node.configDataStore());
 298
 299         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
 300
 301         // Register follower1 candidate for entity1 and verify it becomes owner
 302
 303         follower1EntityOwnershipService.registerCandidate(ENTITY1);
 304         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
 305
 306         // Register leader candidate for entity1
 307
 308         leaderEntityOwnershipService.registerCandidate(ENTITY1);
 309         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-2", "member-1");
 310         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
 311
 312         // Register leader candidate for entity2 and verify it becomes owner
 313
 314         leaderEntityOwnershipService.registerCandidate(ENTITY2);
 315         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
 316
 317         // Register follower2 candidate for entity2
 318
 319         follower2EntityOwnershipService.registerCandidate(ENTITY2);
 320         verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-1", "member-3");
 321         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
 322
 323         // Shutdown the leader and verify its removed from the candidate list
 324
 325         leaderNode.cleanup();
 326         follower1Node.waitForMemberDown("member-1");
 327
 328         // Re-enable elections on follower1 so it becomes the leader
 329
 330         ActorRef follower1Shard = IntegrationTestKit.findLocalShard(follower1Node.configDataStore().getActorContext(),
 331                 ENTITY_OWNERSHIP_SHARD_NAME);
 332         follower1Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build())
 333                 .customRaftPolicyImplementation(null).build(), ActorRef.noSender());
 334
 335         MemberNode.verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
 336             raftState -> assertEquals("Raft state", RaftState.Leader.toString(), raftState.getRaftState()));
 337
 338         // Verify the prior leader's entity owners are re-assigned.
 339
 340         verifyCandidates(follower1Node.configDataStore(), ENTITY1, "member-2", "member-1");
 341         verifyCandidates(follower1Node.configDataStore(), ENTITY2, "member-1", "member-3");
 342         verifyOwner(follower1Node.configDataStore(), ENTITY1, "member-2");
 343         verifyOwner(follower1Node.configDataStore(), ENTITY2, "member-3");
 344     }
 345
 346     @Test
 347     public void testLeaderAndFollowerEntityOwnersReassignedAfterShutdown() throws Exception {
 348         followerDatastoreContextBuilder.shardElectionTimeoutFactor(5)
 349                 .customRaftPolicyImplementation(DisableElectionsRaftPolicy.class.getName());
 350
 351         String name = "test";
 352         final MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1")
 353                 .useAkkaArtery(false).testName(name)
 354                 .moduleShardsConfig(MODULE_SHARDS_5_NODE_CONFIG).schemaContext(SCHEMA_CONTEXT)
 355                 .createOperDatastore(false).datastoreContextBuilder(leaderDatastoreContextBuilder).build();
 356
 357         final MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2")
 358                 .useAkkaArtery(false).testName(name)
 359                 .moduleShardsConfig(MODULE_SHARDS_5_NODE_CONFIG).schemaContext(SCHEMA_CONTEXT)
 360                 .createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
 361
 362         final MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3")
 363                 .useAkkaArtery(false).testName(name)
 364                 .moduleShardsConfig(MODULE_SHARDS_5_NODE_CONFIG).schemaContext(SCHEMA_CONTEXT)
 365                 .createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
 366
 367         final MemberNode follower3Node = MemberNode.builder(memberNodes).akkaConfig("Member4")
 368                 .useAkkaArtery(false).testName(name)
 369                 .moduleShardsConfig(MODULE_SHARDS_5_NODE_CONFIG).schemaContext(SCHEMA_CONTEXT)
 370                 .createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
 371
 372         final MemberNode follower4Node = MemberNode.builder(memberNodes).akkaConfig("Member5")
 373                 .useAkkaArtery(false).testName(name)
 374                 .moduleShardsConfig(MODULE_SHARDS_5_NODE_CONFIG).schemaContext(SCHEMA_CONTEXT)
 375                 .createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
 376
 377         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
 378
 379         leaderDistributedDataStore.waitTillReady();
 380         follower1Node.configDataStore().waitTillReady();
 381         follower2Node.configDataStore().waitTillReady();
 382
 383         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
 384         final DOMEntityOwnershipService follower1EntityOwnershipService =
 385                 newOwnershipService(follower1Node.configDataStore());
 386         final DOMEntityOwnershipService follower2EntityOwnershipService =
 387                 newOwnershipService(follower2Node.configDataStore());
 388         final DOMEntityOwnershipService follower3EntityOwnershipService =
 389                 newOwnershipService(follower3Node.configDataStore());
 390         newOwnershipService(follower4Node.configDataStore());
 391
 392         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
 393
 394         // Register follower1 candidate for entity1 and verify it becomes owner
 395
 396         follower1EntityOwnershipService.registerCandidate(ENTITY1);
 397         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
 398
 399         // Register leader candidate for entity1
 400
 401         leaderEntityOwnershipService.registerCandidate(ENTITY1);
 402         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-2", "member-1");
 403         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
 404
 405         // Register leader candidate for entity2 and verify it becomes owner
 406
 407         leaderEntityOwnershipService.registerCandidate(ENTITY2);
 408         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
 409
 410         // Register follower2 candidate for entity2
 411
 412         follower2EntityOwnershipService.registerCandidate(ENTITY2);
 413         verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-1", "member-3");
 414         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
 415
 416         // Register follower3 as a candidate for entity2 as well
 417
 418         follower3EntityOwnershipService.registerCandidate(ENTITY2);
 419         verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-1", "member-3", "member-4");
 420         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
 421
 422         // Shutdown the leader and follower3
 423
 424         leaderNode.cleanup();
 425         follower3Node.cleanup();
 426
 427         follower1Node.waitForMemberDown("member-1");
 428         follower1Node.waitForMemberDown("member-4");
 429
 430         // Re-enable elections on follower1 so it becomes the leader
 431
 432         ActorRef follower1Shard = IntegrationTestKit.findLocalShard(follower1Node.configDataStore().getActorContext(),
 433                 ENTITY_OWNERSHIP_SHARD_NAME);
 434         follower1Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build())
 435                 .customRaftPolicyImplementation(null).build(), ActorRef.noSender());
 436
 437         MemberNode.verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
 438             raftState -> assertEquals("Raft state", RaftState.Leader.toString(), raftState.getRaftState()));
 439
 440         // Verify the prior leader's and follower3 entity owners are re-assigned.
 441
 442         verifyCandidates(follower1Node.configDataStore(), ENTITY1, "member-2", "member-1");
 443         verifyCandidates(follower1Node.configDataStore(), ENTITY2, "member-1", "member-3", "member-4");
 444         verifyOwner(follower1Node.configDataStore(), ENTITY1, "member-2");
 445         verifyOwner(follower1Node.configDataStore(), ENTITY2, "member-3");
 446     }
 447
 448     /**
 449      * Reproduces bug <a href="https://bugs.opendaylight.org/show_bug.cgi?id=4554">4554</a>.
 450      */
 451     @Test
 452     public void testCloseCandidateRegistrationInQuickSuccession() throws CandidateAlreadyRegisteredException {
 453         String name = "testCloseCandidateRegistrationInQuickSuccession";
 454         MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
 455                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 456                 .datastoreContextBuilder(leaderDatastoreContextBuilder).build();
 457
 458         MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
 459                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 460                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
 461
 462         MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name)
 463                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 464                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
 465
 466         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
 467
 468         leaderDistributedDataStore.waitTillReady();
 469         follower1Node.configDataStore().waitTillReady();
 470         follower2Node.configDataStore().waitTillReady();
 471
 472         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
 473         final DOMEntityOwnershipService follower1EntityOwnershipService =
 474                 newOwnershipService(follower1Node.configDataStore());
 475         final DOMEntityOwnershipService follower2EntityOwnershipService =
 476                 newOwnershipService(follower2Node.configDataStore());
 477
 478         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
 479
 480         leaderEntityOwnershipService.registerListener(ENTITY_TYPE1, leaderMockListener);
 481         follower1EntityOwnershipService.registerListener(ENTITY_TYPE1, follower1MockListener);
 482         follower2EntityOwnershipService.registerListener(ENTITY_TYPE1, follower2MockListener);
 483
 484         final DOMEntityOwnershipCandidateRegistration candidate1 =
 485                 leaderEntityOwnershipService.registerCandidate(ENTITY1);
 486         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, true, true));
 487
 488         final DOMEntityOwnershipCandidateRegistration candidate2 =
 489                 follower1EntityOwnershipService.registerCandidate(ENTITY1);
 490         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
 491
 492         final DOMEntityOwnershipCandidateRegistration candidate3 =
 493                 follower2EntityOwnershipService.registerCandidate(ENTITY1);
 494         verify(follower2MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
 495
 496         Mockito.reset(leaderMockListener, follower1MockListener, follower2MockListener);
 497
 498         ArgumentCaptor<DOMEntityOwnershipChange> leaderChangeCaptor =
 499                 ArgumentCaptor.forClass(DOMEntityOwnershipChange.class);
 500         ArgumentCaptor<DOMEntityOwnershipChange> follower1ChangeCaptor =
 501                 ArgumentCaptor.forClass(DOMEntityOwnershipChange.class);
 502         ArgumentCaptor<DOMEntityOwnershipChange> follower2ChangeCaptor =
 503                 ArgumentCaptor.forClass(DOMEntityOwnershipChange.class);
 504         doNothing().when(leaderMockListener).ownershipChanged(leaderChangeCaptor.capture());
 505         doNothing().when(follower1MockListener).ownershipChanged(follower1ChangeCaptor.capture());
 506         doNothing().when(follower2MockListener).ownershipChanged(follower2ChangeCaptor.capture());
 507
 508         candidate1.close();
 509         candidate2.close();
 510         candidate3.close();
 511
 512         boolean passed = false;
 513         for (int i = 0; i < 100; i++) {
 514             Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
 515             final Optional<EntityOwnershipState> leaderState = leaderEntityOwnershipService.getOwnershipState(ENTITY1);
 516             final Optional<EntityOwnershipState> follower1State =
 517                     follower1EntityOwnershipService.getOwnershipState(ENTITY1);
 518             final Optional<EntityOwnershipState> follower2State =
 519                     follower2EntityOwnershipService.getOwnershipState(ENTITY1);
 520             final Optional<DOMEntityOwnershipChange> leaderChange = getValueSafely(leaderChangeCaptor);
 521             final Optional<DOMEntityOwnershipChange> follower1Change = getValueSafely(follower1ChangeCaptor);
 522             final Optional<DOMEntityOwnershipChange> follower2Change = getValueSafely(follower2ChangeCaptor);
 523             if (!leaderState.isPresent() || leaderState.get() == EntityOwnershipState.NO_OWNER
 524                     && follower1State.isPresent() && follower1State.get() == EntityOwnershipState.NO_OWNER
 525                     && follower2State.isPresent() && follower2State.get() == EntityOwnershipState.NO_OWNER
 526                     && leaderChange.isPresent() && !leaderChange.get().getState().hasOwner()
 527                     && follower1Change.isPresent() && !follower1Change.get().getState().hasOwner()
 528                     && follower2Change.isPresent() && !follower2Change.get().getState().hasOwner()) {
 529                 passed = true;
 530                 break;
 531             }
 532         }
 533
 534         assertTrue("No ownership change message was sent with hasOwner=false", passed);
 535     }
 536
 537     private static Optional<DOMEntityOwnershipChange> getValueSafely(ArgumentCaptor<DOMEntityOwnershipChange> captor) {
 538         try {
 539             return Optional.fromNullable(captor.getValue());
 540         } catch (MockitoException e) {
 541             // No value was captured
 542             return Optional.absent();
 543         }
 544     }
 545
 546     /**
 547      * Tests bootstrapping the entity-ownership shard when there's no shards initially configured for local
 548      * member. The entity-ownership shard is initially created as inactive (ie remains a follower), requiring
 549      * an AddShardReplica request to join it to an existing leader.
 550      */
 551     @Test
 552     public void testEntityOwnershipShardBootstrapping() throws Exception {
 553         String name = "testEntityOwnershipShardBootstrapping";
 554         String moduleShardsConfig = MODULE_SHARDS_MEMBER_1_CONFIG;
 555         MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
 556                 .moduleShardsConfig(moduleShardsConfig).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 557                 .datastoreContextBuilder(leaderDatastoreContextBuilder).build();
 558
 559         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
 560         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
 561
 562         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
 563
 564         MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
 565                 .moduleShardsConfig(moduleShardsConfig).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 566                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
 567
 568         AbstractDataStore follower1DistributedDataStore = follower1Node.configDataStore();
 569         follower1DistributedDataStore.waitTillReady();
 570
 571         leaderNode.waitForMembersUp("member-2");
 572         follower1Node.waitForMembersUp("member-1");
 573
 574         DOMEntityOwnershipService follower1EntityOwnershipService = newOwnershipService(follower1DistributedDataStore);
 575
 576         leaderEntityOwnershipService.registerListener(ENTITY_TYPE1, leaderMockListener);
 577
 578         // Register a candidate for follower1 - should get queued since follower1 has no leader
 579         final DOMEntityOwnershipCandidateRegistration candidateReg =
 580                 follower1EntityOwnershipService.registerCandidate(ENTITY1);
 581         Uninterruptibles.sleepUninterruptibly(300, TimeUnit.MILLISECONDS);
 582         verify(leaderMockListener, never()).ownershipChanged(ownershipChange(ENTITY1));
 583
 584         // Add replica in follower1
 585         AddShardReplica addReplica = new AddShardReplica(ENTITY_OWNERSHIP_SHARD_NAME);
 586         follower1DistributedDataStore.getActorContext().getShardManager().tell(addReplica,
 587                 follower1Node.kit().getRef());
 588         Object reply = follower1Node.kit().expectMsgAnyClassOf(JavaTestKit.duration("5 sec"),
 589                 Success.class, Failure.class);
 590         if (reply instanceof Failure) {
 591             throw new AssertionError("AddShardReplica failed", ((Failure)reply).cause());
 592         }
 593
 594         // The queued candidate registration should proceed
 595         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
 596         reset(leaderMockListener);
 597
 598         candidateReg.close();
 599         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, false));
 600         reset(leaderMockListener);
 601
 602         // Restart follower1 and verify the entity ownership shard is re-instated by registering.
 603         Cluster.get(leaderNode.kit().getSystem()).down(Cluster.get(follower1Node.kit().getSystem()).selfAddress());
 604         follower1Node.cleanup();
 605
 606         follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
 607                 .moduleShardsConfig(moduleShardsConfig).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 608                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
 609         follower1EntityOwnershipService = newOwnershipService(follower1Node.configDataStore());
 610
 611         follower1EntityOwnershipService.registerCandidate(ENTITY1);
 612         verify(leaderMockListener, timeout(20000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
 613
 614         verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME, raftState -> {
 615             assertNull("Custom RaftPolicy class name", raftState.getCustomRaftPolicyClassName());
 616             assertEquals("Peer count", 1, raftState.getPeerAddresses().keySet().size());
 617             assertThat("Peer Id", Iterables.<String>getLast(raftState.getPeerAddresses().keySet()),
 618                     org.hamcrest.CoreMatchers.containsString("member-1"));
 619         });
 620     }
 621
 622     @Test
 623     public void testOwnerSelectedOnRapidUnregisteringAndRegisteringOfCandidates() throws Exception {
 624         String name = "test";
 625         MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
 626                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 627                 .datastoreContextBuilder(leaderDatastoreContextBuilder).build();
 628
 629         MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
 630                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 631                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
 632
 633         MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name)
 634                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 635                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
 636
 637         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
 638
 639         leaderDistributedDataStore.waitTillReady();
 640         follower1Node.configDataStore().waitTillReady();
 641         follower2Node.configDataStore().waitTillReady();
 642
 643         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
 644         final DOMEntityOwnershipService follower1EntityOwnershipService =
 645                 newOwnershipService(follower1Node.configDataStore());
 646         newOwnershipService(follower2Node.configDataStore());
 647
 648         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
 649
 650         // Register leader candidate for entity1 and verify it becomes owner
 651
 652         DOMEntityOwnershipCandidateRegistration leaderEntity1Reg =
 653                 leaderEntityOwnershipService.registerCandidate(ENTITY1);
 654
 655         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-1");
 656         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-1");
 657
 658         leaderEntity1Reg.close();
 659         follower1EntityOwnershipService.registerCandidate(ENTITY1);
 660
 661         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-2");
 662         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
 663     }
 664
 665     @Test
 666     public void testOwnerSelectedOnRapidRegisteringAndUnregisteringOfCandidates() throws Exception {
 667         String name = "test";
 668         MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
 669                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 670                 .datastoreContextBuilder(leaderDatastoreContextBuilder).build();
 671
 672         MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
 673                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 674                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
 675
 676         MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name)
 677                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
 678                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
 679
 680         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
 681
 682         leaderDistributedDataStore.waitTillReady();
 683         follower1Node.configDataStore().waitTillReady();
 684         follower2Node.configDataStore().waitTillReady();
 685
 686         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
 687         final DOMEntityOwnershipService follower1EntityOwnershipService =
 688                 newOwnershipService(follower1Node.configDataStore());
 689         newOwnershipService(follower2Node.configDataStore());
 690
 691         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
 692
 693         // Register leader candidate for entity1 and verify it becomes owner
 694
 695         final DOMEntityOwnershipCandidateRegistration leaderEntity1Reg =
 696                 leaderEntityOwnershipService.registerCandidate(ENTITY1);
 697
 698         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-1");
 699         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-1");
 700
 701         follower1EntityOwnershipService.registerCandidate(ENTITY1);
 702         leaderEntity1Reg.close();
 703
 704         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-2");
 705         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
 706     }
 707
 708     private static void verifyGetOwnershipState(final DOMEntityOwnershipService service, final DOMEntity entity,
 709             final EntityOwnershipState expState) {
 710         Optional<EntityOwnershipState> state = service.getOwnershipState(entity);
 711         assertEquals("getOwnershipState present", true, state.isPresent());
 712         assertEquals("EntityOwnershipState", expState, state.get());
 713     }
 714
 715     private static void verifyCandidates(final AbstractDataStore dataStore, final DOMEntity entity,
 716             final String... expCandidates) throws Exception {
 717         AssertionError lastError = null;
 718         Stopwatch sw = Stopwatch.createStarted();
 719         while (sw.elapsed(TimeUnit.MILLISECONDS) <= 10000) {
 720             Optional<NormalizedNode<?, ?>> possible = dataStore.newReadOnlyTransaction()
 721                     .read(entityPath(entity.getType(), entity.getIdentifier()).node(Candidate.QNAME))
 722                     .get(5, TimeUnit.SECONDS);
 723             try {
 724                 assertEquals("Candidates not found for " + entity, true, possible.isPresent());
 725                 Collection<String> actual = new ArrayList<>();
 726                 for (MapEntryNode candidate: ((MapNode)possible.get()).getValue()) {
 727                     actual.add(candidate.getChild(CANDIDATE_NAME_NODE_ID).get().getValue().toString());
 728                 }
 729
 730                 assertEquals("Candidates for " + entity, Arrays.asList(expCandidates), actual);
 731                 return;
 732             } catch (AssertionError e) {
 733                 lastError = e;
 734                 Uninterruptibles.sleepUninterruptibly(300, TimeUnit.MILLISECONDS);
 735             }
 736         }
 737
 738         throw lastError;
 739     }
 740
 741     @SuppressWarnings("checkstyle:IllegalCatch")
 742     private static void verifyOwner(final AbstractDataStore dataStore, final DOMEntity entity,
 743             final String expOwner) {
 744         AbstractEntityOwnershipTest.verifyOwner(expOwner, entity.getType(), entity.getIdentifier(), path -> {
 745             try {
 746                 return dataStore.newReadOnlyTransaction().read(path).get(5, TimeUnit.SECONDS).get();
 747             } catch (Exception e) {
 748                 return null;
 749             }
 750         });
 751     }
 752 }