Fix testLeaderAndFollowerEntityOwnersReassignedAfterShutdown failure
[controller.git] / opendaylight / md-sal / sal-distributed-datastore / src / test / java / org / opendaylight / controller / cluster / datastore / entityownership / DistributedEntityOwnershipIntegrationTest.java
1 /*
2  * Copyright (c) 2015 Brocade Communications Systems, Inc. and others.  All rights reserved.
3  *
4  * This program and the accompanying materials are made available under the
5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6  * and is available at http://www.eclipse.org/legal/epl-v10.html
7  */
8 package org.opendaylight.controller.cluster.datastore.entityownership;
9
10 import static org.junit.Assert.assertEquals;
11 import static org.junit.Assert.assertNull;
12 import static org.junit.Assert.assertThat;
13 import static org.junit.Assert.assertTrue;
14 import static org.mockito.AdditionalMatchers.or;
15 import static org.mockito.Mockito.doNothing;
16 import static org.mockito.Mockito.never;
17 import static org.mockito.Mockito.reset;
18 import static org.mockito.Mockito.timeout;
19 import static org.mockito.Mockito.verify;
20 import static org.opendaylight.controller.cluster.datastore.MemberNode.verifyRaftState;
21 import static org.opendaylight.controller.cluster.datastore.entityownership.AbstractEntityOwnershipTest.ownershipChange;
22 import static org.opendaylight.controller.cluster.datastore.entityownership.DistributedEntityOwnershipService.ENTITY_OWNERSHIP_SHARD_NAME;
23 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.CANDIDATE_NAME_NODE_ID;
24 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.entityPath;
25
26 import akka.actor.ActorRef;
27 import akka.actor.Status.Failure;
28 import akka.actor.Status.Success;
29 import akka.cluster.Cluster;
30 import akka.testkit.JavaTestKit;
31 import com.google.common.base.Optional;
32 import com.google.common.base.Stopwatch;
33 import com.google.common.collect.Iterables;
34 import com.google.common.collect.Lists;
35 import com.google.common.util.concurrent.Uninterruptibles;
36 import java.util.ArrayList;
37 import java.util.Arrays;
38 import java.util.Collection;
39 import java.util.List;
40 import java.util.concurrent.TimeUnit;
41 import java.util.concurrent.atomic.AtomicLong;
42 import org.junit.After;
43 import org.junit.Before;
44 import org.junit.Test;
45 import org.mockito.ArgumentCaptor;
46 import org.mockito.Mock;
47 import org.mockito.Mockito;
48 import org.mockito.MockitoAnnotations;
49 import org.mockito.exceptions.base.MockitoException;
50 import org.opendaylight.controller.cluster.datastore.AbstractDataStore;
51 import org.opendaylight.controller.cluster.datastore.DatastoreContext;
52 import org.opendaylight.controller.cluster.datastore.IntegrationTestKit;
53 import org.opendaylight.controller.cluster.datastore.MemberNode;
54 import org.opendaylight.controller.cluster.datastore.entityownership.selectionstrategy.EntityOwnerSelectionStrategyConfig;
55 import org.opendaylight.controller.cluster.datastore.messages.AddShardReplica;
56 import org.opendaylight.controller.cluster.raft.RaftState;
57 import org.opendaylight.controller.cluster.raft.policy.DisableElectionsRaftPolicy;
58 import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal;
59 import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore;
60 import org.opendaylight.controller.md.cluster.datastore.model.SchemaContextHelper;
61 import org.opendaylight.mdsal.eos.common.api.EntityOwnershipState;
62 import org.opendaylight.mdsal.eos.dom.api.DOMEntity;
63 import org.opendaylight.mdsal.eos.dom.api.DOMEntityOwnershipCandidateRegistration;
64 import org.opendaylight.mdsal.eos.dom.api.DOMEntityOwnershipChange;
65 import org.opendaylight.mdsal.eos.dom.api.DOMEntityOwnershipListener;
66 import org.opendaylight.mdsal.eos.dom.api.DOMEntityOwnershipService;
67 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.clustering.entity.owners.rev150804.entity.owners.entity.type.entity.Candidate;
68 import org.opendaylight.yangtools.yang.data.api.schema.MapEntryNode;
69 import org.opendaylight.yangtools.yang.data.api.schema.MapNode;
70 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
71 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
72
73 /**
74  * End-to-end integration tests for the entity ownership functionality.
75  *
76  * @author Thomas Pantelis
77  */
78 public class DistributedEntityOwnershipIntegrationTest {
79     private static final String MODULE_SHARDS_CONFIG = "module-shards-default.conf";
80     private static final String MODULE_SHARDS_5_NODE_CONFIG = "module-shards-default-5-node.conf";
81     private static final String MODULE_SHARDS_MEMBER_1_CONFIG = "module-shards-default-member-1.conf";
82     private static final String ENTITY_TYPE1 = "entityType1";
83     private static final String ENTITY_TYPE2 = "entityType2";
84     private static final DOMEntity ENTITY1 = new DOMEntity(ENTITY_TYPE1, "entity1");
85     private static final DOMEntity ENTITY1_2 = new DOMEntity(ENTITY_TYPE2, "entity1");
86     private static final DOMEntity ENTITY2 = new DOMEntity(ENTITY_TYPE1, "entity2");
87     private static final DOMEntity ENTITY3 = new DOMEntity(ENTITY_TYPE1, "entity3");
88     private static final DOMEntity ENTITY4 = new DOMEntity(ENTITY_TYPE1, "entity4");
89     private static final SchemaContext SCHEMA_CONTEXT = SchemaContextHelper.entityOwners();
90
91     private final DatastoreContext.Builder leaderDatastoreContextBuilder =
92             DatastoreContext.newBuilder().shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(5)
93                     .shardIsolatedLeaderCheckIntervalInMillis(1000000);
94
95     private final DatastoreContext.Builder followerDatastoreContextBuilder =
96             DatastoreContext.newBuilder().shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(10000);
97
98     private final List<MemberNode> memberNodes = new ArrayList<>();
99
100     @Mock
101     private DOMEntityOwnershipListener leaderMockListener;
102
103     @Mock
104     private DOMEntityOwnershipListener leaderMockListener2;
105
106     @Mock
107     private DOMEntityOwnershipListener follower1MockListener;
108
109     @Mock
110     private DOMEntityOwnershipListener follower2MockListener;
111
112     @Before
113     public void setUp() {
114         MockitoAnnotations.initMocks(this);
115         InMemoryJournal.clear();
116         InMemorySnapshotStore.clear();
117     }
118
119     @After
120     public void tearDown() {
121         for (MemberNode m : Lists.reverse(memberNodes)) {
122             m.cleanup();
123         }
124         memberNodes.clear();
125     }
126
127     private static DistributedEntityOwnershipService newOwnershipService(final AbstractDataStore datastore) {
128         return DistributedEntityOwnershipService.start(datastore.getActorContext(),
129                 EntityOwnerSelectionStrategyConfig.newBuilder().build());
130     }
131
132     @Test
133     public void testFunctionalityWithThreeNodes() throws Exception {
134         String name = "testFunctionalityWithThreeNodes";
135         MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
136                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
137                 .datastoreContextBuilder(leaderDatastoreContextBuilder).build();
138
139         MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
140                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
141                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
142
143         MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name)
144                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
145                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
146
147         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
148
149         leaderDistributedDataStore.waitTillReady();
150         follower1Node.configDataStore().waitTillReady();
151         follower2Node.configDataStore().waitTillReady();
152
153         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
154         final DOMEntityOwnershipService follower1EntityOwnershipService =
155                 newOwnershipService(follower1Node.configDataStore());
156         final DOMEntityOwnershipService follower2EntityOwnershipService =
157                 newOwnershipService(follower2Node.configDataStore());
158
159         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
160
161         leaderEntityOwnershipService.registerListener(ENTITY_TYPE1, leaderMockListener);
162         leaderEntityOwnershipService.registerListener(ENTITY_TYPE2, leaderMockListener2);
163         follower1EntityOwnershipService.registerListener(ENTITY_TYPE1, follower1MockListener);
164
165         // Register leader candidate for entity1 and verify it becomes owner
166
167         leaderEntityOwnershipService.registerCandidate(ENTITY1);
168         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, true, true));
169         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
170         reset(leaderMockListener, follower1MockListener);
171
172         verifyGetOwnershipState(leaderEntityOwnershipService, ENTITY1, EntityOwnershipState.IS_OWNER);
173         verifyGetOwnershipState(follower1EntityOwnershipService, ENTITY1, EntityOwnershipState.OWNED_BY_OTHER);
174
175         // Register leader candidate for entity1_2 (same id, different type) and verify it becomes owner
176
177         leaderEntityOwnershipService.registerCandidate(ENTITY1_2);
178         verify(leaderMockListener2, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1_2, false, true, true));
179         Uninterruptibles.sleepUninterruptibly(300, TimeUnit.MILLISECONDS);
180         verify(leaderMockListener, never()).ownershipChanged(ownershipChange(ENTITY1_2));
181         reset(leaderMockListener2);
182
183         // Register follower1 candidate for entity1 and verify it gets added but doesn't become owner
184
185         follower1EntityOwnershipService.registerCandidate(ENTITY1);
186         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-1", "member-2");
187         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-1");
188         verifyOwner(follower2Node.configDataStore(), ENTITY1, "member-1");
189         Uninterruptibles.sleepUninterruptibly(300, TimeUnit.MILLISECONDS);
190         verify(leaderMockListener, never()).ownershipChanged(ownershipChange(ENTITY1));
191         verify(follower1MockListener, never()).ownershipChanged(ownershipChange(ENTITY1));
192
193         // Register follower1 candidate for entity2 and verify it becomes owner
194
195         final DOMEntityOwnershipCandidateRegistration follower1Entity2Reg =
196                 follower1EntityOwnershipService.registerCandidate(ENTITY2);
197         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, true, true));
198         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, false, true));
199         verifyOwner(follower2Node.configDataStore(), ENTITY2, "member-2");
200         reset(leaderMockListener, follower1MockListener);
201
202         // Register follower2 candidate for entity2 and verify it gets added but doesn't become owner
203
204         follower2EntityOwnershipService.registerListener(ENTITY_TYPE1, follower2MockListener);
205         verify(follower2MockListener, timeout(5000).times(2)).ownershipChanged(or(
206                 ownershipChange(ENTITY1, false, false, true), ownershipChange(ENTITY2, false, false, true)));
207
208         follower2EntityOwnershipService.registerCandidate(ENTITY2);
209         verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-2", "member-3");
210         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-2");
211
212         // Unregister follower1 candidate for entity2 and verify follower2 becomes owner
213
214         follower1Entity2Reg.close();
215         verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-3");
216         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-3");
217         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, true, false, true));
218         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, false, true));
219
220         // Depending on timing, follower2MockListener could get ownershipChanged with "false, false, true" if
221         // if the original ownership change with "member-2 is replicated to follower2 after the listener is
222         // registered.
223         Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
224         verify(follower2MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, true, true));
225
226         // Register follower1 candidate for entity3 and verify it becomes owner
227
228         follower1EntityOwnershipService.registerCandidate(ENTITY3);
229         verifyOwner(leaderDistributedDataStore, ENTITY3, "member-2");
230         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY3, false, true, true));
231         verify(follower2MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY3, false, false, true));
232         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY3, false, false, true));
233
234         // Register follower2 candidate for entity4 and verify it becomes owner
235
236         follower2EntityOwnershipService.registerCandidate(ENTITY4);
237         verifyOwner(leaderDistributedDataStore, ENTITY4, "member-3");
238         verify(follower2MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY4, false, true, true));
239         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY4, false, false, true));
240         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY4, false, false, true));
241         reset(follower1MockListener, follower2MockListener);
242
243         // Register follower1 candidate for entity4 and verify it gets added but doesn't become owner
244
245         follower1EntityOwnershipService.registerCandidate(ENTITY4);
246         verifyCandidates(leaderDistributedDataStore, ENTITY4, "member-3", "member-2");
247         verifyOwner(leaderDistributedDataStore, ENTITY4, "member-3");
248
249         // Shutdown follower2 and verify it's owned entities (entity 4) get re-assigned
250
251         reset(leaderMockListener, follower1MockListener);
252         follower2Node.cleanup();
253
254         verify(follower1MockListener, timeout(15000)).ownershipChanged(ownershipChange(ENTITY4, false, true, true));
255         verify(leaderMockListener, timeout(15000)).ownershipChanged(ownershipChange(ENTITY4, false, false, true));
256
257         // Register leader candidate for entity2 and verify it becomes owner
258
259         DOMEntityOwnershipCandidateRegistration leaderEntity2Reg =
260                 leaderEntityOwnershipService.registerCandidate(ENTITY2);
261         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
262         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, true, true));
263
264         // Unregister leader candidate for entity2 and verify the owner is cleared
265
266         leaderEntity2Reg.close();
267         verifyOwner(leaderDistributedDataStore, ENTITY2, "");
268         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, true, false, false));
269         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, false, false));
270     }
271
272     @Test
273     public void testLeaderEntityOwnersReassignedAfterShutdown() throws Exception {
274         followerDatastoreContextBuilder.shardElectionTimeoutFactor(5)
275                     .customRaftPolicyImplementation(DisableElectionsRaftPolicy.class.getName());
276
277         String name = "testLeaderEntityOwnersReassignedAfterShutdown";
278         MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
279                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
280                 .datastoreContextBuilder(leaderDatastoreContextBuilder).build();
281
282         MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
283                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
284                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
285
286         MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name)
287                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
288                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
289
290         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
291
292         leaderDistributedDataStore.waitTillReady();
293         follower1Node.configDataStore().waitTillReady();
294         follower2Node.configDataStore().waitTillReady();
295
296         follower1Node.waitForMembersUp("member-1", "member-3");
297
298         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
299         final DOMEntityOwnershipService follower1EntityOwnershipService =
300                 newOwnershipService(follower1Node.configDataStore());
301         final DOMEntityOwnershipService follower2EntityOwnershipService =
302                 newOwnershipService(follower2Node.configDataStore());
303
304         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
305
306         // Register follower1 candidate for entity1 and verify it becomes owner
307
308         follower1EntityOwnershipService.registerCandidate(ENTITY1);
309         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
310
311         // Register leader candidate for entity1
312
313         leaderEntityOwnershipService.registerCandidate(ENTITY1);
314         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-2", "member-1");
315         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
316
317         // Register leader candidate for entity2 and verify it becomes owner
318
319         leaderEntityOwnershipService.registerCandidate(ENTITY2);
320         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
321
322         // Register follower2 candidate for entity2
323
324         follower2EntityOwnershipService.registerCandidate(ENTITY2);
325         verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-1", "member-3");
326         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
327
328         // Get the leader's lastIndex and verify followers are fully synced before shutting down the leader
329
330         AtomicLong leaderLastIndex = new AtomicLong();
331         MemberNode.verifyRaftState(leaderDistributedDataStore, ENTITY_OWNERSHIP_SHARD_NAME,
332             raftState -> leaderLastIndex.set(raftState.getLastIndex()));
333
334         MemberNode.verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
335             raftState -> assertEquals("Last index", leaderLastIndex.get(), raftState.getLastIndex()));
336
337         MemberNode.verifyRaftState(follower2Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
338             raftState -> assertEquals("Last index", leaderLastIndex.get(), raftState.getLastIndex()));
339
340         // Shutdown the leader and verify its removed from the candidate list
341
342         leaderNode.cleanup();
343         follower1Node.waitForMemberDown("member-1");
344
345         // Re-enable elections on follower1 so it becomes the leader
346
347         ActorRef follower1Shard = IntegrationTestKit.findLocalShard(follower1Node.configDataStore().getActorContext(),
348                 ENTITY_OWNERSHIP_SHARD_NAME);
349         follower1Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build())
350                 .customRaftPolicyImplementation(null).build(), ActorRef.noSender());
351
352         MemberNode.verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
353             raftState -> assertEquals("Raft state", RaftState.Leader.toString(), raftState.getRaftState()));
354
355         // Verify the prior leader's entity owners are re-assigned.
356
357         verifyCandidates(follower1Node.configDataStore(), ENTITY1, "member-2", "member-1");
358         verifyCandidates(follower1Node.configDataStore(), ENTITY2, "member-1", "member-3");
359         verifyOwner(follower1Node.configDataStore(), ENTITY1, "member-2");
360         verifyOwner(follower1Node.configDataStore(), ENTITY2, "member-3");
361     }
362
363     @Test
364     public void testLeaderAndFollowerEntityOwnersReassignedAfterShutdown() throws Exception {
365         followerDatastoreContextBuilder.shardElectionTimeoutFactor(5)
366                 .customRaftPolicyImplementation(DisableElectionsRaftPolicy.class.getName());
367
368         String name = "testLeaderAndFollowerEntityOwnersReassignedAfterShutdown";
369         final MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1")
370                 .useAkkaArtery(false).testName(name)
371                 .moduleShardsConfig(MODULE_SHARDS_5_NODE_CONFIG).schemaContext(SCHEMA_CONTEXT)
372                 .createOperDatastore(false).datastoreContextBuilder(leaderDatastoreContextBuilder).build();
373
374         final MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2")
375                 .useAkkaArtery(false).testName(name)
376                 .moduleShardsConfig(MODULE_SHARDS_5_NODE_CONFIG).schemaContext(SCHEMA_CONTEXT)
377                 .createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
378
379         final MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3")
380                 .useAkkaArtery(false).testName(name)
381                 .moduleShardsConfig(MODULE_SHARDS_5_NODE_CONFIG).schemaContext(SCHEMA_CONTEXT)
382                 .createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
383
384         final MemberNode follower3Node = MemberNode.builder(memberNodes).akkaConfig("Member4")
385                 .useAkkaArtery(false).testName(name)
386                 .moduleShardsConfig(MODULE_SHARDS_5_NODE_CONFIG).schemaContext(SCHEMA_CONTEXT)
387                 .createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
388
389         final MemberNode follower4Node = MemberNode.builder(memberNodes).akkaConfig("Member5")
390                 .useAkkaArtery(false).testName(name)
391                 .moduleShardsConfig(MODULE_SHARDS_5_NODE_CONFIG).schemaContext(SCHEMA_CONTEXT)
392                 .createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
393
394         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
395
396         leaderDistributedDataStore.waitTillReady();
397         follower1Node.configDataStore().waitTillReady();
398         follower2Node.configDataStore().waitTillReady();
399         follower3Node.configDataStore().waitTillReady();
400         follower4Node.configDataStore().waitTillReady();
401
402         leaderNode.waitForMembersUp("member-2", "member-3", "member-4", "member-5");
403         follower1Node.waitForMembersUp("member-1", "member-3", "member-4", "member-5");
404
405         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
406         final DOMEntityOwnershipService follower1EntityOwnershipService =
407                 newOwnershipService(follower1Node.configDataStore());
408         final DOMEntityOwnershipService follower2EntityOwnershipService =
409                 newOwnershipService(follower2Node.configDataStore());
410         final DOMEntityOwnershipService follower3EntityOwnershipService =
411                 newOwnershipService(follower3Node.configDataStore());
412         newOwnershipService(follower4Node.configDataStore());
413
414         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
415
416         // Register follower1 candidate for entity1 and verify it becomes owner
417
418         follower1EntityOwnershipService.registerCandidate(ENTITY1);
419         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
420
421         // Register leader candidate for entity1
422
423         leaderEntityOwnershipService.registerCandidate(ENTITY1);
424         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-2", "member-1");
425         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
426
427         // Register leader candidate for entity2 and verify it becomes owner
428
429         leaderEntityOwnershipService.registerCandidate(ENTITY2);
430         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
431
432         // Register follower2 candidate for entity2
433
434         follower2EntityOwnershipService.registerCandidate(ENTITY2);
435         verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-1", "member-3");
436         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
437
438         // Register follower3 as a candidate for entity2 as well
439
440         follower3EntityOwnershipService.registerCandidate(ENTITY2);
441         verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-1", "member-3", "member-4");
442         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
443
444         // Get the leader's lastIndex and verify followers are fully synced before shutting down the leader
445         AtomicLong leaderLastIndex = new AtomicLong();
446         MemberNode.verifyRaftState(leaderDistributedDataStore, ENTITY_OWNERSHIP_SHARD_NAME,
447             raftState -> leaderLastIndex.set(raftState.getLastIndex()));
448
449         MemberNode.verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
450             raftState -> assertEquals("Last index", leaderLastIndex.get(), raftState.getLastIndex()));
451
452         MemberNode.verifyRaftState(follower2Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
453             raftState -> assertEquals("Last index", leaderLastIndex.get(), raftState.getLastIndex()));
454
455         MemberNode.verifyRaftState(follower4Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
456             raftState -> assertEquals("Last index", leaderLastIndex.get(), raftState.getLastIndex()));
457
458         // Shutdown the leader and follower3
459
460         leaderNode.cleanup();
461         follower3Node.cleanup();
462
463         follower1Node.waitForMemberDown("member-1");
464         follower1Node.waitForMemberDown("member-4");
465
466         // Re-enable elections on follower1 so it becomes the leader
467
468         ActorRef follower1Shard = IntegrationTestKit.findLocalShard(follower1Node.configDataStore().getActorContext(),
469                 ENTITY_OWNERSHIP_SHARD_NAME);
470         follower1Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build())
471                 .customRaftPolicyImplementation(null).build(), ActorRef.noSender());
472
473         MemberNode.verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
474             raftState -> assertEquals("Raft state", RaftState.Leader.toString(), raftState.getRaftState()));
475
476         // Verify the prior leader's and follower3 entity owners are re-assigned.
477
478         verifyCandidates(follower1Node.configDataStore(), ENTITY1, "member-2", "member-1");
479         verifyCandidates(follower1Node.configDataStore(), ENTITY2, "member-1", "member-3", "member-4");
480         verifyOwner(follower1Node.configDataStore(), ENTITY1, "member-2");
481         verifyOwner(follower1Node.configDataStore(), ENTITY2, "member-3");
482     }
483
484     /**
485      * Reproduces bug <a href="https://bugs.opendaylight.org/show_bug.cgi?id=4554">4554</a>.
486      */
487     @Test
488     public void testCloseCandidateRegistrationInQuickSuccession() throws Exception {
489         String name = "testCloseCandidateRegistrationInQuickSuccession";
490         MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
491                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
492                 .datastoreContextBuilder(leaderDatastoreContextBuilder).build();
493
494         MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
495                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
496                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
497
498         MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name)
499                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
500                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
501
502         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
503
504         leaderDistributedDataStore.waitTillReady();
505         follower1Node.configDataStore().waitTillReady();
506         follower2Node.configDataStore().waitTillReady();
507
508         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
509         final DOMEntityOwnershipService follower1EntityOwnershipService =
510                 newOwnershipService(follower1Node.configDataStore());
511         final DOMEntityOwnershipService follower2EntityOwnershipService =
512                 newOwnershipService(follower2Node.configDataStore());
513
514         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
515
516         leaderEntityOwnershipService.registerListener(ENTITY_TYPE1, leaderMockListener);
517         follower1EntityOwnershipService.registerListener(ENTITY_TYPE1, follower1MockListener);
518         follower2EntityOwnershipService.registerListener(ENTITY_TYPE1, follower2MockListener);
519
520         final DOMEntityOwnershipCandidateRegistration candidate1 =
521                 leaderEntityOwnershipService.registerCandidate(ENTITY1);
522         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, true, true));
523
524         final DOMEntityOwnershipCandidateRegistration candidate2 =
525                 follower1EntityOwnershipService.registerCandidate(ENTITY1);
526         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
527
528         final DOMEntityOwnershipCandidateRegistration candidate3 =
529                 follower2EntityOwnershipService.registerCandidate(ENTITY1);
530         verify(follower2MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
531
532         Mockito.reset(leaderMockListener, follower1MockListener, follower2MockListener);
533
534         ArgumentCaptor<DOMEntityOwnershipChange> leaderChangeCaptor =
535                 ArgumentCaptor.forClass(DOMEntityOwnershipChange.class);
536         ArgumentCaptor<DOMEntityOwnershipChange> follower1ChangeCaptor =
537                 ArgumentCaptor.forClass(DOMEntityOwnershipChange.class);
538         ArgumentCaptor<DOMEntityOwnershipChange> follower2ChangeCaptor =
539                 ArgumentCaptor.forClass(DOMEntityOwnershipChange.class);
540         doNothing().when(leaderMockListener).ownershipChanged(leaderChangeCaptor.capture());
541         doNothing().when(follower1MockListener).ownershipChanged(follower1ChangeCaptor.capture());
542         doNothing().when(follower2MockListener).ownershipChanged(follower2ChangeCaptor.capture());
543
544         candidate1.close();
545         candidate2.close();
546         candidate3.close();
547
548         boolean passed = false;
549         for (int i = 0; i < 100; i++) {
550             Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
551             final Optional<EntityOwnershipState> leaderState = leaderEntityOwnershipService.getOwnershipState(ENTITY1);
552             final Optional<EntityOwnershipState> follower1State =
553                     follower1EntityOwnershipService.getOwnershipState(ENTITY1);
554             final Optional<EntityOwnershipState> follower2State =
555                     follower2EntityOwnershipService.getOwnershipState(ENTITY1);
556             final Optional<DOMEntityOwnershipChange> leaderChange = getValueSafely(leaderChangeCaptor);
557             final Optional<DOMEntityOwnershipChange> follower1Change = getValueSafely(follower1ChangeCaptor);
558             final Optional<DOMEntityOwnershipChange> follower2Change = getValueSafely(follower2ChangeCaptor);
559             if (!leaderState.isPresent() || leaderState.get() == EntityOwnershipState.NO_OWNER
560                     && follower1State.isPresent() && follower1State.get() == EntityOwnershipState.NO_OWNER
561                     && follower2State.isPresent() && follower2State.get() == EntityOwnershipState.NO_OWNER
562                     && leaderChange.isPresent() && !leaderChange.get().getState().hasOwner()
563                     && follower1Change.isPresent() && !follower1Change.get().getState().hasOwner()
564                     && follower2Change.isPresent() && !follower2Change.get().getState().hasOwner()) {
565                 passed = true;
566                 break;
567             }
568         }
569
570         assertTrue("No ownership change message was sent with hasOwner=false", passed);
571     }
572
573     private static Optional<DOMEntityOwnershipChange> getValueSafely(ArgumentCaptor<DOMEntityOwnershipChange> captor) {
574         try {
575             return Optional.fromNullable(captor.getValue());
576         } catch (MockitoException e) {
577             // No value was captured
578             return Optional.absent();
579         }
580     }
581
582     /**
583      * Tests bootstrapping the entity-ownership shard when there's no shards initially configured for local
584      * member. The entity-ownership shard is initially created as inactive (ie remains a follower), requiring
585      * an AddShardReplica request to join it to an existing leader.
586      */
587     @Test
588     public void testEntityOwnershipShardBootstrapping() throws Exception {
589         String name = "testEntityOwnershipShardBootstrapping";
590         String moduleShardsConfig = MODULE_SHARDS_MEMBER_1_CONFIG;
591         MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
592                 .moduleShardsConfig(moduleShardsConfig).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
593                 .datastoreContextBuilder(leaderDatastoreContextBuilder).build();
594
595         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
596         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
597
598         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
599
600         MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
601                 .moduleShardsConfig(moduleShardsConfig).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
602                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
603
604         AbstractDataStore follower1DistributedDataStore = follower1Node.configDataStore();
605         follower1DistributedDataStore.waitTillReady();
606
607         leaderNode.waitForMembersUp("member-2");
608         follower1Node.waitForMembersUp("member-1");
609
610         DOMEntityOwnershipService follower1EntityOwnershipService = newOwnershipService(follower1DistributedDataStore);
611
612         leaderEntityOwnershipService.registerListener(ENTITY_TYPE1, leaderMockListener);
613
614         // Register a candidate for follower1 - should get queued since follower1 has no leader
615         final DOMEntityOwnershipCandidateRegistration candidateReg =
616                 follower1EntityOwnershipService.registerCandidate(ENTITY1);
617         Uninterruptibles.sleepUninterruptibly(300, TimeUnit.MILLISECONDS);
618         verify(leaderMockListener, never()).ownershipChanged(ownershipChange(ENTITY1));
619
620         // Add replica in follower1
621         AddShardReplica addReplica = new AddShardReplica(ENTITY_OWNERSHIP_SHARD_NAME);
622         follower1DistributedDataStore.getActorContext().getShardManager().tell(addReplica,
623                 follower1Node.kit().getRef());
624         Object reply = follower1Node.kit().expectMsgAnyClassOf(JavaTestKit.duration("5 sec"),
625                 Success.class, Failure.class);
626         if (reply instanceof Failure) {
627             throw new AssertionError("AddShardReplica failed", ((Failure)reply).cause());
628         }
629
630         // The queued candidate registration should proceed
631         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
632         reset(leaderMockListener);
633
634         candidateReg.close();
635         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, false));
636         reset(leaderMockListener);
637
638         // Restart follower1 and verify the entity ownership shard is re-instated by registering.
639         Cluster.get(leaderNode.kit().getSystem()).down(Cluster.get(follower1Node.kit().getSystem()).selfAddress());
640         follower1Node.cleanup();
641
642         follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
643                 .moduleShardsConfig(moduleShardsConfig).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
644                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
645         follower1EntityOwnershipService = newOwnershipService(follower1Node.configDataStore());
646
647         follower1EntityOwnershipService.registerCandidate(ENTITY1);
648         verify(leaderMockListener, timeout(20000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
649
650         verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME, raftState -> {
651             assertNull("Custom RaftPolicy class name", raftState.getCustomRaftPolicyClassName());
652             assertEquals("Peer count", 1, raftState.getPeerAddresses().keySet().size());
653             assertThat("Peer Id", Iterables.<String>getLast(raftState.getPeerAddresses().keySet()),
654                     org.hamcrest.CoreMatchers.containsString("member-1"));
655         });
656     }
657
658     @Test
659     public void testOwnerSelectedOnRapidUnregisteringAndRegisteringOfCandidates() throws Exception {
660         String name = "testOwnerSelectedOnRapidUnregisteringAndRegisteringOfCandidates";
661         MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
662                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
663                 .datastoreContextBuilder(leaderDatastoreContextBuilder).build();
664
665         MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
666                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
667                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
668
669         MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name)
670                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
671                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
672
673         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
674
675         leaderDistributedDataStore.waitTillReady();
676         follower1Node.configDataStore().waitTillReady();
677         follower2Node.configDataStore().waitTillReady();
678
679         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
680         final DOMEntityOwnershipService follower1EntityOwnershipService =
681                 newOwnershipService(follower1Node.configDataStore());
682         newOwnershipService(follower2Node.configDataStore());
683
684         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
685
686         // Register leader candidate for entity1 and verify it becomes owner
687
688         DOMEntityOwnershipCandidateRegistration leaderEntity1Reg =
689                 leaderEntityOwnershipService.registerCandidate(ENTITY1);
690
691         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-1");
692         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-1");
693
694         leaderEntity1Reg.close();
695         follower1EntityOwnershipService.registerCandidate(ENTITY1);
696
697         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-2");
698         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
699     }
700
701     @Test
702     public void testOwnerSelectedOnRapidRegisteringAndUnregisteringOfCandidates() throws Exception {
703         String name = "testOwnerSelectedOnRapidRegisteringAndUnregisteringOfCandidates";
704         MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
705                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
706                 .datastoreContextBuilder(leaderDatastoreContextBuilder).build();
707
708         MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
709                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
710                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
711
712         MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name)
713                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
714                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
715
716         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
717
718         leaderDistributedDataStore.waitTillReady();
719         follower1Node.configDataStore().waitTillReady();
720         follower2Node.configDataStore().waitTillReady();
721
722         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
723         final DOMEntityOwnershipService follower1EntityOwnershipService =
724                 newOwnershipService(follower1Node.configDataStore());
725         newOwnershipService(follower2Node.configDataStore());
726
727         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
728
729         // Register leader candidate for entity1 and verify it becomes owner
730
731         final DOMEntityOwnershipCandidateRegistration leaderEntity1Reg =
732                 leaderEntityOwnershipService.registerCandidate(ENTITY1);
733
734         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-1");
735         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-1");
736
737         follower1EntityOwnershipService.registerCandidate(ENTITY1);
738         leaderEntity1Reg.close();
739
740         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-2");
741         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
742     }
743
744     private static void verifyGetOwnershipState(final DOMEntityOwnershipService service, final DOMEntity entity,
745             final EntityOwnershipState expState) {
746         Optional<EntityOwnershipState> state = service.getOwnershipState(entity);
747         assertEquals("getOwnershipState present", true, state.isPresent());
748         assertEquals("EntityOwnershipState", expState, state.get());
749     }
750
751     private static void verifyCandidates(final AbstractDataStore dataStore, final DOMEntity entity,
752             final String... expCandidates) throws Exception {
753         AssertionError lastError = null;
754         Stopwatch sw = Stopwatch.createStarted();
755         while (sw.elapsed(TimeUnit.MILLISECONDS) <= 10000) {
756             Optional<NormalizedNode<?, ?>> possible = dataStore.newReadOnlyTransaction()
757                     .read(entityPath(entity.getType(), entity.getIdentifier()).node(Candidate.QNAME))
758                     .get(5, TimeUnit.SECONDS);
759             try {
760                 assertEquals("Candidates not found for " + entity, true, possible.isPresent());
761                 Collection<String> actual = new ArrayList<>();
762                 for (MapEntryNode candidate: ((MapNode)possible.get()).getValue()) {
763                     actual.add(candidate.getChild(CANDIDATE_NAME_NODE_ID).get().getValue().toString());
764                 }
765
766                 assertEquals("Candidates for " + entity, Arrays.asList(expCandidates), actual);
767                 return;
768             } catch (AssertionError e) {
769                 lastError = e;
770                 Uninterruptibles.sleepUninterruptibly(300, TimeUnit.MILLISECONDS);
771             }
772         }
773
774         throw lastError;
775     }
776
777     @SuppressWarnings("checkstyle:IllegalCatch")
778     private static void verifyOwner(final AbstractDataStore dataStore, final DOMEntity entity,
779             final String expOwner) {
780         AbstractEntityOwnershipTest.verifyOwner(expOwner, entity.getType(), entity.getIdentifier(), path -> {
781             try {
782                 return dataStore.newReadOnlyTransaction().read(path).get(5, TimeUnit.SECONDS).get();
783             } catch (Exception e) {
784                 return null;
785             }
786         });
787     }
788 }

©2013 OpenDaylight, A Linux Foundation Collaborative Project. All Rights Reserved.
OpenDaylight is a registered trademark of The OpenDaylight Project, Inc.
Linux Foundation and OpenDaylight are registered trademarks of the Linux Foundation.
Linux is a registered trademark of Linus Torvalds.