Improve test logging in DistributedEntityOwnershipIntegrationTest
[controller.git] / opendaylight / md-sal / sal-distributed-datastore / src / test / java / org / opendaylight / controller / cluster / datastore / entityownership / DistributedEntityOwnershipIntegrationTest.java
1 /*
2  * Copyright (c) 2015 Brocade Communications Systems, Inc. and others.  All rights reserved.
3  *
4  * This program and the accompanying materials are made available under the
5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6  * and is available at http://www.eclipse.org/legal/epl-v10.html
7  */
8 package org.opendaylight.controller.cluster.datastore.entityownership;
9
10 import static org.junit.Assert.assertEquals;
11 import static org.junit.Assert.assertNull;
12 import static org.junit.Assert.assertThat;
13 import static org.junit.Assert.assertTrue;
14 import static org.mockito.AdditionalMatchers.or;
15 import static org.mockito.Mockito.doNothing;
16 import static org.mockito.Mockito.never;
17 import static org.mockito.Mockito.reset;
18 import static org.mockito.Mockito.timeout;
19 import static org.mockito.Mockito.verify;
20 import static org.opendaylight.controller.cluster.datastore.MemberNode.verifyRaftState;
21 import static org.opendaylight.controller.cluster.datastore.entityownership.AbstractEntityOwnershipTest.ownershipChange;
22 import static org.opendaylight.controller.cluster.datastore.entityownership.DistributedEntityOwnershipService.ENTITY_OWNERSHIP_SHARD_NAME;
23 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.CANDIDATE_NAME_NODE_ID;
24 import static org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnersModel.entityPath;
25
26 import akka.actor.ActorRef;
27 import akka.actor.Status.Failure;
28 import akka.actor.Status.Success;
29 import akka.cluster.Cluster;
30 import akka.testkit.JavaTestKit;
31 import com.google.common.base.Optional;
32 import com.google.common.base.Stopwatch;
33 import com.google.common.collect.Iterables;
34 import com.google.common.collect.Lists;
35 import com.google.common.util.concurrent.Uninterruptibles;
36 import java.util.ArrayList;
37 import java.util.Arrays;
38 import java.util.Collection;
39 import java.util.List;
40 import java.util.concurrent.TimeUnit;
41 import org.junit.After;
42 import org.junit.Before;
43 import org.junit.Test;
44 import org.mockito.ArgumentCaptor;
45 import org.mockito.Mock;
46 import org.mockito.Mockito;
47 import org.mockito.MockitoAnnotations;
48 import org.mockito.exceptions.base.MockitoException;
49 import org.opendaylight.controller.cluster.datastore.AbstractDataStore;
50 import org.opendaylight.controller.cluster.datastore.DatastoreContext;
51 import org.opendaylight.controller.cluster.datastore.IntegrationTestKit;
52 import org.opendaylight.controller.cluster.datastore.MemberNode;
53 import org.opendaylight.controller.cluster.datastore.entityownership.selectionstrategy.EntityOwnerSelectionStrategyConfig;
54 import org.opendaylight.controller.cluster.datastore.messages.AddShardReplica;
55 import org.opendaylight.controller.cluster.raft.RaftState;
56 import org.opendaylight.controller.cluster.raft.policy.DisableElectionsRaftPolicy;
57 import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal;
58 import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore;
59 import org.opendaylight.controller.md.cluster.datastore.model.SchemaContextHelper;
60 import org.opendaylight.mdsal.eos.common.api.EntityOwnershipState;
61 import org.opendaylight.mdsal.eos.dom.api.DOMEntity;
62 import org.opendaylight.mdsal.eos.dom.api.DOMEntityOwnershipCandidateRegistration;
63 import org.opendaylight.mdsal.eos.dom.api.DOMEntityOwnershipChange;
64 import org.opendaylight.mdsal.eos.dom.api.DOMEntityOwnershipListener;
65 import org.opendaylight.mdsal.eos.dom.api.DOMEntityOwnershipService;
66 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.clustering.entity.owners.rev150804.entity.owners.entity.type.entity.Candidate;
67 import org.opendaylight.yangtools.yang.data.api.schema.MapEntryNode;
68 import org.opendaylight.yangtools.yang.data.api.schema.MapNode;
69 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
70 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
71
72 /**
73  * End-to-end integration tests for the entity ownership functionality.
74  *
75  * @author Thomas Pantelis
76  */
77 public class DistributedEntityOwnershipIntegrationTest {
78     private static final String MODULE_SHARDS_CONFIG = "module-shards-default.conf";
79     private static final String MODULE_SHARDS_5_NODE_CONFIG = "module-shards-default-5-node.conf";
80     private static final String MODULE_SHARDS_MEMBER_1_CONFIG = "module-shards-default-member-1.conf";
81     private static final String ENTITY_TYPE1 = "entityType1";
82     private static final String ENTITY_TYPE2 = "entityType2";
83     private static final DOMEntity ENTITY1 = new DOMEntity(ENTITY_TYPE1, "entity1");
84     private static final DOMEntity ENTITY1_2 = new DOMEntity(ENTITY_TYPE2, "entity1");
85     private static final DOMEntity ENTITY2 = new DOMEntity(ENTITY_TYPE1, "entity2");
86     private static final DOMEntity ENTITY3 = new DOMEntity(ENTITY_TYPE1, "entity3");
87     private static final DOMEntity ENTITY4 = new DOMEntity(ENTITY_TYPE1, "entity4");
88     private static final SchemaContext SCHEMA_CONTEXT = SchemaContextHelper.entityOwners();
89
90     private final DatastoreContext.Builder leaderDatastoreContextBuilder =
91             DatastoreContext.newBuilder().shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(5)
92                     .shardIsolatedLeaderCheckIntervalInMillis(1000000);
93
94     private final DatastoreContext.Builder followerDatastoreContextBuilder =
95             DatastoreContext.newBuilder().shardHeartbeatIntervalInMillis(100).shardElectionTimeoutFactor(10000);
96
97     private final List<MemberNode> memberNodes = new ArrayList<>();
98
99     @Mock
100     private DOMEntityOwnershipListener leaderMockListener;
101
102     @Mock
103     private DOMEntityOwnershipListener leaderMockListener2;
104
105     @Mock
106     private DOMEntityOwnershipListener follower1MockListener;
107
108     @Mock
109     private DOMEntityOwnershipListener follower2MockListener;
110
111     @Before
112     public void setUp() {
113         MockitoAnnotations.initMocks(this);
114         InMemoryJournal.clear();
115         InMemorySnapshotStore.clear();
116     }
117
118     @After
119     public void tearDown() {
120         for (MemberNode m : Lists.reverse(memberNodes)) {
121             m.cleanup();
122         }
123         memberNodes.clear();
124     }
125
126     private static DistributedEntityOwnershipService newOwnershipService(final AbstractDataStore datastore) {
127         return DistributedEntityOwnershipService.start(datastore.getActorContext(),
128                 EntityOwnerSelectionStrategyConfig.newBuilder().build());
129     }
130
131     @Test
132     public void testFunctionalityWithThreeNodes() throws Exception {
133         String name = "testFunctionalityWithThreeNodes";
134         MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
135                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
136                 .datastoreContextBuilder(leaderDatastoreContextBuilder).build();
137
138         MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
139                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
140                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
141
142         MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name)
143                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
144                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
145
146         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
147
148         leaderDistributedDataStore.waitTillReady();
149         follower1Node.configDataStore().waitTillReady();
150         follower2Node.configDataStore().waitTillReady();
151
152         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
153         final DOMEntityOwnershipService follower1EntityOwnershipService =
154                 newOwnershipService(follower1Node.configDataStore());
155         final DOMEntityOwnershipService follower2EntityOwnershipService =
156                 newOwnershipService(follower2Node.configDataStore());
157
158         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
159
160         leaderEntityOwnershipService.registerListener(ENTITY_TYPE1, leaderMockListener);
161         leaderEntityOwnershipService.registerListener(ENTITY_TYPE2, leaderMockListener2);
162         follower1EntityOwnershipService.registerListener(ENTITY_TYPE1, follower1MockListener);
163
164         // Register leader candidate for entity1 and verify it becomes owner
165
166         leaderEntityOwnershipService.registerCandidate(ENTITY1);
167         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, true, true));
168         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
169         reset(leaderMockListener, follower1MockListener);
170
171         verifyGetOwnershipState(leaderEntityOwnershipService, ENTITY1, EntityOwnershipState.IS_OWNER);
172         verifyGetOwnershipState(follower1EntityOwnershipService, ENTITY1, EntityOwnershipState.OWNED_BY_OTHER);
173
174         // Register leader candidate for entity1_2 (same id, different type) and verify it becomes owner
175
176         leaderEntityOwnershipService.registerCandidate(ENTITY1_2);
177         verify(leaderMockListener2, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1_2, false, true, true));
178         Uninterruptibles.sleepUninterruptibly(300, TimeUnit.MILLISECONDS);
179         verify(leaderMockListener, never()).ownershipChanged(ownershipChange(ENTITY1_2));
180         reset(leaderMockListener2);
181
182         // Register follower1 candidate for entity1 and verify it gets added but doesn't become owner
183
184         follower1EntityOwnershipService.registerCandidate(ENTITY1);
185         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-1", "member-2");
186         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-1");
187         verifyOwner(follower2Node.configDataStore(), ENTITY1, "member-1");
188         Uninterruptibles.sleepUninterruptibly(300, TimeUnit.MILLISECONDS);
189         verify(leaderMockListener, never()).ownershipChanged(ownershipChange(ENTITY1));
190         verify(follower1MockListener, never()).ownershipChanged(ownershipChange(ENTITY1));
191
192         // Register follower1 candidate for entity2 and verify it becomes owner
193
194         final DOMEntityOwnershipCandidateRegistration follower1Entity2Reg =
195                 follower1EntityOwnershipService.registerCandidate(ENTITY2);
196         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, true, true));
197         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, false, true));
198         verifyOwner(follower2Node.configDataStore(), ENTITY2, "member-2");
199         reset(leaderMockListener, follower1MockListener);
200
201         // Register follower2 candidate for entity2 and verify it gets added but doesn't become owner
202
203         follower2EntityOwnershipService.registerListener(ENTITY_TYPE1, follower2MockListener);
204         verify(follower2MockListener, timeout(5000).times(2)).ownershipChanged(or(
205                 ownershipChange(ENTITY1, false, false, true), ownershipChange(ENTITY2, false, false, true)));
206
207         follower2EntityOwnershipService.registerCandidate(ENTITY2);
208         verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-2", "member-3");
209         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-2");
210
211         // Unregister follower1 candidate for entity2 and verify follower2 becomes owner
212
213         follower1Entity2Reg.close();
214         verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-3");
215         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-3");
216         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, true, false, true));
217         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, false, true));
218
219         // Depending on timing, follower2MockListener could get ownershipChanged with "false, false, true" if
220         // if the original ownership change with "member-2 is replicated to follower2 after the listener is
221         // registered.
222         Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
223         verify(follower2MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, true, true));
224
225         // Register follower1 candidate for entity3 and verify it becomes owner
226
227         follower1EntityOwnershipService.registerCandidate(ENTITY3);
228         verifyOwner(leaderDistributedDataStore, ENTITY3, "member-2");
229         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY3, false, true, true));
230         verify(follower2MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY3, false, false, true));
231         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY3, false, false, true));
232
233         // Register follower2 candidate for entity4 and verify it becomes owner
234
235         follower2EntityOwnershipService.registerCandidate(ENTITY4);
236         verifyOwner(leaderDistributedDataStore, ENTITY4, "member-3");
237         verify(follower2MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY4, false, true, true));
238         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY4, false, false, true));
239         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY4, false, false, true));
240         reset(follower1MockListener, follower2MockListener);
241
242         // Register follower1 candidate for entity4 and verify it gets added but doesn't become owner
243
244         follower1EntityOwnershipService.registerCandidate(ENTITY4);
245         verifyCandidates(leaderDistributedDataStore, ENTITY4, "member-3", "member-2");
246         verifyOwner(leaderDistributedDataStore, ENTITY4, "member-3");
247
248         // Shutdown follower2 and verify it's owned entities (entity 4) get re-assigned
249
250         reset(leaderMockListener, follower1MockListener);
251         follower2Node.cleanup();
252
253         verify(follower1MockListener, timeout(15000)).ownershipChanged(ownershipChange(ENTITY4, false, true, true));
254         verify(leaderMockListener, timeout(15000)).ownershipChanged(ownershipChange(ENTITY4, false, false, true));
255
256         // Register leader candidate for entity2 and verify it becomes owner
257
258         DOMEntityOwnershipCandidateRegistration leaderEntity2Reg =
259                 leaderEntityOwnershipService.registerCandidate(ENTITY2);
260         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
261         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, true, true));
262
263         // Unregister leader candidate for entity2 and verify the owner is cleared
264
265         leaderEntity2Reg.close();
266         verifyOwner(leaderDistributedDataStore, ENTITY2, "");
267         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, true, false, false));
268         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY2, false, false, false));
269     }
270
271     @Test
272     public void testLeaderEntityOwnersReassignedAfterShutdown() throws Exception {
273         followerDatastoreContextBuilder.shardElectionTimeoutFactor(5)
274                     .customRaftPolicyImplementation(DisableElectionsRaftPolicy.class.getName());
275
276         String name = "testLeaderEntityOwnersReassignedAfterShutdown";
277         MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
278                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
279                 .datastoreContextBuilder(leaderDatastoreContextBuilder).build();
280
281         MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
282                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
283                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
284
285         MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name)
286                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
287                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
288
289         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
290
291         leaderDistributedDataStore.waitTillReady();
292         follower1Node.configDataStore().waitTillReady();
293         follower2Node.configDataStore().waitTillReady();
294
295         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
296         final DOMEntityOwnershipService follower1EntityOwnershipService =
297                 newOwnershipService(follower1Node.configDataStore());
298         final DOMEntityOwnershipService follower2EntityOwnershipService =
299                 newOwnershipService(follower2Node.configDataStore());
300
301         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
302
303         // Register follower1 candidate for entity1 and verify it becomes owner
304
305         follower1EntityOwnershipService.registerCandidate(ENTITY1);
306         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
307
308         // Register leader candidate for entity1
309
310         leaderEntityOwnershipService.registerCandidate(ENTITY1);
311         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-2", "member-1");
312         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
313
314         // Register leader candidate for entity2 and verify it becomes owner
315
316         leaderEntityOwnershipService.registerCandidate(ENTITY2);
317         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
318
319         // Register follower2 candidate for entity2
320
321         follower2EntityOwnershipService.registerCandidate(ENTITY2);
322         verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-1", "member-3");
323         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
324
325         // Shutdown the leader and verify its removed from the candidate list
326
327         leaderNode.cleanup();
328         follower1Node.waitForMemberDown("member-1");
329
330         // Re-enable elections on follower1 so it becomes the leader
331
332         ActorRef follower1Shard = IntegrationTestKit.findLocalShard(follower1Node.configDataStore().getActorContext(),
333                 ENTITY_OWNERSHIP_SHARD_NAME);
334         follower1Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build())
335                 .customRaftPolicyImplementation(null).build(), ActorRef.noSender());
336
337         MemberNode.verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
338             raftState -> assertEquals("Raft state", RaftState.Leader.toString(), raftState.getRaftState()));
339
340         // Verify the prior leader's entity owners are re-assigned.
341
342         verifyCandidates(follower1Node.configDataStore(), ENTITY1, "member-2", "member-1");
343         verifyCandidates(follower1Node.configDataStore(), ENTITY2, "member-1", "member-3");
344         verifyOwner(follower1Node.configDataStore(), ENTITY1, "member-2");
345         verifyOwner(follower1Node.configDataStore(), ENTITY2, "member-3");
346     }
347
348     @Test
349     public void testLeaderAndFollowerEntityOwnersReassignedAfterShutdown() throws Exception {
350         followerDatastoreContextBuilder.shardElectionTimeoutFactor(5)
351                 .customRaftPolicyImplementation(DisableElectionsRaftPolicy.class.getName());
352
353         String name = "testLeaderAndFollowerEntityOwnersReassignedAfterShutdown";
354         final MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1")
355                 .useAkkaArtery(false).testName(name)
356                 .moduleShardsConfig(MODULE_SHARDS_5_NODE_CONFIG).schemaContext(SCHEMA_CONTEXT)
357                 .createOperDatastore(false).datastoreContextBuilder(leaderDatastoreContextBuilder).build();
358
359         final MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2")
360                 .useAkkaArtery(false).testName(name)
361                 .moduleShardsConfig(MODULE_SHARDS_5_NODE_CONFIG).schemaContext(SCHEMA_CONTEXT)
362                 .createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
363
364         final MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3")
365                 .useAkkaArtery(false).testName(name)
366                 .moduleShardsConfig(MODULE_SHARDS_5_NODE_CONFIG).schemaContext(SCHEMA_CONTEXT)
367                 .createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
368
369         final MemberNode follower3Node = MemberNode.builder(memberNodes).akkaConfig("Member4")
370                 .useAkkaArtery(false).testName(name)
371                 .moduleShardsConfig(MODULE_SHARDS_5_NODE_CONFIG).schemaContext(SCHEMA_CONTEXT)
372                 .createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
373
374         final MemberNode follower4Node = MemberNode.builder(memberNodes).akkaConfig("Member5")
375                 .useAkkaArtery(false).testName(name)
376                 .moduleShardsConfig(MODULE_SHARDS_5_NODE_CONFIG).schemaContext(SCHEMA_CONTEXT)
377                 .createOperDatastore(false).datastoreContextBuilder(followerDatastoreContextBuilder).build();
378
379         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
380
381         leaderDistributedDataStore.waitTillReady();
382         follower1Node.configDataStore().waitTillReady();
383         follower2Node.configDataStore().waitTillReady();
384
385         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
386         final DOMEntityOwnershipService follower1EntityOwnershipService =
387                 newOwnershipService(follower1Node.configDataStore());
388         final DOMEntityOwnershipService follower2EntityOwnershipService =
389                 newOwnershipService(follower2Node.configDataStore());
390         final DOMEntityOwnershipService follower3EntityOwnershipService =
391                 newOwnershipService(follower3Node.configDataStore());
392         newOwnershipService(follower4Node.configDataStore());
393
394         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
395
396         // Register follower1 candidate for entity1 and verify it becomes owner
397
398         follower1EntityOwnershipService.registerCandidate(ENTITY1);
399         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
400
401         // Register leader candidate for entity1
402
403         leaderEntityOwnershipService.registerCandidate(ENTITY1);
404         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-2", "member-1");
405         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
406
407         // Register leader candidate for entity2 and verify it becomes owner
408
409         leaderEntityOwnershipService.registerCandidate(ENTITY2);
410         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
411
412         // Register follower2 candidate for entity2
413
414         follower2EntityOwnershipService.registerCandidate(ENTITY2);
415         verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-1", "member-3");
416         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
417
418         // Register follower3 as a candidate for entity2 as well
419
420         follower3EntityOwnershipService.registerCandidate(ENTITY2);
421         verifyCandidates(leaderDistributedDataStore, ENTITY2, "member-1", "member-3", "member-4");
422         verifyOwner(leaderDistributedDataStore, ENTITY2, "member-1");
423
424         // Shutdown the leader and follower3
425
426         leaderNode.cleanup();
427         follower3Node.cleanup();
428
429         follower1Node.waitForMemberDown("member-1");
430         follower1Node.waitForMemberDown("member-4");
431
432         // Re-enable elections on follower1 so it becomes the leader
433
434         ActorRef follower1Shard = IntegrationTestKit.findLocalShard(follower1Node.configDataStore().getActorContext(),
435                 ENTITY_OWNERSHIP_SHARD_NAME);
436         follower1Shard.tell(DatastoreContext.newBuilderFrom(followerDatastoreContextBuilder.build())
437                 .customRaftPolicyImplementation(null).build(), ActorRef.noSender());
438
439         MemberNode.verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME,
440             raftState -> assertEquals("Raft state", RaftState.Leader.toString(), raftState.getRaftState()));
441
442         // Verify the prior leader's and follower3 entity owners are re-assigned.
443
444         verifyCandidates(follower1Node.configDataStore(), ENTITY1, "member-2", "member-1");
445         verifyCandidates(follower1Node.configDataStore(), ENTITY2, "member-1", "member-3", "member-4");
446         verifyOwner(follower1Node.configDataStore(), ENTITY1, "member-2");
447         verifyOwner(follower1Node.configDataStore(), ENTITY2, "member-3");
448     }
449
450     /**
451      * Reproduces bug <a href="https://bugs.opendaylight.org/show_bug.cgi?id=4554">4554</a>.
452      */
453     @Test
454     public void testCloseCandidateRegistrationInQuickSuccession() throws Exception {
455         String name = "testCloseCandidateRegistrationInQuickSuccession";
456         MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
457                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
458                 .datastoreContextBuilder(leaderDatastoreContextBuilder).build();
459
460         MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
461                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
462                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
463
464         MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name)
465                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
466                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
467
468         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
469
470         leaderDistributedDataStore.waitTillReady();
471         follower1Node.configDataStore().waitTillReady();
472         follower2Node.configDataStore().waitTillReady();
473
474         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
475         final DOMEntityOwnershipService follower1EntityOwnershipService =
476                 newOwnershipService(follower1Node.configDataStore());
477         final DOMEntityOwnershipService follower2EntityOwnershipService =
478                 newOwnershipService(follower2Node.configDataStore());
479
480         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
481
482         leaderEntityOwnershipService.registerListener(ENTITY_TYPE1, leaderMockListener);
483         follower1EntityOwnershipService.registerListener(ENTITY_TYPE1, follower1MockListener);
484         follower2EntityOwnershipService.registerListener(ENTITY_TYPE1, follower2MockListener);
485
486         final DOMEntityOwnershipCandidateRegistration candidate1 =
487                 leaderEntityOwnershipService.registerCandidate(ENTITY1);
488         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, true, true));
489
490         final DOMEntityOwnershipCandidateRegistration candidate2 =
491                 follower1EntityOwnershipService.registerCandidate(ENTITY1);
492         verify(follower1MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
493
494         final DOMEntityOwnershipCandidateRegistration candidate3 =
495                 follower2EntityOwnershipService.registerCandidate(ENTITY1);
496         verify(follower2MockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
497
498         Mockito.reset(leaderMockListener, follower1MockListener, follower2MockListener);
499
500         ArgumentCaptor<DOMEntityOwnershipChange> leaderChangeCaptor =
501                 ArgumentCaptor.forClass(DOMEntityOwnershipChange.class);
502         ArgumentCaptor<DOMEntityOwnershipChange> follower1ChangeCaptor =
503                 ArgumentCaptor.forClass(DOMEntityOwnershipChange.class);
504         ArgumentCaptor<DOMEntityOwnershipChange> follower2ChangeCaptor =
505                 ArgumentCaptor.forClass(DOMEntityOwnershipChange.class);
506         doNothing().when(leaderMockListener).ownershipChanged(leaderChangeCaptor.capture());
507         doNothing().when(follower1MockListener).ownershipChanged(follower1ChangeCaptor.capture());
508         doNothing().when(follower2MockListener).ownershipChanged(follower2ChangeCaptor.capture());
509
510         candidate1.close();
511         candidate2.close();
512         candidate3.close();
513
514         boolean passed = false;
515         for (int i = 0; i < 100; i++) {
516             Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
517             final Optional<EntityOwnershipState> leaderState = leaderEntityOwnershipService.getOwnershipState(ENTITY1);
518             final Optional<EntityOwnershipState> follower1State =
519                     follower1EntityOwnershipService.getOwnershipState(ENTITY1);
520             final Optional<EntityOwnershipState> follower2State =
521                     follower2EntityOwnershipService.getOwnershipState(ENTITY1);
522             final Optional<DOMEntityOwnershipChange> leaderChange = getValueSafely(leaderChangeCaptor);
523             final Optional<DOMEntityOwnershipChange> follower1Change = getValueSafely(follower1ChangeCaptor);
524             final Optional<DOMEntityOwnershipChange> follower2Change = getValueSafely(follower2ChangeCaptor);
525             if (!leaderState.isPresent() || leaderState.get() == EntityOwnershipState.NO_OWNER
526                     && follower1State.isPresent() && follower1State.get() == EntityOwnershipState.NO_OWNER
527                     && follower2State.isPresent() && follower2State.get() == EntityOwnershipState.NO_OWNER
528                     && leaderChange.isPresent() && !leaderChange.get().getState().hasOwner()
529                     && follower1Change.isPresent() && !follower1Change.get().getState().hasOwner()
530                     && follower2Change.isPresent() && !follower2Change.get().getState().hasOwner()) {
531                 passed = true;
532                 break;
533             }
534         }
535
536         assertTrue("No ownership change message was sent with hasOwner=false", passed);
537     }
538
539     private static Optional<DOMEntityOwnershipChange> getValueSafely(ArgumentCaptor<DOMEntityOwnershipChange> captor) {
540         try {
541             return Optional.fromNullable(captor.getValue());
542         } catch (MockitoException e) {
543             // No value was captured
544             return Optional.absent();
545         }
546     }
547
548     /**
549      * Tests bootstrapping the entity-ownership shard when there's no shards initially configured for local
550      * member. The entity-ownership shard is initially created as inactive (ie remains a follower), requiring
551      * an AddShardReplica request to join it to an existing leader.
552      */
553     @Test
554     public void testEntityOwnershipShardBootstrapping() throws Exception {
555         String name = "testEntityOwnershipShardBootstrapping";
556         String moduleShardsConfig = MODULE_SHARDS_MEMBER_1_CONFIG;
557         MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
558                 .moduleShardsConfig(moduleShardsConfig).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
559                 .datastoreContextBuilder(leaderDatastoreContextBuilder).build();
560
561         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
562         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
563
564         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
565
566         MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
567                 .moduleShardsConfig(moduleShardsConfig).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
568                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
569
570         AbstractDataStore follower1DistributedDataStore = follower1Node.configDataStore();
571         follower1DistributedDataStore.waitTillReady();
572
573         leaderNode.waitForMembersUp("member-2");
574         follower1Node.waitForMembersUp("member-1");
575
576         DOMEntityOwnershipService follower1EntityOwnershipService = newOwnershipService(follower1DistributedDataStore);
577
578         leaderEntityOwnershipService.registerListener(ENTITY_TYPE1, leaderMockListener);
579
580         // Register a candidate for follower1 - should get queued since follower1 has no leader
581         final DOMEntityOwnershipCandidateRegistration candidateReg =
582                 follower1EntityOwnershipService.registerCandidate(ENTITY1);
583         Uninterruptibles.sleepUninterruptibly(300, TimeUnit.MILLISECONDS);
584         verify(leaderMockListener, never()).ownershipChanged(ownershipChange(ENTITY1));
585
586         // Add replica in follower1
587         AddShardReplica addReplica = new AddShardReplica(ENTITY_OWNERSHIP_SHARD_NAME);
588         follower1DistributedDataStore.getActorContext().getShardManager().tell(addReplica,
589                 follower1Node.kit().getRef());
590         Object reply = follower1Node.kit().expectMsgAnyClassOf(JavaTestKit.duration("5 sec"),
591                 Success.class, Failure.class);
592         if (reply instanceof Failure) {
593             throw new AssertionError("AddShardReplica failed", ((Failure)reply).cause());
594         }
595
596         // The queued candidate registration should proceed
597         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
598         reset(leaderMockListener);
599
600         candidateReg.close();
601         verify(leaderMockListener, timeout(5000)).ownershipChanged(ownershipChange(ENTITY1, false, false, false));
602         reset(leaderMockListener);
603
604         // Restart follower1 and verify the entity ownership shard is re-instated by registering.
605         Cluster.get(leaderNode.kit().getSystem()).down(Cluster.get(follower1Node.kit().getSystem()).selfAddress());
606         follower1Node.cleanup();
607
608         follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
609                 .moduleShardsConfig(moduleShardsConfig).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
610                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
611         follower1EntityOwnershipService = newOwnershipService(follower1Node.configDataStore());
612
613         follower1EntityOwnershipService.registerCandidate(ENTITY1);
614         verify(leaderMockListener, timeout(20000)).ownershipChanged(ownershipChange(ENTITY1, false, false, true));
615
616         verifyRaftState(follower1Node.configDataStore(), ENTITY_OWNERSHIP_SHARD_NAME, raftState -> {
617             assertNull("Custom RaftPolicy class name", raftState.getCustomRaftPolicyClassName());
618             assertEquals("Peer count", 1, raftState.getPeerAddresses().keySet().size());
619             assertThat("Peer Id", Iterables.<String>getLast(raftState.getPeerAddresses().keySet()),
620                     org.hamcrest.CoreMatchers.containsString("member-1"));
621         });
622     }
623
624     @Test
625     public void testOwnerSelectedOnRapidUnregisteringAndRegisteringOfCandidates() throws Exception {
626         String name = "testOwnerSelectedOnRapidUnregisteringAndRegisteringOfCandidates";
627         MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
628                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
629                 .datastoreContextBuilder(leaderDatastoreContextBuilder).build();
630
631         MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
632                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
633                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
634
635         MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name)
636                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
637                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
638
639         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
640
641         leaderDistributedDataStore.waitTillReady();
642         follower1Node.configDataStore().waitTillReady();
643         follower2Node.configDataStore().waitTillReady();
644
645         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
646         final DOMEntityOwnershipService follower1EntityOwnershipService =
647                 newOwnershipService(follower1Node.configDataStore());
648         newOwnershipService(follower2Node.configDataStore());
649
650         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
651
652         // Register leader candidate for entity1 and verify it becomes owner
653
654         DOMEntityOwnershipCandidateRegistration leaderEntity1Reg =
655                 leaderEntityOwnershipService.registerCandidate(ENTITY1);
656
657         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-1");
658         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-1");
659
660         leaderEntity1Reg.close();
661         follower1EntityOwnershipService.registerCandidate(ENTITY1);
662
663         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-2");
664         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
665     }
666
667     @Test
668     public void testOwnerSelectedOnRapidRegisteringAndUnregisteringOfCandidates() throws Exception {
669         String name = "testOwnerSelectedOnRapidRegisteringAndUnregisteringOfCandidates";
670         MemberNode leaderNode = MemberNode.builder(memberNodes).akkaConfig("Member1").testName(name)
671                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
672                 .datastoreContextBuilder(leaderDatastoreContextBuilder).build();
673
674         MemberNode follower1Node = MemberNode.builder(memberNodes).akkaConfig("Member2").testName(name)
675                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
676                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
677
678         MemberNode follower2Node = MemberNode.builder(memberNodes).akkaConfig("Member3").testName(name)
679                 .moduleShardsConfig(MODULE_SHARDS_CONFIG).schemaContext(SCHEMA_CONTEXT).createOperDatastore(false)
680                 .datastoreContextBuilder(followerDatastoreContextBuilder).build();
681
682         AbstractDataStore leaderDistributedDataStore = leaderNode.configDataStore();
683
684         leaderDistributedDataStore.waitTillReady();
685         follower1Node.configDataStore().waitTillReady();
686         follower2Node.configDataStore().waitTillReady();
687
688         final DOMEntityOwnershipService leaderEntityOwnershipService = newOwnershipService(leaderDistributedDataStore);
689         final DOMEntityOwnershipService follower1EntityOwnershipService =
690                 newOwnershipService(follower1Node.configDataStore());
691         newOwnershipService(follower2Node.configDataStore());
692
693         leaderNode.kit().waitUntilLeader(leaderNode.configDataStore().getActorContext(), ENTITY_OWNERSHIP_SHARD_NAME);
694
695         // Register leader candidate for entity1 and verify it becomes owner
696
697         final DOMEntityOwnershipCandidateRegistration leaderEntity1Reg =
698                 leaderEntityOwnershipService.registerCandidate(ENTITY1);
699
700         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-1");
701         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-1");
702
703         follower1EntityOwnershipService.registerCandidate(ENTITY1);
704         leaderEntity1Reg.close();
705
706         verifyCandidates(leaderDistributedDataStore, ENTITY1, "member-2");
707         verifyOwner(leaderDistributedDataStore, ENTITY1, "member-2");
708     }
709
710     private static void verifyGetOwnershipState(final DOMEntityOwnershipService service, final DOMEntity entity,
711             final EntityOwnershipState expState) {
712         Optional<EntityOwnershipState> state = service.getOwnershipState(entity);
713         assertEquals("getOwnershipState present", true, state.isPresent());
714         assertEquals("EntityOwnershipState", expState, state.get());
715     }
716
717     private static void verifyCandidates(final AbstractDataStore dataStore, final DOMEntity entity,
718             final String... expCandidates) throws Exception {
719         AssertionError lastError = null;
720         Stopwatch sw = Stopwatch.createStarted();
721         while (sw.elapsed(TimeUnit.MILLISECONDS) <= 10000) {
722             Optional<NormalizedNode<?, ?>> possible = dataStore.newReadOnlyTransaction()
723                     .read(entityPath(entity.getType(), entity.getIdentifier()).node(Candidate.QNAME))
724                     .get(5, TimeUnit.SECONDS);
725             try {
726                 assertEquals("Candidates not found for " + entity, true, possible.isPresent());
727                 Collection<String> actual = new ArrayList<>();
728                 for (MapEntryNode candidate: ((MapNode)possible.get()).getValue()) {
729                     actual.add(candidate.getChild(CANDIDATE_NAME_NODE_ID).get().getValue().toString());
730                 }
731
732                 assertEquals("Candidates for " + entity, Arrays.asList(expCandidates), actual);
733                 return;
734             } catch (AssertionError e) {
735                 lastError = e;
736                 Uninterruptibles.sleepUninterruptibly(300, TimeUnit.MILLISECONDS);
737             }
738         }
739
740         throw lastError;
741     }
742
743     @SuppressWarnings("checkstyle:IllegalCatch")
744     private static void verifyOwner(final AbstractDataStore dataStore, final DOMEntity entity,
745             final String expOwner) {
746         AbstractEntityOwnershipTest.verifyOwner(expOwner, entity.getType(), entity.getIdentifier(), path -> {
747             try {
748                 return dataStore.newReadOnlyTransaction().read(path).get(5, TimeUnit.SECONDS).get();
749             } catch (Exception e) {
750                 return null;
751             }
752         });
753     }
754 }

©2013 OpenDaylight, A Linux Foundation Collaborative Project. All Rights Reserved.
OpenDaylight is a registered trademark of The OpenDaylight Project, Inc.
Linux Foundation and OpenDaylight are registered trademarks of the Linux Foundation.
Linux is a registered trademark of Linus Torvalds.