2 * Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
9 package org.opendaylight.controller.cluster.sharding;
11 import static org.junit.Assert.assertEquals;
12 import static org.junit.Assert.assertNotNull;
13 import static org.mockito.Matchers.anyCollection;
14 import static org.mockito.Matchers.anyMap;
15 import static org.mockito.Mockito.doNothing;
16 import static org.mockito.Mockito.doReturn;
17 import static org.mockito.Mockito.mock;
18 import static org.mockito.Mockito.timeout;
19 import static org.mockito.Mockito.verify;
20 import static org.mockito.Mockito.verifyNoMoreInteractions;
21 import static org.opendaylight.controller.cluster.datastore.IntegrationTestKit.findLocalShard;
22 import static org.opendaylight.controller.cluster.datastore.IntegrationTestKit.waitUntilShardIsDown;
24 import akka.actor.ActorRef;
25 import akka.actor.ActorSystem;
26 import akka.actor.Address;
27 import akka.actor.AddressFromURIString;
28 import akka.cluster.Cluster;
29 import akka.testkit.JavaTestKit;
30 import com.google.common.base.Optional;
31 import com.google.common.collect.Lists;
32 import com.google.common.util.concurrent.CheckedFuture;
33 import com.typesafe.config.ConfigFactory;
34 import java.util.ArrayList;
35 import java.util.Collection;
36 import java.util.Collections;
37 import java.util.List;
40 import java.util.concurrent.CompletionStage;
41 import org.junit.After;
42 import org.junit.Assert;
43 import org.junit.Before;
44 import org.junit.Ignore;
45 import org.junit.Test;
46 import org.mockito.ArgumentCaptor;
47 import org.mockito.Captor;
48 import org.mockito.Mockito;
49 import org.mockito.MockitoAnnotations;
50 import org.opendaylight.controller.cluster.ActorSystemProvider;
51 import org.opendaylight.controller.cluster.access.concepts.MemberName;
52 import org.opendaylight.controller.cluster.datastore.AbstractTest;
53 import org.opendaylight.controller.cluster.datastore.DatastoreContext;
54 import org.opendaylight.controller.cluster.datastore.DatastoreContext.Builder;
55 import org.opendaylight.controller.cluster.datastore.DistributedDataStore;
56 import org.opendaylight.controller.cluster.datastore.IntegrationTestKit;
57 import org.opendaylight.controller.cluster.datastore.utils.ClusterUtils;
58 import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal;
59 import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore;
60 import org.opendaylight.controller.cluster.sharding.DistributedShardFactory.DistributedShardRegistration;
61 import org.opendaylight.controller.md.cluster.datastore.model.SchemaContextHelper;
62 import org.opendaylight.controller.md.cluster.datastore.model.TestModel;
63 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
64 import org.opendaylight.mdsal.common.api.TransactionCommitFailedException;
65 import org.opendaylight.mdsal.dom.api.DOMDataTreeCursorAwareTransaction;
66 import org.opendaylight.mdsal.dom.api.DOMDataTreeIdentifier;
67 import org.opendaylight.mdsal.dom.api.DOMDataTreeListener;
68 import org.opendaylight.mdsal.dom.api.DOMDataTreeProducer;
69 import org.opendaylight.mdsal.dom.api.DOMDataTreeWriteCursor;
70 import org.opendaylight.yangtools.yang.common.QName;
71 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
72 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifier;
73 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifierWithPredicates;
74 import org.opendaylight.yangtools.yang.data.api.schema.ContainerNode;
75 import org.opendaylight.yangtools.yang.data.api.schema.LeafNode;
76 import org.opendaylight.yangtools.yang.data.api.schema.MapEntryNode;
77 import org.opendaylight.yangtools.yang.data.api.schema.MapNode;
78 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
79 import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidate;
80 import org.opendaylight.yangtools.yang.data.impl.schema.ImmutableNodes;
81 import org.opendaylight.yangtools.yang.data.impl.schema.builder.impl.ImmutableContainerNodeBuilder;
82 import org.opendaylight.yangtools.yang.data.impl.schema.builder.impl.ImmutableLeafNodeBuilder;
83 import org.opendaylight.yangtools.yang.data.impl.schema.builder.impl.ImmutableMapNodeBuilder;
84 import org.slf4j.Logger;
85 import org.slf4j.LoggerFactory;
87 public class DistributedShardedDOMDataTreeTest extends AbstractTest {
89 private static final Logger LOG = LoggerFactory.getLogger(DistributedShardedDOMDataTreeRemotingTest.class);
91 private static final Address MEMBER_1_ADDRESS =
92 AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558");
94 private static final DOMDataTreeIdentifier TEST_ID =
95 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, TestModel.TEST_PATH);
97 private static final DOMDataTreeIdentifier INNER_LIST_ID =
98 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION,
99 YangInstanceIdentifier.create(getOuterListIdFor(0).getPathArguments())
100 .node(TestModel.INNER_LIST_QNAME));
101 private static final Set<MemberName> SINGLE_MEMBER = Collections.singleton(AbstractTest.MEMBER_NAME);
103 private static final String MODULE_SHARDS_CONFIG = "module-shards-cars-member-1.conf";
105 private ActorSystem leaderSystem;
107 private final Builder leaderDatastoreContextBuilder =
108 DatastoreContext.newBuilder()
109 .shardHeartbeatIntervalInMillis(100)
110 .shardElectionTimeoutFactor(2)
112 org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType.CONFIGURATION);
114 private DistributedDataStore leaderDistributedDataStore;
115 private DistributedDataStore operDistributedDatastore;
116 private IntegrationTestKit leaderTestKit;
118 private DistributedShardedDOMDataTree leaderShardFactory;
121 private ArgumentCaptor<Collection<DataTreeCandidate>> captorForChanges;
123 private ArgumentCaptor<Map<DOMDataTreeIdentifier, NormalizedNode<?, ?>>> captorForSubtrees;
125 private ActorSystemProvider leaderSystemProvider;
128 public void setUp() {
129 MockitoAnnotations.initMocks(this);
131 InMemoryJournal.clear();
132 InMemorySnapshotStore.clear();
134 leaderSystem = ActorSystem.create("cluster-test", ConfigFactory.load().getConfig("Member1"));
135 Cluster.get(leaderSystem).join(MEMBER_1_ADDRESS);
137 leaderSystemProvider = Mockito.mock(ActorSystemProvider.class);
138 doReturn(leaderSystem).when(leaderSystemProvider).getActorSystem();
142 public void tearDown() throws Exception {
143 if (leaderDistributedDataStore != null) {
144 leaderDistributedDataStore.close();
147 if (operDistributedDatastore != null) {
148 operDistributedDatastore.close();
151 JavaTestKit.shutdownActorSystem(leaderSystem);
153 InMemoryJournal.clear();
154 InMemorySnapshotStore.clear();
157 private void initEmptyDatastores() {
158 leaderTestKit = new IntegrationTestKit(leaderSystem, leaderDatastoreContextBuilder);
160 leaderDistributedDataStore = (DistributedDataStore) leaderTestKit.setupDistributedDataStore(
161 "config", MODULE_SHARDS_CONFIG, "empty-modules.conf", true,
162 SchemaContextHelper.distributedShardedDOMDataTreeSchemaContext());
164 operDistributedDatastore = (DistributedDataStore) leaderTestKit.setupDistributedDataStore(
165 "operational", MODULE_SHARDS_CONFIG, "empty-modules.conf",true,
166 SchemaContextHelper.distributedShardedDOMDataTreeSchemaContext());
168 leaderShardFactory = new DistributedShardedDOMDataTree(leaderSystemProvider,
169 operDistributedDatastore,
170 leaderDistributedDataStore);
172 leaderShardFactory.init();
177 public void testWritesIntoDefaultShard() throws Exception {
178 initEmptyDatastores();
180 final DOMDataTreeIdentifier configRoot =
181 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, YangInstanceIdentifier.EMPTY);
183 final DOMDataTreeProducer producer = leaderShardFactory.createProducer(Collections.singleton(configRoot));
185 final DOMDataTreeCursorAwareTransaction tx = producer.createTransaction(true);
186 final DOMDataTreeWriteCursor cursor =
187 tx.createCursor(new DOMDataTreeIdentifier(
188 LogicalDatastoreType.CONFIGURATION, YangInstanceIdentifier.EMPTY));
189 Assert.assertNotNull(cursor);
191 final ContainerNode test =
192 ImmutableContainerNodeBuilder.create()
193 .withNodeIdentifier(new NodeIdentifier(TestModel.TEST_QNAME)).build();
195 cursor.write(test.getIdentifier(), test);
198 tx.submit().checkedGet();
202 public void testSingleNodeWrites() throws Exception {
203 initEmptyDatastores();
205 final DistributedShardRegistration shardRegistration = waitOnAsyncTask(
206 leaderShardFactory.createDistributedShard(TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
207 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
209 leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
210 ClusterUtils.getCleanShardName(TEST_ID.getRootIdentifier()));
212 final DOMDataTreeProducer producer = leaderShardFactory.createProducer(Collections.singleton(TEST_ID));
214 final DOMDataTreeCursorAwareTransaction tx = producer.createTransaction(true);
215 final DOMDataTreeWriteCursor cursor = tx.createCursor(TEST_ID);
216 Assert.assertNotNull(cursor);
217 final YangInstanceIdentifier nameId =
218 YangInstanceIdentifier.builder(TestModel.TEST_PATH).node(TestModel.NAME_QNAME).build();
219 final LeafNode<String> valueToCheck = ImmutableLeafNodeBuilder.<String>create().withNodeIdentifier(
220 new NodeIdentifier(TestModel.NAME_QNAME)).withValue("Test Value").build();
221 LOG.debug("Writing data {} at {}, cursor {}", nameId.getLastPathArgument(), valueToCheck, cursor);
222 cursor.write(nameId.getLastPathArgument(),
226 LOG.debug("Got to pre submit");
228 tx.submit().checkedGet();
230 final DOMDataTreeListener mockedDataTreeListener = mock(DOMDataTreeListener.class);
231 doNothing().when(mockedDataTreeListener).onDataTreeChanged(anyCollection(), anyMap());
233 leaderShardFactory.registerListener(mockedDataTreeListener, Collections.singletonList(TEST_ID),
234 true, Collections.emptyList());
236 verify(mockedDataTreeListener, timeout(1000).times(1)).onDataTreeChanged(captorForChanges.capture(),
237 captorForSubtrees.capture());
238 final List<Collection<DataTreeCandidate>> capturedValue = captorForChanges.getAllValues();
240 final Optional<NormalizedNode<?, ?>> dataAfter =
241 capturedValue.get(0).iterator().next().getRootNode().getDataAfter();
243 final NormalizedNode<?,?> expected = ImmutableContainerNodeBuilder.create()
244 .withNodeIdentifier(new NodeIdentifier(TestModel.TEST_QNAME)).withChild(valueToCheck).build();
245 assertEquals(expected, dataAfter.get());
247 verifyNoMoreInteractions(mockedDataTreeListener);
249 shardRegistration.close().toCompletableFuture().get();
254 public void testMultipleWritesIntoSingleMapEntry() throws Exception {
255 initEmptyDatastores();
257 final DistributedShardRegistration shardRegistration = waitOnAsyncTask(
258 leaderShardFactory.createDistributedShard(TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
259 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
261 leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
262 ClusterUtils.getCleanShardName(TEST_ID.getRootIdentifier()));
264 LOG.warn("Got after waiting for nonleader");
265 final ActorRef leaderShardManager = leaderDistributedDataStore.getActorContext().getShardManager();
267 leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
268 ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
270 final YangInstanceIdentifier oid1 = getOuterListIdFor(0);
271 final DOMDataTreeIdentifier outerListPath = new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, oid1);
273 final DistributedShardRegistration outerListShardReg = waitOnAsyncTask(
274 leaderShardFactory.createDistributedShard(outerListPath, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
275 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
277 leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
278 ClusterUtils.getCleanShardName(outerListPath.getRootIdentifier()));
280 final DOMDataTreeProducer shardProducer = leaderShardFactory.createProducer(
281 Collections.singletonList(outerListPath));
283 final DOMDataTreeCursorAwareTransaction tx = shardProducer.createTransaction(false);
284 final DOMDataTreeWriteCursor cursor =
285 tx.createCursor(new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, oid1));
286 assertNotNull(cursor);
288 MapNode innerList = ImmutableMapNodeBuilder
290 .withNodeIdentifier(new NodeIdentifier(TestModel.INNER_LIST_QNAME))
293 cursor.write(new NodeIdentifier(TestModel.INNER_LIST_QNAME), innerList);
295 tx.submit().checkedGet();
297 final ArrayList<CheckedFuture<Void, TransactionCommitFailedException>> futures = new ArrayList<>();
298 for (int i = 0; i < 1000; i++) {
299 final Collection<MapEntryNode> innerListMapEntries = createInnerListMapEntries(1000, "run-" + i);
300 for (final MapEntryNode innerListMapEntry : innerListMapEntries) {
301 final DOMDataTreeCursorAwareTransaction tx1 = shardProducer.createTransaction(false);
302 final DOMDataTreeWriteCursor cursor1 = tx1.createCursor(
303 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION,
304 oid1.node(new NodeIdentifier(TestModel.INNER_LIST_QNAME))));
305 cursor1.write(innerListMapEntry.getIdentifier(), innerListMapEntry);
307 futures.add(tx1.submit());
311 futures.get(futures.size() - 1).checkedGet();
313 final DOMDataTreeListener mockedDataTreeListener = mock(DOMDataTreeListener.class);
314 doNothing().when(mockedDataTreeListener).onDataTreeChanged(anyCollection(), anyMap());
316 leaderShardFactory.registerListener(mockedDataTreeListener, Collections.singletonList(INNER_LIST_ID),
317 true, Collections.emptyList());
319 verify(mockedDataTreeListener, timeout(1000).times(1)).onDataTreeChanged(captorForChanges.capture(),
320 captorForSubtrees.capture());
321 verifyNoMoreInteractions(mockedDataTreeListener);
322 final List<Collection<DataTreeCandidate>> capturedValue = captorForChanges.getAllValues();
324 final NormalizedNode<?,?> expected =
325 ImmutableMapNodeBuilder
327 .withNodeIdentifier(new NodeIdentifier(TestModel.INNER_LIST_QNAME))
328 // only the values from the last run should be present
329 .withValue(createInnerListMapEntries(1000, "run-999"))
332 assertEquals("List values dont match the expected values from the last run",
333 expected, capturedValue.get(0).iterator().next().getRootNode().getDataAfter().get());
337 // top level shard at TEST element, with subshards on each outer-list map entry
338 @Ignore("https://bugs.opendaylight.org/show_bug.cgi?id=8116")
340 public void testMultipleShardLevels() throws Exception {
341 initEmptyDatastores();
343 final DistributedShardRegistration testShardReg = waitOnAsyncTask(
344 leaderShardFactory.createDistributedShard(TEST_ID, SINGLE_MEMBER),
345 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
347 final ArrayList<DistributedShardRegistration> registrations = new ArrayList<>();
348 final int listSize = 5;
349 for (int i = 0; i < listSize; i++) {
350 final YangInstanceIdentifier entryYID = getOuterListIdFor(i);
351 final CompletionStage<DistributedShardRegistration> future = leaderShardFactory.createDistributedShard(
352 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, entryYID), SINGLE_MEMBER);
354 registrations.add(waitOnAsyncTask(future, DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION));
357 final DOMDataTreeIdentifier rootId =
358 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, YangInstanceIdentifier.EMPTY);
359 final DOMDataTreeProducer producer = leaderShardFactory.createProducer(Collections.singletonList(
362 DOMDataTreeCursorAwareTransaction transaction = producer.createTransaction(false);
364 DOMDataTreeWriteCursor cursor = transaction.createCursor(rootId);
365 assertNotNull(cursor);
367 final MapNode outerList =
368 ImmutableMapNodeBuilder.create()
369 .withNodeIdentifier(new NodeIdentifier(TestModel.OUTER_LIST_QNAME)).build();
371 final ContainerNode testNode =
372 ImmutableContainerNodeBuilder.create()
373 .withNodeIdentifier(new NodeIdentifier(TestModel.TEST_QNAME))
374 .withChild(outerList)
377 cursor.write(testNode.getIdentifier(), testNode);
380 transaction.submit().checkedGet();
382 final DOMDataTreeListener mockedDataTreeListener = mock(DOMDataTreeListener.class);
383 doNothing().when(mockedDataTreeListener).onDataTreeChanged(anyCollection(), anyMap());
385 final MapNode wholeList = ImmutableMapNodeBuilder.create(outerList)
386 .withValue(createOuterEntries(listSize, "testing-values")).build();
388 transaction = producer.createTransaction(false);
389 cursor = transaction.createCursor(TEST_ID);
390 assertNotNull(cursor);
392 cursor.write(wholeList.getIdentifier(), wholeList);
395 transaction.submit().checkedGet();
397 leaderShardFactory.registerListener(mockedDataTreeListener, Collections.singletonList(TEST_ID),
398 true, Collections.emptyList());
400 // need 6 invocations, first initial thats from the parent shard, and then each individual subshard
401 verify(mockedDataTreeListener, timeout(20000).times(6)).onDataTreeChanged(captorForChanges.capture(),
402 captorForSubtrees.capture());
403 verifyNoMoreInteractions(mockedDataTreeListener);
404 final List<Map<DOMDataTreeIdentifier, NormalizedNode<?, ?>>> allSubtrees = captorForSubtrees.getAllValues();
406 final Map<DOMDataTreeIdentifier, NormalizedNode<?, ?>> lastSubtree = allSubtrees.get(allSubtrees.size() - 1);
408 final NormalizedNode<?, ?> actual = lastSubtree.get(TEST_ID);
409 assertNotNull(actual);
411 final NormalizedNode<?, ?> expected =
412 ImmutableContainerNodeBuilder.create()
413 .withNodeIdentifier(new NodeIdentifier(TestModel.TEST_QNAME))
414 .withChild(ImmutableMapNodeBuilder.create(outerList)
415 .withValue(createOuterEntries(listSize, "testing-values")).build())
419 for (final DistributedShardRegistration registration : registrations) {
420 waitOnAsyncTask(registration.close(), DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
423 waitOnAsyncTask(testShardReg.close(), DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
425 assertEquals(expected, actual);
429 public void testMultipleRegistrationsAtOnePrefix() throws Exception {
430 initEmptyDatastores();
432 for (int i = 0; i < 10; i++) {
433 LOG.debug("Round {}", i);
434 final DistributedShardRegistration reg1 = waitOnAsyncTask(leaderShardFactory.createDistributedShard(
435 TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
436 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
438 leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
439 ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
441 assertNotNull(findLocalShard(leaderDistributedDataStore.getActorContext(),
442 ClusterUtils.getCleanShardName(TestModel.TEST_PATH)));
444 waitOnAsyncTask(reg1.close(), DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
446 waitUntilShardIsDown(leaderDistributedDataStore.getActorContext(),
447 ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
451 private static Collection<MapEntryNode> createOuterEntries(final int amount, final String valuePrefix) {
452 final Collection<MapEntryNode> ret = new ArrayList<>();
453 for (int i = 0; i < amount; i++) {
454 ret.add(ImmutableNodes.mapEntryBuilder()
455 .withNodeIdentifier(new NodeIdentifierWithPredicates(TestModel.OUTER_LIST_QNAME,
456 QName.create(TestModel.OUTER_LIST_QNAME, "id"), i))
457 .withChild(ImmutableNodes
458 .leafNode(QName.create(TestModel.OUTER_LIST_QNAME, "id"), i))
459 .withChild(createWholeInnerList(amount, "outer id: " + i + " " + valuePrefix))
466 private static MapNode createWholeInnerList(final int amount, final String valuePrefix) {
467 return ImmutableMapNodeBuilder.create().withNodeIdentifier(new NodeIdentifier(TestModel.INNER_LIST_QNAME))
468 .withValue(createInnerListMapEntries(amount, valuePrefix)).build();
471 private static Collection<MapEntryNode> createInnerListMapEntries(final int amount, final String valuePrefix) {
472 final Collection<MapEntryNode> ret = new ArrayList<>();
473 for (int i = 0; i < amount; i++) {
474 ret.add(ImmutableNodes.mapEntryBuilder()
475 .withNodeIdentifier(new NodeIdentifierWithPredicates(TestModel.INNER_LIST_QNAME,
476 QName.create(TestModel.INNER_LIST_QNAME, "name"), Integer.toString(i)))
477 .withChild(ImmutableNodes
478 .leafNode(QName.create(TestModel.INNER_LIST_QNAME, "value"), valuePrefix + "-" + i))
485 private static YangInstanceIdentifier getOuterListIdFor(final int id) {
486 return TestModel.OUTER_LIST_PATH.node(new NodeIdentifierWithPredicates(
487 TestModel.OUTER_LIST_QNAME, QName.create(TestModel.OUTER_LIST_QNAME, "id"), id));