2 * Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
9 package org.opendaylight.controller.cluster.sharding;
11 import static org.junit.Assert.assertEquals;
12 import static org.junit.Assert.assertNotNull;
13 import static org.mockito.Matchers.anyCollection;
14 import static org.mockito.Matchers.anyMap;
15 import static org.mockito.Mockito.doNothing;
16 import static org.mockito.Mockito.doReturn;
17 import static org.mockito.Mockito.mock;
18 import static org.mockito.Mockito.timeout;
19 import static org.mockito.Mockito.verify;
20 import static org.mockito.Mockito.verifyNoMoreInteractions;
21 import static org.opendaylight.controller.cluster.datastore.IntegrationTestKit.findLocalShard;
22 import static org.opendaylight.controller.cluster.datastore.IntegrationTestKit.waitUntilShardIsDown;
24 import akka.actor.ActorRef;
25 import akka.actor.ActorSystem;
26 import akka.actor.Address;
27 import akka.actor.AddressFromURIString;
28 import akka.cluster.Cluster;
29 import akka.testkit.JavaTestKit;
30 import com.google.common.base.Optional;
31 import com.google.common.collect.Lists;
32 import com.google.common.util.concurrent.CheckedFuture;
33 import com.typesafe.config.ConfigFactory;
34 import java.util.ArrayList;
35 import java.util.Collection;
36 import java.util.Collections;
37 import java.util.List;
40 import java.util.concurrent.CompletionStage;
41 import org.junit.After;
42 import org.junit.Assert;
43 import org.junit.Before;
44 import org.junit.Ignore;
45 import org.junit.Test;
46 import org.mockito.ArgumentCaptor;
47 import org.mockito.Captor;
48 import org.mockito.Mockito;
49 import org.mockito.MockitoAnnotations;
50 import org.opendaylight.controller.cluster.ActorSystemProvider;
51 import org.opendaylight.controller.cluster.access.concepts.MemberName;
52 import org.opendaylight.controller.cluster.datastore.AbstractTest;
53 import org.opendaylight.controller.cluster.datastore.DatastoreContext;
54 import org.opendaylight.controller.cluster.datastore.DatastoreContext.Builder;
55 import org.opendaylight.controller.cluster.datastore.DistributedDataStore;
56 import org.opendaylight.controller.cluster.datastore.IntegrationTestKit;
57 import org.opendaylight.controller.cluster.datastore.utils.ClusterUtils;
58 import org.opendaylight.controller.cluster.sharding.DistributedShardFactory.DistributedShardRegistration;
59 import org.opendaylight.controller.md.cluster.datastore.model.SchemaContextHelper;
60 import org.opendaylight.controller.md.cluster.datastore.model.TestModel;
61 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
62 import org.opendaylight.mdsal.common.api.TransactionCommitFailedException;
63 import org.opendaylight.mdsal.dom.api.DOMDataTreeCursorAwareTransaction;
64 import org.opendaylight.mdsal.dom.api.DOMDataTreeIdentifier;
65 import org.opendaylight.mdsal.dom.api.DOMDataTreeListener;
66 import org.opendaylight.mdsal.dom.api.DOMDataTreeProducer;
67 import org.opendaylight.mdsal.dom.api.DOMDataTreeWriteCursor;
68 import org.opendaylight.yangtools.yang.common.QName;
69 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
70 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifier;
71 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifierWithPredicates;
72 import org.opendaylight.yangtools.yang.data.api.schema.ContainerNode;
73 import org.opendaylight.yangtools.yang.data.api.schema.LeafNode;
74 import org.opendaylight.yangtools.yang.data.api.schema.MapEntryNode;
75 import org.opendaylight.yangtools.yang.data.api.schema.MapNode;
76 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
77 import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidate;
78 import org.opendaylight.yangtools.yang.data.impl.schema.ImmutableNodes;
79 import org.opendaylight.yangtools.yang.data.impl.schema.builder.impl.ImmutableContainerNodeBuilder;
80 import org.opendaylight.yangtools.yang.data.impl.schema.builder.impl.ImmutableLeafNodeBuilder;
81 import org.opendaylight.yangtools.yang.data.impl.schema.builder.impl.ImmutableMapNodeBuilder;
82 import org.slf4j.Logger;
83 import org.slf4j.LoggerFactory;
85 @Ignore("distributed-data is broken needs to be removed")
86 public class DistributedShardedDOMDataTreeTest extends AbstractTest {
88 private static final Logger LOG = LoggerFactory.getLogger(DistributedShardedDOMDataTreeRemotingTest.class);
90 private static final Address MEMBER_1_ADDRESS =
91 AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558");
93 private static final DOMDataTreeIdentifier TEST_ID =
94 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, TestModel.TEST_PATH);
96 private static final DOMDataTreeIdentifier INNER_LIST_ID =
97 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION,
98 YangInstanceIdentifier.create(getOuterListIdFor(0).getPathArguments())
99 .node(TestModel.INNER_LIST_QNAME));
100 private static final Set<MemberName> SINGLE_MEMBER = Collections.singleton(AbstractTest.MEMBER_NAME);
102 private ActorSystem leaderSystem;
104 private final Builder leaderDatastoreContextBuilder =
105 DatastoreContext.newBuilder()
106 .shardHeartbeatIntervalInMillis(100)
107 .shardElectionTimeoutFactor(2)
109 org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType.CONFIGURATION);
111 private DistributedDataStore leaderDistributedDataStore;
112 private IntegrationTestKit leaderTestKit;
114 private DistributedShardedDOMDataTree leaderShardFactory;
117 private ArgumentCaptor<Collection<DataTreeCandidate>> captorForChanges;
119 private ArgumentCaptor<Map<DOMDataTreeIdentifier, NormalizedNode<?, ?>>> captorForSubtrees;
121 private ActorSystemProvider leaderSystemProvider;
124 public void setUp() {
125 MockitoAnnotations.initMocks(this);
127 leaderSystem = ActorSystem.create("cluster-test", ConfigFactory.load().getConfig("Member1"));
128 Cluster.get(leaderSystem).join(MEMBER_1_ADDRESS);
130 leaderSystemProvider = Mockito.mock(ActorSystemProvider.class);
131 doReturn(leaderSystem).when(leaderSystemProvider).getActorSystem();
135 public void tearDown() {
136 if (leaderDistributedDataStore != null) {
137 leaderDistributedDataStore.close();
140 JavaTestKit.shutdownActorSystem(leaderSystem);
143 private void initEmptyDatastore(final String type) {
144 leaderTestKit = new IntegrationTestKit(leaderSystem, leaderDatastoreContextBuilder);
146 leaderDistributedDataStore =
147 leaderTestKit.setupDistributedDataStoreWithoutConfig(type, SchemaContextHelper.full());
150 leaderShardFactory = new DistributedShardedDOMDataTree(leaderSystemProvider,
151 leaderDistributedDataStore,
152 leaderDistributedDataStore);
157 public void testWritesIntoDefaultShard() throws Exception {
158 initEmptyDatastore("config");
160 final DOMDataTreeIdentifier configRoot =
161 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, YangInstanceIdentifier.EMPTY);
163 final DOMDataTreeProducer producer = leaderShardFactory.createProducer(Collections.singleton(configRoot));
165 final DOMDataTreeCursorAwareTransaction tx = producer.createTransaction(true);
166 final DOMDataTreeWriteCursor cursor =
167 tx.createCursor(new DOMDataTreeIdentifier(
168 LogicalDatastoreType.CONFIGURATION, YangInstanceIdentifier.EMPTY));
169 Assert.assertNotNull(cursor);
171 final ContainerNode test =
172 ImmutableContainerNodeBuilder.create()
173 .withNodeIdentifier(new NodeIdentifier(TestModel.TEST_QNAME)).build();
175 cursor.write(test.getIdentifier(), test);
178 tx.submit().checkedGet();
182 public void testSingleNodeWrites() throws Exception {
183 initEmptyDatastore("config");
185 final DistributedShardRegistration shardRegistration = waitOnAsyncTask(
186 leaderShardFactory.createDistributedShard(TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
187 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
189 leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
190 ClusterUtils.getCleanShardName(TEST_ID.getRootIdentifier()));
192 final DOMDataTreeProducer producer = leaderShardFactory.createProducer(Collections.singleton(TEST_ID));
194 final DOMDataTreeCursorAwareTransaction tx = producer.createTransaction(true);
195 final DOMDataTreeWriteCursor cursor = tx.createCursor(TEST_ID);
196 Assert.assertNotNull(cursor);
197 final YangInstanceIdentifier nameId =
198 YangInstanceIdentifier.builder(TestModel.TEST_PATH).node(TestModel.NAME_QNAME).build();
199 final LeafNode<String> valueToCheck = ImmutableLeafNodeBuilder.<String>create().withNodeIdentifier(
200 new NodeIdentifier(TestModel.NAME_QNAME)).withValue("Test Value").build();
201 cursor.write(nameId.getLastPathArgument(),
205 LOG.debug("Got to pre submit");
207 tx.submit().checkedGet();
209 final DOMDataTreeListener mockedDataTreeListener = mock(DOMDataTreeListener.class);
210 doNothing().when(mockedDataTreeListener).onDataTreeChanged(anyCollection(), anyMap());
212 leaderShardFactory.registerListener(mockedDataTreeListener, Collections.singletonList(TEST_ID),
213 true, Collections.emptyList());
215 verify(mockedDataTreeListener, timeout(1000).times(1)).onDataTreeChanged(captorForChanges.capture(),
216 captorForSubtrees.capture());
217 final List<Collection<DataTreeCandidate>> capturedValue = captorForChanges.getAllValues();
219 final Optional<NormalizedNode<?, ?>> dataAfter =
220 capturedValue.get(0).iterator().next().getRootNode().getDataAfter();
222 final NormalizedNode<?,?> expected = ImmutableContainerNodeBuilder.create()
223 .withNodeIdentifier(new NodeIdentifier(TestModel.TEST_QNAME)).withChild(valueToCheck).build();
224 assertEquals(expected, dataAfter.get());
226 verifyNoMoreInteractions(mockedDataTreeListener);
231 public void testMultipleWritesIntoSingleMapEntry() throws Exception {
232 initEmptyDatastore("config");
234 final DistributedShardRegistration shardRegistration = waitOnAsyncTask(
235 leaderShardFactory.createDistributedShard(TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
236 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
238 leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
239 ClusterUtils.getCleanShardName(TEST_ID.getRootIdentifier()));
241 LOG.warn("Got after waiting for nonleader");
242 final ActorRef leaderShardManager = leaderDistributedDataStore.getActorContext().getShardManager();
244 leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
245 ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
247 final YangInstanceIdentifier oid1 = getOuterListIdFor(0);
248 final DOMDataTreeIdentifier outerListPath = new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, oid1);
250 final DistributedShardRegistration outerListShardReg = waitOnAsyncTask(
251 leaderShardFactory.createDistributedShard(outerListPath, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
252 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
254 leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
255 ClusterUtils.getCleanShardName(outerListPath.getRootIdentifier()));
257 final DOMDataTreeProducer shardProducer = leaderShardFactory.createProducer(
258 Collections.singletonList(outerListPath));
260 final DOMDataTreeCursorAwareTransaction tx = shardProducer.createTransaction(false);
261 final DOMDataTreeWriteCursor cursor =
262 tx.createCursor(new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, oid1));
263 assertNotNull(cursor);
265 MapNode innerList = ImmutableMapNodeBuilder
267 .withNodeIdentifier(new NodeIdentifier(TestModel.INNER_LIST_QNAME))
270 cursor.write(new NodeIdentifier(TestModel.INNER_LIST_QNAME), innerList);
272 tx.submit().checkedGet();
274 final ArrayList<CheckedFuture<Void, TransactionCommitFailedException>> futures = new ArrayList<>();
275 for (int i = 0; i < 1000; i++) {
276 final Collection<MapEntryNode> innerListMapEntries = createInnerListMapEntries(1000, "run-" + i);
277 for (final MapEntryNode innerListMapEntry : innerListMapEntries) {
278 final DOMDataTreeCursorAwareTransaction tx1 = shardProducer.createTransaction(false);
279 final DOMDataTreeWriteCursor cursor1 = tx1.createCursor(
280 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION,
281 oid1.node(new NodeIdentifier(TestModel.INNER_LIST_QNAME))));
282 cursor1.write(innerListMapEntry.getIdentifier(), innerListMapEntry);
284 futures.add(tx1.submit());
288 futures.get(futures.size() - 1).checkedGet();
290 final DOMDataTreeListener mockedDataTreeListener = mock(DOMDataTreeListener.class);
291 doNothing().when(mockedDataTreeListener).onDataTreeChanged(anyCollection(), anyMap());
293 leaderShardFactory.registerListener(mockedDataTreeListener, Collections.singletonList(INNER_LIST_ID),
294 true, Collections.emptyList());
296 verify(mockedDataTreeListener, timeout(1000).times(1)).onDataTreeChanged(captorForChanges.capture(),
297 captorForSubtrees.capture());
298 verifyNoMoreInteractions(mockedDataTreeListener);
299 final List<Collection<DataTreeCandidate>> capturedValue = captorForChanges.getAllValues();
301 final NormalizedNode<?,?> expected =
302 ImmutableMapNodeBuilder
304 .withNodeIdentifier(new NodeIdentifier(TestModel.INNER_LIST_QNAME))
305 // only the values from the last run should be present
306 .withValue(createInnerListMapEntries(1000, "run-999"))
309 assertEquals("List values dont match the expected values from the last run",
310 expected, capturedValue.get(0).iterator().next().getRootNode().getDataAfter().get());
314 // top level shard at TEST element, with subshards on each outer-list map entry
316 public void testMultipleShardLevels() throws Exception {
317 initEmptyDatastore("config");
319 final DistributedShardRegistration testShardId = waitOnAsyncTask(
320 leaderShardFactory.createDistributedShard(TEST_ID, SINGLE_MEMBER),
321 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
323 final ArrayList<DistributedShardRegistration> registrations = new ArrayList<>();
324 final int listSize = 5;
325 for (int i = 0; i < listSize; i++) {
326 final YangInstanceIdentifier entryYID = getOuterListIdFor(i);
327 final CompletionStage<DistributedShardRegistration> future = leaderShardFactory.createDistributedShard(
328 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, entryYID), SINGLE_MEMBER);
330 registrations.add(waitOnAsyncTask(future, DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION));
333 final DOMDataTreeIdentifier rootId =
334 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, YangInstanceIdentifier.EMPTY);
335 final DOMDataTreeProducer producer = leaderShardFactory.createProducer(Collections.singletonList(
338 DOMDataTreeCursorAwareTransaction transaction = producer.createTransaction(false);
340 DOMDataTreeWriteCursor cursor = transaction.createCursor(rootId);
341 assertNotNull(cursor);
343 final MapNode outerList =
344 ImmutableMapNodeBuilder.create()
345 .withNodeIdentifier(new NodeIdentifier(TestModel.OUTER_LIST_QNAME)).build();
347 final ContainerNode testNode =
348 ImmutableContainerNodeBuilder.create()
349 .withNodeIdentifier(new NodeIdentifier(TestModel.TEST_QNAME))
350 .withChild(outerList)
353 cursor.write(testNode.getIdentifier(), testNode);
356 transaction.submit().checkedGet();
358 final DOMDataTreeListener mockedDataTreeListener = mock(DOMDataTreeListener.class);
359 doNothing().when(mockedDataTreeListener).onDataTreeChanged(anyCollection(), anyMap());
361 final MapNode wholeList = ImmutableMapNodeBuilder.create(outerList)
362 .withValue(createOuterEntries(listSize, "testing-values")).build();
364 transaction = producer.createTransaction(false);
365 cursor = transaction.createCursor(TEST_ID);
366 assertNotNull(cursor);
368 cursor.write(wholeList.getIdentifier(), wholeList);
371 transaction.submit().checkedGet();
373 leaderShardFactory.registerListener(mockedDataTreeListener, Collections.singletonList(TEST_ID),
374 true, Collections.emptyList());
376 // need 6 invocations, first initial thats from the parent shard, and then each individual subshard
377 verify(mockedDataTreeListener, timeout(10000).times(6)).onDataTreeChanged(captorForChanges.capture(),
378 captorForSubtrees.capture());
379 verifyNoMoreInteractions(mockedDataTreeListener);
380 final List<Map<DOMDataTreeIdentifier, NormalizedNode<?, ?>>> allSubtrees = captorForSubtrees.getAllValues();
382 final Map<DOMDataTreeIdentifier, NormalizedNode<?, ?>> lastSubtree = allSubtrees.get(allSubtrees.size() - 1);
384 final NormalizedNode<?, ?> actual = lastSubtree.get(TEST_ID);
385 assertNotNull(actual);
387 final NormalizedNode<?, ?> expected =
388 ImmutableContainerNodeBuilder.create()
389 .withNodeIdentifier(new NodeIdentifier(TestModel.TEST_QNAME))
390 .withChild(ImmutableMapNodeBuilder.create(outerList)
391 .withValue(createOuterEntries(listSize, "testing-values")).build())
394 assertEquals(expected, actual);
398 public void testDistributedData() throws Exception {
399 initEmptyDatastore("config");
402 leaderShardFactory.createDistributedShard(TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
403 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
406 leaderShardFactory.createDistributedShard(
407 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, TestModel.OUTER_CONTAINER_PATH),
408 Lists.newArrayList(AbstractTest.MEMBER_NAME)),
409 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
412 leaderShardFactory.createDistributedShard(
413 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, TestModel.INNER_LIST_PATH),
414 Lists.newArrayList(AbstractTest.MEMBER_NAME)),
415 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
418 leaderShardFactory.createDistributedShard(
419 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, TestModel.JUNK_PATH),
420 Lists.newArrayList(AbstractTest.MEMBER_NAME)),
421 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
423 leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
424 ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
425 leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
426 ClusterUtils.getCleanShardName(TestModel.OUTER_CONTAINER_PATH));
427 leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
428 ClusterUtils.getCleanShardName(TestModel.INNER_LIST_PATH));
429 leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
430 ClusterUtils.getCleanShardName(TestModel.JUNK_PATH));
435 public void testMultipleRegistrationsAtOnePrefix() throws Exception {
436 initEmptyDatastore("config");
438 for (int i = 0; i < 10; i++) {
439 LOG.debug("Round {}", i);
440 final DistributedShardRegistration reg1 = waitOnAsyncTask(leaderShardFactory.createDistributedShard(
441 TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
442 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
444 leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
445 ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
447 assertNotNull(findLocalShard(leaderDistributedDataStore.getActorContext(),
448 ClusterUtils.getCleanShardName(TestModel.TEST_PATH)));
450 waitOnAsyncTask(reg1.close(), DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
452 waitUntilShardIsDown(leaderDistributedDataStore.getActorContext(),
453 ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
457 private static Collection<MapEntryNode> createOuterEntries(final int amount, final String valuePrefix) {
458 final Collection<MapEntryNode> ret = new ArrayList<>();
459 for (int i = 0; i < amount; i++) {
460 ret.add(ImmutableNodes.mapEntryBuilder()
461 .withNodeIdentifier(new NodeIdentifierWithPredicates(TestModel.OUTER_LIST_QNAME,
462 QName.create(TestModel.OUTER_LIST_QNAME, "id"), i))
463 .withChild(ImmutableNodes
464 .leafNode(QName.create(TestModel.OUTER_LIST_QNAME, "id"), i))
465 .withChild(createWholeInnerList(amount, "outer id: " + i + " " + valuePrefix))
472 private static MapNode createWholeInnerList(final int amount, final String valuePrefix) {
473 return ImmutableMapNodeBuilder.create().withNodeIdentifier(new NodeIdentifier(TestModel.INNER_LIST_QNAME))
474 .withValue(createInnerListMapEntries(amount, valuePrefix)).build();
477 private static Collection<MapEntryNode> createInnerListMapEntries(final int amount, final String valuePrefix) {
478 final Collection<MapEntryNode> ret = new ArrayList<>();
479 for (int i = 0; i < amount; i++) {
480 ret.add(ImmutableNodes.mapEntryBuilder()
481 .withNodeIdentifier(new NodeIdentifierWithPredicates(TestModel.INNER_LIST_QNAME,
482 QName.create(TestModel.INNER_LIST_QNAME, "name"), Integer.toString(i)))
483 .withChild(ImmutableNodes
484 .leafNode(QName.create(TestModel.INNER_LIST_QNAME, "value"), valuePrefix + "-" + i))
491 private static YangInstanceIdentifier getOuterListIdFor(final int id) {
492 return TestModel.OUTER_LIST_PATH.node(new NodeIdentifierWithPredicates(
493 TestModel.OUTER_LIST_QNAME, QName.create(TestModel.OUTER_LIST_QNAME, "id"), id));