2 * Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
9 package org.opendaylight.controller.cluster.sharding;
11 import static org.junit.Assert.assertEquals;
12 import static org.junit.Assert.assertNotNull;
13 import static org.mockito.Matchers.anyCollection;
14 import static org.mockito.Matchers.anyMap;
15 import static org.mockito.Mockito.doNothing;
16 import static org.mockito.Mockito.doReturn;
17 import static org.mockito.Mockito.mock;
18 import static org.mockito.Mockito.timeout;
19 import static org.mockito.Mockito.verify;
20 import static org.mockito.Mockito.verifyNoMoreInteractions;
21 import static org.opendaylight.controller.cluster.datastore.IntegrationTestKit.findLocalShard;
22 import static org.opendaylight.controller.cluster.datastore.IntegrationTestKit.waitUntilShardIsDown;
24 import akka.actor.ActorRef;
25 import akka.actor.ActorSystem;
26 import akka.actor.Address;
27 import akka.actor.AddressFromURIString;
28 import akka.cluster.Cluster;
29 import akka.testkit.JavaTestKit;
30 import com.google.common.base.Optional;
31 import com.google.common.collect.Lists;
32 import com.google.common.util.concurrent.CheckedFuture;
33 import com.typesafe.config.ConfigFactory;
34 import java.util.ArrayList;
35 import java.util.Collection;
36 import java.util.Collections;
37 import java.util.List;
39 import org.junit.After;
40 import org.junit.Assert;
41 import org.junit.Before;
42 import org.junit.Ignore;
43 import org.junit.Test;
44 import org.mockito.ArgumentCaptor;
45 import org.mockito.Captor;
46 import org.mockito.Mockito;
47 import org.mockito.MockitoAnnotations;
48 import org.opendaylight.controller.cluster.ActorSystemProvider;
49 import org.opendaylight.controller.cluster.datastore.AbstractTest;
50 import org.opendaylight.controller.cluster.datastore.DatastoreContext;
51 import org.opendaylight.controller.cluster.datastore.DatastoreContext.Builder;
52 import org.opendaylight.controller.cluster.datastore.DistributedDataStore;
53 import org.opendaylight.controller.cluster.datastore.IntegrationTestKit;
54 import org.opendaylight.controller.cluster.datastore.utils.ClusterUtils;
55 import org.opendaylight.controller.cluster.sharding.DistributedShardFactory.DistributedShardRegistration;
56 import org.opendaylight.controller.md.cluster.datastore.model.SchemaContextHelper;
57 import org.opendaylight.controller.md.cluster.datastore.model.TestModel;
58 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
59 import org.opendaylight.mdsal.common.api.TransactionCommitFailedException;
60 import org.opendaylight.mdsal.dom.api.DOMDataTreeCursorAwareTransaction;
61 import org.opendaylight.mdsal.dom.api.DOMDataTreeIdentifier;
62 import org.opendaylight.mdsal.dom.api.DOMDataTreeListener;
63 import org.opendaylight.mdsal.dom.api.DOMDataTreeProducer;
64 import org.opendaylight.mdsal.dom.api.DOMDataTreeWriteCursor;
65 import org.opendaylight.yangtools.yang.common.QName;
66 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
67 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifier;
68 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifierWithPredicates;
69 import org.opendaylight.yangtools.yang.data.api.schema.ContainerNode;
70 import org.opendaylight.yangtools.yang.data.api.schema.LeafNode;
71 import org.opendaylight.yangtools.yang.data.api.schema.MapEntryNode;
72 import org.opendaylight.yangtools.yang.data.api.schema.MapNode;
73 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
74 import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidate;
75 import org.opendaylight.yangtools.yang.data.impl.schema.ImmutableNodes;
76 import org.opendaylight.yangtools.yang.data.impl.schema.builder.impl.ImmutableContainerNodeBuilder;
77 import org.opendaylight.yangtools.yang.data.impl.schema.builder.impl.ImmutableLeafNodeBuilder;
78 import org.opendaylight.yangtools.yang.data.impl.schema.builder.impl.ImmutableMapNodeBuilder;
79 import org.slf4j.Logger;
80 import org.slf4j.LoggerFactory;
82 @Ignore("distributed-data is broken needs to be removed")
83 public class DistributedShardedDOMDataTreeTest extends AbstractTest {
85 private static final Logger LOG = LoggerFactory.getLogger(DistributedShardedDOMDataTreeRemotingTest.class);
87 private static final Address MEMBER_1_ADDRESS =
88 AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558");
90 private static final DOMDataTreeIdentifier TEST_ID =
91 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, TestModel.TEST_PATH);
93 private static final DOMDataTreeIdentifier INNER_LIST_ID =
94 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION,
95 YangInstanceIdentifier.create(getOuterListIdFor(0).getPathArguments())
96 .node(TestModel.INNER_LIST_QNAME));
98 private ActorSystem leaderSystem;
100 private final Builder leaderDatastoreContextBuilder =
101 DatastoreContext.newBuilder()
102 .shardHeartbeatIntervalInMillis(100)
103 .shardElectionTimeoutFactor(2)
105 org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType.CONFIGURATION);
107 private DistributedDataStore leaderDistributedDataStore;
108 private IntegrationTestKit leaderTestKit;
110 private DistributedShardedDOMDataTree leaderShardFactory;
113 private ArgumentCaptor<Collection<DataTreeCandidate>> captorForChanges;
115 private ArgumentCaptor<Map<DOMDataTreeIdentifier, NormalizedNode<?, ?>>> captorForSubtrees;
117 private ActorSystemProvider leaderSystemProvider;
120 public void setUp() {
121 MockitoAnnotations.initMocks(this);
123 leaderSystem = ActorSystem.create("cluster-test", ConfigFactory.load().getConfig("Member1"));
124 Cluster.get(leaderSystem).join(MEMBER_1_ADDRESS);
126 leaderSystemProvider = Mockito.mock(ActorSystemProvider.class);
127 doReturn(leaderSystem).when(leaderSystemProvider).getActorSystem();
131 public void tearDown() {
132 if (leaderDistributedDataStore != null) {
133 leaderDistributedDataStore.close();
136 JavaTestKit.shutdownActorSystem(leaderSystem);
139 private void initEmptyDatastore(final String type) {
140 leaderTestKit = new IntegrationTestKit(leaderSystem, leaderDatastoreContextBuilder);
142 leaderDistributedDataStore =
143 leaderTestKit.setupDistributedDataStoreWithoutConfig(type, SchemaContextHelper.full());
146 leaderShardFactory = new DistributedShardedDOMDataTree(leaderSystemProvider,
147 leaderDistributedDataStore,
148 leaderDistributedDataStore);
153 public void testWritesIntoDefaultShard() throws Exception {
154 initEmptyDatastore("config");
156 final DOMDataTreeIdentifier configRoot =
157 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, YangInstanceIdentifier.EMPTY);
159 final DOMDataTreeProducer producer = leaderShardFactory.createProducer(Collections.singleton(configRoot));
161 final DOMDataTreeCursorAwareTransaction tx = producer.createTransaction(true);
162 final DOMDataTreeWriteCursor cursor =
163 tx.createCursor(new DOMDataTreeIdentifier(
164 LogicalDatastoreType.CONFIGURATION, YangInstanceIdentifier.EMPTY));
165 Assert.assertNotNull(cursor);
167 final ContainerNode test =
168 ImmutableContainerNodeBuilder.create()
169 .withNodeIdentifier(new NodeIdentifier(TestModel.TEST_QNAME)).build();
171 cursor.write(test.getIdentifier(), test);
174 tx.submit().checkedGet();
178 public void testSingleNodeWrites() throws Exception {
179 initEmptyDatastore("config");
181 final DistributedShardRegistration shardRegistration = waitOnAsyncTask(
182 leaderShardFactory.createDistributedShard(TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
183 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
185 leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
186 ClusterUtils.getCleanShardName(TEST_ID.getRootIdentifier()));
188 final DOMDataTreeProducer producer = leaderShardFactory.createProducer(Collections.singleton(TEST_ID));
190 final DOMDataTreeCursorAwareTransaction tx = producer.createTransaction(true);
191 final DOMDataTreeWriteCursor cursor = tx.createCursor(TEST_ID);
192 Assert.assertNotNull(cursor);
193 final YangInstanceIdentifier nameId =
194 YangInstanceIdentifier.builder(TestModel.TEST_PATH).node(TestModel.NAME_QNAME).build();
195 final LeafNode<String> valueToCheck = ImmutableLeafNodeBuilder.<String>create().withNodeIdentifier(
196 new NodeIdentifier(TestModel.NAME_QNAME)).withValue("Test Value").build();
197 cursor.write(nameId.getLastPathArgument(),
201 LOG.debug("Got to pre submit");
203 tx.submit().checkedGet();
205 final DOMDataTreeListener mockedDataTreeListener = mock(DOMDataTreeListener.class);
206 doNothing().when(mockedDataTreeListener).onDataTreeChanged(anyCollection(), anyMap());
208 leaderShardFactory.registerListener(mockedDataTreeListener, Collections.singletonList(TEST_ID),
209 true, Collections.emptyList());
211 verify(mockedDataTreeListener, timeout(1000).times(1)).onDataTreeChanged(captorForChanges.capture(),
212 captorForSubtrees.capture());
213 final List<Collection<DataTreeCandidate>> capturedValue = captorForChanges.getAllValues();
215 final Optional<NormalizedNode<?, ?>> dataAfter =
216 capturedValue.get(0).iterator().next().getRootNode().getDataAfter();
218 final NormalizedNode<?,?> expected = ImmutableContainerNodeBuilder.create()
219 .withNodeIdentifier(new NodeIdentifier(TestModel.TEST_QNAME)).withChild(valueToCheck).build();
220 assertEquals(expected, dataAfter.get());
222 verifyNoMoreInteractions(mockedDataTreeListener);
227 public void testMultipleWritesIntoSingleMapEntry() throws Exception {
228 initEmptyDatastore("config");
230 final DistributedShardRegistration shardRegistration = waitOnAsyncTask(
231 leaderShardFactory.createDistributedShard(TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
232 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
234 leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
235 ClusterUtils.getCleanShardName(TEST_ID.getRootIdentifier()));
237 LOG.warn("Got after waiting for nonleader");
238 final ActorRef leaderShardManager = leaderDistributedDataStore.getActorContext().getShardManager();
240 leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
241 ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
243 final YangInstanceIdentifier oid1 = getOuterListIdFor(0);
244 final DOMDataTreeIdentifier outerListPath = new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, oid1);
246 final DistributedShardRegistration outerListShardReg = waitOnAsyncTask(
247 leaderShardFactory.createDistributedShard(outerListPath, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
248 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
250 leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
251 ClusterUtils.getCleanShardName(outerListPath.getRootIdentifier()));
253 final DOMDataTreeProducer shardProducer = leaderShardFactory.createProducer(
254 Collections.singletonList(outerListPath));
256 final DOMDataTreeCursorAwareTransaction tx = shardProducer.createTransaction(false);
257 final DOMDataTreeWriteCursor cursor =
258 tx.createCursor(new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, oid1));
259 assertNotNull(cursor);
261 MapNode innerList = ImmutableMapNodeBuilder
263 .withNodeIdentifier(new NodeIdentifier(TestModel.INNER_LIST_QNAME))
266 cursor.write(new NodeIdentifier(TestModel.INNER_LIST_QNAME), innerList);
268 tx.submit().checkedGet();
270 final ArrayList<CheckedFuture<Void, TransactionCommitFailedException>> futures = new ArrayList<>();
271 for (int i = 0; i < 1000; i++) {
272 final Collection<MapEntryNode> innerListMapEntries = createInnerListMapEntries(1000, "run-" + i);
273 for (final MapEntryNode innerListMapEntry : innerListMapEntries) {
274 final DOMDataTreeCursorAwareTransaction tx1 = shardProducer.createTransaction(false);
275 final DOMDataTreeWriteCursor cursor1 = tx1.createCursor(
276 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION,
277 oid1.node(new NodeIdentifier(TestModel.INNER_LIST_QNAME))));
278 cursor1.write(innerListMapEntry.getIdentifier(), innerListMapEntry);
280 futures.add(tx1.submit());
284 futures.get(futures.size() - 1).checkedGet();
286 final DOMDataTreeListener mockedDataTreeListener = mock(DOMDataTreeListener.class);
287 doNothing().when(mockedDataTreeListener).onDataTreeChanged(anyCollection(), anyMap());
289 leaderShardFactory.registerListener(mockedDataTreeListener, Collections.singletonList(INNER_LIST_ID),
290 true, Collections.emptyList());
292 verify(mockedDataTreeListener, timeout(1000).times(1)).onDataTreeChanged(captorForChanges.capture(),
293 captorForSubtrees.capture());
294 verifyNoMoreInteractions(mockedDataTreeListener);
295 final List<Collection<DataTreeCandidate>> capturedValue = captorForChanges.getAllValues();
297 final NormalizedNode<?,?> expected =
298 ImmutableMapNodeBuilder
300 .withNodeIdentifier(new NodeIdentifier(TestModel.INNER_LIST_QNAME))
301 // only the values from the last run should be present
302 .withValue(createInnerListMapEntries(1000, "run-999"))
305 assertEquals("List values dont match the expected values from the last run",
306 expected, capturedValue.get(0).iterator().next().getRootNode().getDataAfter().get());
310 private static Collection<MapEntryNode> createInnerListMapEntries(final int amount, final String valuePrefix) {
311 final Collection<MapEntryNode> ret = new ArrayList<>();
312 for (int i = 0; i < amount; i++) {
313 ret.add(ImmutableNodes.mapEntryBuilder()
314 .withNodeIdentifier(new NodeIdentifierWithPredicates(TestModel.INNER_LIST_QNAME,
315 QName.create(TestModel.INNER_LIST_QNAME, "name"), Integer.toString(i)))
316 .withChild(ImmutableNodes
317 .leafNode(QName.create(TestModel.INNER_LIST_QNAME, "value"), valuePrefix + "-" + i))
325 public void testDistributedData() throws Exception {
326 initEmptyDatastore("config");
329 leaderShardFactory.createDistributedShard(TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
330 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
333 leaderShardFactory.createDistributedShard(
334 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, TestModel.OUTER_CONTAINER_PATH),
335 Lists.newArrayList(AbstractTest.MEMBER_NAME)),
336 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
339 leaderShardFactory.createDistributedShard(
340 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, TestModel.INNER_LIST_PATH),
341 Lists.newArrayList(AbstractTest.MEMBER_NAME)),
342 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
345 leaderShardFactory.createDistributedShard(
346 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, TestModel.JUNK_PATH),
347 Lists.newArrayList(AbstractTest.MEMBER_NAME)),
348 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
350 leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
351 ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
352 leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
353 ClusterUtils.getCleanShardName(TestModel.OUTER_CONTAINER_PATH));
354 leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
355 ClusterUtils.getCleanShardName(TestModel.INNER_LIST_PATH));
356 leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
357 ClusterUtils.getCleanShardName(TestModel.JUNK_PATH));
362 public void testMultipleRegistrationsAtOnePrefix() throws Exception {
363 initEmptyDatastore("config");
365 for (int i = 0; i < 10; i++) {
366 LOG.debug("Round {}", i);
367 final DistributedShardRegistration reg1 = waitOnAsyncTask(leaderShardFactory.createDistributedShard(
368 TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
369 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
371 leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
372 ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
374 assertNotNull(findLocalShard(leaderDistributedDataStore.getActorContext(),
375 ClusterUtils.getCleanShardName(TestModel.TEST_PATH)));
377 waitOnAsyncTask(reg1.close(), DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
379 waitUntilShardIsDown(leaderDistributedDataStore.getActorContext(),
380 ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
384 private static YangInstanceIdentifier getOuterListIdFor(final int id) {
385 return TestModel.OUTER_LIST_PATH.node(new NodeIdentifierWithPredicates(
386 TestModel.OUTER_LIST_QNAME, QName.create(TestModel.OUTER_LIST_QNAME, "id"), id));