opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/sharding/DistributedShardedDOMDataTreeTest.java

   1 /*
   2  * Copyright (c) 2016 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.controller.cluster.sharding;
  10
  11 import static org.junit.Assert.assertEquals;
  12 import static org.junit.Assert.assertNotNull;
  13 import static org.junit.Assert.assertTrue;
  14 import static org.mockito.ArgumentMatchers.anyCollection;
  15 import static org.mockito.ArgumentMatchers.anyMap;
  16 import static org.mockito.Mockito.doNothing;
  17 import static org.mockito.Mockito.doReturn;
  18 import static org.mockito.Mockito.mock;
  19 import static org.mockito.Mockito.timeout;
  20 import static org.mockito.Mockito.verify;
  21 import static org.mockito.Mockito.verifyNoMoreInteractions;
  22 import static org.opendaylight.controller.cluster.datastore.IntegrationTestKit.findLocalShard;
  23 import static org.opendaylight.controller.cluster.datastore.IntegrationTestKit.waitUntilShardIsDown;
  24
  25 import akka.actor.ActorRef;
  26 import akka.actor.ActorSystem;
  27 import akka.actor.Address;
  28 import akka.actor.AddressFromURIString;
  29 import akka.actor.Props;
  30 import akka.cluster.Cluster;
  31 import akka.testkit.javadsl.TestKit;
  32 import com.google.common.collect.Lists;
  33 import com.google.common.util.concurrent.FluentFuture;
  34 import com.google.common.util.concurrent.ListenableFuture;
  35 import com.typesafe.config.ConfigFactory;
  36 import java.util.ArrayList;
  37 import java.util.Collection;
  38 import java.util.Collections;
  39 import java.util.List;
  40 import java.util.Map;
  41 import java.util.Optional;
  42 import java.util.Set;
  43 import java.util.concurrent.CompletionStage;
  44 import java.util.concurrent.TimeUnit;
  45 import org.junit.After;
  46 import org.junit.Assert;
  47 import org.junit.Before;
  48 import org.junit.Ignore;
  49 import org.junit.Test;
  50 import org.mockito.ArgumentCaptor;
  51 import org.mockito.Captor;
  52 import org.mockito.Mockito;
  53 import org.mockito.MockitoAnnotations;
  54 import org.opendaylight.controller.cluster.ActorSystemProvider;
  55 import org.opendaylight.controller.cluster.access.concepts.MemberName;
  56 import org.opendaylight.controller.cluster.databroker.actors.dds.ClientLocalHistory;
  57 import org.opendaylight.controller.cluster.databroker.actors.dds.ClientTransaction;
  58 import org.opendaylight.controller.cluster.databroker.actors.dds.DataStoreClient;
  59 import org.opendaylight.controller.cluster.databroker.actors.dds.SimpleDataStoreClientActor;
  60 import org.opendaylight.controller.cluster.datastore.AbstractTest;
  61 import org.opendaylight.controller.cluster.datastore.DatastoreContext;
  62 import org.opendaylight.controller.cluster.datastore.DatastoreContext.Builder;
  63 import org.opendaylight.controller.cluster.datastore.DistributedDataStore;
  64 import org.opendaylight.controller.cluster.datastore.IntegrationTestKit;
  65 import org.opendaylight.controller.cluster.datastore.utils.ActorUtils;
  66 import org.opendaylight.controller.cluster.datastore.utils.ClusterUtils;
  67 import org.opendaylight.controller.cluster.dom.api.CDSDataTreeProducer;
  68 import org.opendaylight.controller.cluster.dom.api.CDSShardAccess;
  69 import org.opendaylight.controller.cluster.raft.utils.InMemoryJournal;
  70 import org.opendaylight.controller.cluster.raft.utils.InMemorySnapshotStore;
  71 import org.opendaylight.controller.md.cluster.datastore.model.SchemaContextHelper;
  72 import org.opendaylight.controller.md.cluster.datastore.model.TestModel;
  73 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  74 import org.opendaylight.mdsal.dom.api.DOMDataTreeCursorAwareTransaction;
  75 import org.opendaylight.mdsal.dom.api.DOMDataTreeIdentifier;
  76 import org.opendaylight.mdsal.dom.api.DOMDataTreeListener;
  77 import org.opendaylight.mdsal.dom.api.DOMDataTreeProducer;
  78 import org.opendaylight.mdsal.dom.api.DOMDataTreeWriteCursor;
  79 import org.opendaylight.yangtools.yang.common.QName;
  80 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
  81 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifier;
  82 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifierWithPredicates;
  83 import org.opendaylight.yangtools.yang.data.api.schema.ContainerNode;
  84 import org.opendaylight.yangtools.yang.data.api.schema.LeafNode;
  85 import org.opendaylight.yangtools.yang.data.api.schema.MapEntryNode;
  86 import org.opendaylight.yangtools.yang.data.api.schema.MapNode;
  87 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
  88 import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidate;
  89 import org.opendaylight.yangtools.yang.data.impl.schema.ImmutableNodes;
  90 import org.opendaylight.yangtools.yang.data.impl.schema.builder.impl.ImmutableContainerNodeBuilder;
  91 import org.opendaylight.yangtools.yang.data.impl.schema.builder.impl.ImmutableLeafNodeBuilder;
  92 import org.opendaylight.yangtools.yang.data.impl.schema.builder.impl.ImmutableMapNodeBuilder;
  93 import org.slf4j.Logger;
  94 import org.slf4j.LoggerFactory;
  95
  96 public class DistributedShardedDOMDataTreeTest extends AbstractTest {
  97
  98     private static final Logger LOG = LoggerFactory.getLogger(DistributedShardedDOMDataTreeRemotingTest.class);
  99
 100     private static final Address MEMBER_1_ADDRESS =
 101             AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558");
 102
 103     private static final DOMDataTreeIdentifier TEST_ID =
 104             new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, TestModel.TEST_PATH);
 105
 106     private static final DOMDataTreeIdentifier INNER_LIST_ID =
 107             new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION,
 108                     YangInstanceIdentifier.create(getOuterListIdFor(0).getPathArguments())
 109                             .node(TestModel.INNER_LIST_QNAME));
 110     private static final Set<MemberName> SINGLE_MEMBER = Collections.singleton(AbstractTest.MEMBER_NAME);
 111
 112     private static final String MODULE_SHARDS_CONFIG = "module-shards-default-member-1.conf";
 113
 114     private ActorSystem leaderSystem;
 115
 116     private final Builder leaderDatastoreContextBuilder =
 117             DatastoreContext.newBuilder()
 118                     .shardHeartbeatIntervalInMillis(100)
 119                     .shardElectionTimeoutFactor(2)
 120                     .logicalStoreType(LogicalDatastoreType.CONFIGURATION);
 121
 122     private DistributedDataStore leaderDistributedDataStore;
 123     private DistributedDataStore operDistributedDatastore;
 124     private IntegrationTestKit leaderTestKit;
 125
 126     private DistributedShardedDOMDataTree leaderShardFactory;
 127
 128     @Captor
 129     private ArgumentCaptor<Collection<DataTreeCandidate>> captorForChanges;
 130     @Captor
 131     private ArgumentCaptor<Map<DOMDataTreeIdentifier, NormalizedNode<?, ?>>> captorForSubtrees;
 132
 133     private ActorSystemProvider leaderSystemProvider;
 134
 135     @Before
 136     public void setUp() {
 137         MockitoAnnotations.initMocks(this);
 138
 139         InMemoryJournal.clear();
 140         InMemorySnapshotStore.clear();
 141
 142         leaderSystem = ActorSystem.create("cluster-test", ConfigFactory.load().getConfig("Member1"));
 143         Cluster.get(leaderSystem).join(MEMBER_1_ADDRESS);
 144
 145         leaderSystemProvider = Mockito.mock(ActorSystemProvider.class);
 146         doReturn(leaderSystem).when(leaderSystemProvider).getActorSystem();
 147     }
 148
 149     @After
 150     public void tearDown() {
 151         if (leaderDistributedDataStore != null) {
 152             leaderDistributedDataStore.close();
 153         }
 154
 155         if (operDistributedDatastore != null) {
 156             operDistributedDatastore.close();
 157         }
 158
 159         TestKit.shutdownActorSystem(leaderSystem);
 160
 161         InMemoryJournal.clear();
 162         InMemorySnapshotStore.clear();
 163     }
 164
 165     private void initEmptyDatastores() throws Exception {
 166         leaderTestKit = new IntegrationTestKit(leaderSystem, leaderDatastoreContextBuilder);
 167
 168         leaderDistributedDataStore = leaderTestKit.setupDistributedDataStore(
 169                 "config", MODULE_SHARDS_CONFIG, "empty-modules.conf", true,
 170                 SchemaContextHelper.distributedShardedDOMDataTreeSchemaContext());
 171
 172         operDistributedDatastore = leaderTestKit.setupDistributedDataStore(
 173                 "operational", MODULE_SHARDS_CONFIG, "empty-modules.conf",true,
 174                 SchemaContextHelper.distributedShardedDOMDataTreeSchemaContext());
 175
 176         leaderShardFactory = new DistributedShardedDOMDataTree(leaderSystemProvider,
 177                 operDistributedDatastore,
 178                 leaderDistributedDataStore);
 179
 180         leaderShardFactory.init();
 181     }
 182
 183
 184     @Test
 185     public void testWritesIntoDefaultShard() throws Exception {
 186         initEmptyDatastores();
 187
 188         final DOMDataTreeIdentifier configRoot =
 189                 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, YangInstanceIdentifier.empty());
 190
 191         final DOMDataTreeProducer producer = leaderShardFactory.createProducer(Collections.singleton(configRoot));
 192
 193         final DOMDataTreeCursorAwareTransaction tx = producer.createTransaction(true);
 194         final DOMDataTreeWriteCursor cursor =
 195                 tx.createCursor(new DOMDataTreeIdentifier(
 196                         LogicalDatastoreType.CONFIGURATION, YangInstanceIdentifier.empty()));
 197         Assert.assertNotNull(cursor);
 198
 199         final ContainerNode test =
 200                 ImmutableContainerNodeBuilder.create()
 201                         .withNodeIdentifier(new NodeIdentifier(TestModel.TEST_QNAME)).build();
 202
 203         cursor.write(test.getIdentifier(), test);
 204         cursor.close();
 205
 206         tx.commit().get();
 207     }
 208
 209     @Test
 210     public void testSingleNodeWritesAndRead() throws Exception {
 211         initEmptyDatastores();
 212
 213         final DistributedShardRegistration shardRegistration = waitOnAsyncTask(
 214                 leaderShardFactory.createDistributedShard(TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
 215                 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 216
 217         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorUtils(),
 218                 ClusterUtils.getCleanShardName(TEST_ID.getRootIdentifier()));
 219
 220         final DOMDataTreeProducer producer = leaderShardFactory.createProducer(Collections.singleton(TEST_ID));
 221
 222         final DOMDataTreeCursorAwareTransaction tx = producer.createTransaction(true);
 223         final DOMDataTreeWriteCursor cursor = tx.createCursor(TEST_ID);
 224         Assert.assertNotNull(cursor);
 225         final YangInstanceIdentifier nameId =
 226                 YangInstanceIdentifier.builder(TestModel.TEST_PATH).node(TestModel.NAME_QNAME).build();
 227         final LeafNode<String> valueToCheck = ImmutableLeafNodeBuilder.<String>create().withNodeIdentifier(
 228                 new NodeIdentifier(TestModel.NAME_QNAME)).withValue("Test Value").build();
 229         LOG.debug("Writing data {} at {}, cursor {}", nameId.getLastPathArgument(), valueToCheck, cursor);
 230         cursor.write(nameId.getLastPathArgument(),
 231                 valueToCheck);
 232
 233         cursor.close();
 234         LOG.debug("Got to pre submit");
 235
 236         tx.commit().get();
 237
 238         final DOMDataTreeListener mockedDataTreeListener = mock(DOMDataTreeListener.class);
 239         doNothing().when(mockedDataTreeListener).onDataTreeChanged(anyCollection(), anyMap());
 240
 241         leaderShardFactory.registerListener(mockedDataTreeListener, Collections.singletonList(TEST_ID),
 242                 true, Collections.emptyList());
 243
 244         verify(mockedDataTreeListener, timeout(1000).times(1)).onDataTreeChanged(captorForChanges.capture(),
 245                 captorForSubtrees.capture());
 246         final List<Collection<DataTreeCandidate>> capturedValue = captorForChanges.getAllValues();
 247
 248         final Optional<NormalizedNode<?, ?>> dataAfter =
 249                 capturedValue.get(0).iterator().next().getRootNode().getDataAfter();
 250
 251         final NormalizedNode<?,?> expected = ImmutableContainerNodeBuilder.create()
 252                 .withNodeIdentifier(new NodeIdentifier(TestModel.TEST_QNAME)).withChild(valueToCheck).build();
 253         assertEquals(expected, dataAfter.get());
 254
 255         verifyNoMoreInteractions(mockedDataTreeListener);
 256
 257         final String shardName = ClusterUtils.getCleanShardName(TEST_ID.getRootIdentifier());
 258         LOG.debug("Creating distributed datastore client for shard {}", shardName);
 259
 260         final ActorUtils actorUtils = leaderDistributedDataStore.getActorUtils();
 261         final Props distributedDataStoreClientProps =
 262                 SimpleDataStoreClientActor.props(actorUtils.getCurrentMemberName(), "Shard-" + shardName, actorUtils,
 263                     shardName);
 264
 265         final ActorRef clientActor = leaderSystem.actorOf(distributedDataStoreClientProps);
 266         final DataStoreClient distributedDataStoreClient = SimpleDataStoreClientActor
 267                     .getDistributedDataStoreClient(clientActor, 30, TimeUnit.SECONDS);
 268
 269         final ClientLocalHistory localHistory = distributedDataStoreClient.createLocalHistory();
 270         final ClientTransaction tx2 = localHistory.createTransaction();
 271         final FluentFuture<Optional<NormalizedNode<?, ?>>> read = tx2.read(YangInstanceIdentifier.empty());
 272
 273         final Optional<NormalizedNode<?, ?>> optional = read.get();
 274         tx2.abort();
 275         localHistory.close();
 276
 277         shardRegistration.close().toCompletableFuture().get();
 278
 279     }
 280
 281     @Test
 282     public void testMultipleWritesIntoSingleMapEntry() throws Exception {
 283         initEmptyDatastores();
 284
 285         final DistributedShardRegistration shardRegistration = waitOnAsyncTask(
 286                 leaderShardFactory.createDistributedShard(TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
 287                 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 288
 289         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorUtils(),
 290                 ClusterUtils.getCleanShardName(TEST_ID.getRootIdentifier()));
 291
 292         LOG.warn("Got after waiting for nonleader");
 293         final ActorRef leaderShardManager = leaderDistributedDataStore.getActorUtils().getShardManager();
 294
 295         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorUtils(),
 296                 ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
 297
 298         final YangInstanceIdentifier oid1 = getOuterListIdFor(0);
 299         final DOMDataTreeIdentifier outerListPath = new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, oid1);
 300
 301         final DistributedShardRegistration outerListShardReg = waitOnAsyncTask(
 302                 leaderShardFactory.createDistributedShard(outerListPath, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
 303                 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 304
 305         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorUtils(),
 306                 ClusterUtils.getCleanShardName(outerListPath.getRootIdentifier()));
 307
 308         final DOMDataTreeProducer shardProducer = leaderShardFactory.createProducer(
 309                 Collections.singletonList(outerListPath));
 310
 311         final DOMDataTreeCursorAwareTransaction tx = shardProducer.createTransaction(false);
 312         final DOMDataTreeWriteCursor cursor =
 313                 tx.createCursor(new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, oid1));
 314         assertNotNull(cursor);
 315
 316         MapNode innerList = ImmutableMapNodeBuilder
 317                 .create()
 318                 .withNodeIdentifier(new NodeIdentifier(TestModel.INNER_LIST_QNAME))
 319                 .build();
 320
 321         cursor.write(new NodeIdentifier(TestModel.INNER_LIST_QNAME), innerList);
 322         cursor.close();
 323         tx.commit().get();
 324
 325         final ArrayList<ListenableFuture<?>> futures = new ArrayList<>();
 326         for (int i = 0; i < 1000; i++) {
 327             final Collection<MapEntryNode> innerListMapEntries = createInnerListMapEntries(1000, "run-" + i);
 328             for (final MapEntryNode innerListMapEntry : innerListMapEntries) {
 329                 final DOMDataTreeCursorAwareTransaction tx1 = shardProducer.createTransaction(false);
 330                 final DOMDataTreeWriteCursor cursor1 = tx1.createCursor(
 331                         new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION,
 332                                 oid1.node(new NodeIdentifier(TestModel.INNER_LIST_QNAME))));
 333                 cursor1.write(innerListMapEntry.getIdentifier(), innerListMapEntry);
 334                 cursor1.close();
 335                 futures.add(tx1.commit());
 336             }
 337         }
 338
 339         futures.get(futures.size() - 1).get();
 340
 341         final DOMDataTreeListener mockedDataTreeListener = mock(DOMDataTreeListener.class);
 342         doNothing().when(mockedDataTreeListener).onDataTreeChanged(anyCollection(), anyMap());
 343
 344         leaderShardFactory.registerListener(mockedDataTreeListener, Collections.singletonList(INNER_LIST_ID),
 345                 true, Collections.emptyList());
 346
 347         verify(mockedDataTreeListener, timeout(1000).times(1)).onDataTreeChanged(captorForChanges.capture(),
 348                 captorForSubtrees.capture());
 349         verifyNoMoreInteractions(mockedDataTreeListener);
 350         final List<Collection<DataTreeCandidate>> capturedValue = captorForChanges.getAllValues();
 351
 352         final NormalizedNode<?,?> expected =
 353                 ImmutableMapNodeBuilder
 354                         .create()
 355                         .withNodeIdentifier(new NodeIdentifier(TestModel.INNER_LIST_QNAME))
 356                                 // only the values from the last run should be present
 357                         .withValue(createInnerListMapEntries(1000, "run-999"))
 358                         .build();
 359
 360         assertEquals("List values dont match the expected values from the last run",
 361                 expected, capturedValue.get(0).iterator().next().getRootNode().getDataAfter().get());
 362
 363     }
 364
 365     // top level shard at TEST element, with subshards on each outer-list map entry
 366     @Test
 367     @Ignore
 368     public void testMultipleShardLevels() throws Exception {
 369         initEmptyDatastores();
 370
 371         final DistributedShardRegistration testShardReg = waitOnAsyncTask(
 372                 leaderShardFactory.createDistributedShard(TEST_ID, SINGLE_MEMBER),
 373                 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 374
 375         final ArrayList<DistributedShardRegistration> registrations = new ArrayList<>();
 376         final int listSize = 5;
 377         for (int i = 0; i < listSize; i++) {
 378             final YangInstanceIdentifier entryYID = getOuterListIdFor(i);
 379             final CompletionStage<DistributedShardRegistration> future = leaderShardFactory.createDistributedShard(
 380                     new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, entryYID), SINGLE_MEMBER);
 381
 382             registrations.add(waitOnAsyncTask(future, DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION));
 383         }
 384
 385         final DOMDataTreeIdentifier rootId =
 386                 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, YangInstanceIdentifier.empty());
 387         final DOMDataTreeProducer producer = leaderShardFactory.createProducer(Collections.singletonList(
 388                 rootId));
 389
 390         DOMDataTreeCursorAwareTransaction transaction = producer.createTransaction(false);
 391
 392         DOMDataTreeWriteCursor cursor = transaction.createCursor(rootId);
 393         assertNotNull(cursor);
 394
 395         final MapNode outerList =
 396                 ImmutableMapNodeBuilder.create()
 397                         .withNodeIdentifier(new NodeIdentifier(TestModel.OUTER_LIST_QNAME)).build();
 398
 399         final ContainerNode testNode =
 400                 ImmutableContainerNodeBuilder.create()
 401                         .withNodeIdentifier(new NodeIdentifier(TestModel.TEST_QNAME))
 402                         .withChild(outerList)
 403                         .build();
 404
 405         cursor.write(testNode.getIdentifier(), testNode);
 406
 407         cursor.close();
 408         transaction.commit().get();
 409
 410         final DOMDataTreeListener mockedDataTreeListener = mock(DOMDataTreeListener.class);
 411         doNothing().when(mockedDataTreeListener).onDataTreeChanged(anyCollection(), anyMap());
 412
 413         final MapNode wholeList = ImmutableMapNodeBuilder.create(outerList)
 414                 .withValue(createOuterEntries(listSize, "testing-values")).build();
 415
 416         transaction = producer.createTransaction(false);
 417         cursor = transaction.createCursor(TEST_ID);
 418         assertNotNull(cursor);
 419
 420         cursor.write(wholeList.getIdentifier(), wholeList);
 421         cursor.close();
 422
 423         transaction.commit().get();
 424
 425         leaderShardFactory.registerListener(mockedDataTreeListener, Collections.singletonList(TEST_ID),
 426                 true, Collections.emptyList());
 427
 428         verify(mockedDataTreeListener, timeout(35000).atLeast(2)).onDataTreeChanged(captorForChanges.capture(),
 429                 captorForSubtrees.capture());
 430         verifyNoMoreInteractions(mockedDataTreeListener);
 431         final List<Map<DOMDataTreeIdentifier, NormalizedNode<?, ?>>> allSubtrees = captorForSubtrees.getAllValues();
 432
 433         final Map<DOMDataTreeIdentifier, NormalizedNode<?, ?>> lastSubtree = allSubtrees.get(allSubtrees.size() - 1);
 434
 435         final NormalizedNode<?, ?> actual = lastSubtree.get(TEST_ID);
 436         assertNotNull(actual);
 437
 438         final NormalizedNode<?, ?> expected =
 439                 ImmutableContainerNodeBuilder.create()
 440                         .withNodeIdentifier(new NodeIdentifier(TestModel.TEST_QNAME))
 441                         .withChild(ImmutableMapNodeBuilder.create(outerList)
 442                                 .withValue(createOuterEntries(listSize, "testing-values")).build())
 443                         .build();
 444
 445
 446         for (final DistributedShardRegistration registration : registrations) {
 447             waitOnAsyncTask(registration.close(), DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 448         }
 449
 450         waitOnAsyncTask(testShardReg.close(), DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 451
 452         assertEquals(expected, actual);
 453     }
 454
 455     @Test
 456     public void testMultipleRegistrationsAtOnePrefix() throws Exception {
 457         initEmptyDatastores();
 458
 459         for (int i = 0; i < 10; i++) {
 460             LOG.debug("Round {}", i);
 461             final DistributedShardRegistration reg1 = waitOnAsyncTask(leaderShardFactory.createDistributedShard(
 462                     TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
 463                     DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 464
 465             leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorUtils(),
 466                     ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
 467
 468             assertNotNull(findLocalShard(leaderDistributedDataStore.getActorUtils(),
 469                     ClusterUtils.getCleanShardName(TestModel.TEST_PATH)));
 470
 471             waitOnAsyncTask(reg1.close(), DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 472
 473             waitUntilShardIsDown(leaderDistributedDataStore.getActorUtils(),
 474                     ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
 475         }
 476     }
 477
 478     @Test
 479     public void testCDSDataTreeProducer() throws Exception {
 480         initEmptyDatastores();
 481
 482         final DistributedShardRegistration reg1 = waitOnAsyncTask(leaderShardFactory.createDistributedShard(
 483                 TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
 484                 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 485
 486         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorUtils(),
 487                 ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
 488
 489         assertNotNull(findLocalShard(leaderDistributedDataStore.getActorUtils(),
 490                 ClusterUtils.getCleanShardName(TestModel.TEST_PATH)));
 491
 492
 493         final DOMDataTreeIdentifier configRoot =
 494                 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, YangInstanceIdentifier.empty());
 495         final DOMDataTreeProducer producer = leaderShardFactory.createProducer(Collections.singleton(configRoot));
 496
 497         assertTrue(producer instanceof CDSDataTreeProducer);
 498
 499         final CDSDataTreeProducer cdsProducer = (CDSDataTreeProducer) producer;
 500         CDSShardAccess shardAccess = cdsProducer.getShardAccess(TEST_ID);
 501         assertEquals(shardAccess.getShardIdentifier(), TEST_ID);
 502
 503         shardAccess = cdsProducer.getShardAccess(INNER_LIST_ID);
 504         assertEquals(TEST_ID, shardAccess.getShardIdentifier());
 505
 506         shardAccess = cdsProducer.getShardAccess(configRoot);
 507         assertEquals(configRoot, shardAccess.getShardIdentifier());
 508
 509         waitOnAsyncTask(reg1.close(), DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 510     }
 511
 512     private static Collection<MapEntryNode> createOuterEntries(final int amount, final String valuePrefix) {
 513         final Collection<MapEntryNode> ret = new ArrayList<>();
 514         for (int i = 0; i < amount; i++) {
 515             ret.add(ImmutableNodes.mapEntryBuilder()
 516                     .withNodeIdentifier(NodeIdentifierWithPredicates.of(TestModel.OUTER_LIST_QNAME,
 517                             QName.create(TestModel.OUTER_LIST_QNAME, "id"), i))
 518                     .withChild(ImmutableNodes
 519                             .leafNode(QName.create(TestModel.OUTER_LIST_QNAME, "id"), i))
 520                     .withChild(createWholeInnerList(amount, "outer id: " + i + " " + valuePrefix))
 521                     .build());
 522         }
 523
 524         return ret;
 525     }
 526
 527     private static MapNode createWholeInnerList(final int amount, final String valuePrefix) {
 528         return ImmutableMapNodeBuilder.create().withNodeIdentifier(new NodeIdentifier(TestModel.INNER_LIST_QNAME))
 529                 .withValue(createInnerListMapEntries(amount, valuePrefix)).build();
 530     }
 531
 532     private static Collection<MapEntryNode> createInnerListMapEntries(final int amount, final String valuePrefix) {
 533         final Collection<MapEntryNode> ret = new ArrayList<>();
 534         for (int i = 0; i < amount; i++) {
 535             ret.add(ImmutableNodes.mapEntryBuilder()
 536                     .withNodeIdentifier(NodeIdentifierWithPredicates.of(TestModel.INNER_LIST_QNAME,
 537                             QName.create(TestModel.INNER_LIST_QNAME, "name"), Integer.toString(i)))
 538                     .withChild(ImmutableNodes
 539                             .leafNode(QName.create(TestModel.INNER_LIST_QNAME, "value"), valuePrefix + "-" + i))
 540                     .build());
 541         }
 542
 543         return ret;
 544     }
 545
 546     private static YangInstanceIdentifier getOuterListIdFor(final int id) {
 547         return TestModel.OUTER_LIST_PATH.node(NodeIdentifierWithPredicates.of(
 548                 TestModel.OUTER_LIST_QNAME, QName.create(TestModel.OUTER_LIST_QNAME, "id"), id));
 549     }
 550 }