opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/sharding/DistributedShardedDOMDataTreeTest.java

   1 /*
   2  * Copyright (c) 2016 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.controller.cluster.sharding;
  10
  11 import static org.junit.Assert.assertEquals;
  12 import static org.junit.Assert.assertNotNull;
  13 import static org.mockito.Matchers.anyCollection;
  14 import static org.mockito.Matchers.anyMap;
  15 import static org.mockito.Mockito.doNothing;
  16 import static org.mockito.Mockito.doReturn;
  17 import static org.mockito.Mockito.mock;
  18 import static org.mockito.Mockito.timeout;
  19 import static org.mockito.Mockito.verify;
  20 import static org.mockito.Mockito.verifyNoMoreInteractions;
  21 import static org.opendaylight.controller.cluster.datastore.IntegrationTestKit.findLocalShard;
  22 import static org.opendaylight.controller.cluster.datastore.IntegrationTestKit.waitUntilShardIsDown;
  23
  24 import akka.actor.ActorRef;
  25 import akka.actor.ActorSystem;
  26 import akka.actor.Address;
  27 import akka.actor.AddressFromURIString;
  28 import akka.cluster.Cluster;
  29 import akka.testkit.JavaTestKit;
  30 import com.google.common.base.Optional;
  31 import com.google.common.collect.Lists;
  32 import com.google.common.util.concurrent.CheckedFuture;
  33 import com.typesafe.config.ConfigFactory;
  34 import java.util.ArrayList;
  35 import java.util.Collection;
  36 import java.util.Collections;
  37 import java.util.List;
  38 import java.util.Map;
  39 import java.util.Set;
  40 import java.util.concurrent.CompletionStage;
  41 import org.junit.After;
  42 import org.junit.Assert;
  43 import org.junit.Before;
  44 import org.junit.Ignore;
  45 import org.junit.Test;
  46 import org.mockito.ArgumentCaptor;
  47 import org.mockito.Captor;
  48 import org.mockito.Mockito;
  49 import org.mockito.MockitoAnnotations;
  50 import org.opendaylight.controller.cluster.ActorSystemProvider;
  51 import org.opendaylight.controller.cluster.access.concepts.MemberName;
  52 import org.opendaylight.controller.cluster.datastore.AbstractTest;
  53 import org.opendaylight.controller.cluster.datastore.DatastoreContext;
  54 import org.opendaylight.controller.cluster.datastore.DatastoreContext.Builder;
  55 import org.opendaylight.controller.cluster.datastore.DistributedDataStore;
  56 import org.opendaylight.controller.cluster.datastore.IntegrationTestKit;
  57 import org.opendaylight.controller.cluster.datastore.utils.ClusterUtils;
  58 import org.opendaylight.controller.cluster.sharding.DistributedShardFactory.DistributedShardRegistration;
  59 import org.opendaylight.controller.md.cluster.datastore.model.SchemaContextHelper;
  60 import org.opendaylight.controller.md.cluster.datastore.model.TestModel;
  61 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  62 import org.opendaylight.mdsal.common.api.TransactionCommitFailedException;
  63 import org.opendaylight.mdsal.dom.api.DOMDataTreeCursorAwareTransaction;
  64 import org.opendaylight.mdsal.dom.api.DOMDataTreeIdentifier;
  65 import org.opendaylight.mdsal.dom.api.DOMDataTreeListener;
  66 import org.opendaylight.mdsal.dom.api.DOMDataTreeProducer;
  67 import org.opendaylight.mdsal.dom.api.DOMDataTreeWriteCursor;
  68 import org.opendaylight.yangtools.yang.common.QName;
  69 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
  70 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifier;
  71 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifierWithPredicates;
  72 import org.opendaylight.yangtools.yang.data.api.schema.ContainerNode;
  73 import org.opendaylight.yangtools.yang.data.api.schema.LeafNode;
  74 import org.opendaylight.yangtools.yang.data.api.schema.MapEntryNode;
  75 import org.opendaylight.yangtools.yang.data.api.schema.MapNode;
  76 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
  77 import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidate;
  78 import org.opendaylight.yangtools.yang.data.impl.schema.ImmutableNodes;
  79 import org.opendaylight.yangtools.yang.data.impl.schema.builder.impl.ImmutableContainerNodeBuilder;
  80 import org.opendaylight.yangtools.yang.data.impl.schema.builder.impl.ImmutableLeafNodeBuilder;
  81 import org.opendaylight.yangtools.yang.data.impl.schema.builder.impl.ImmutableMapNodeBuilder;
  82 import org.slf4j.Logger;
  83 import org.slf4j.LoggerFactory;
  84
  85 @Ignore("distributed-data is broken needs to be removed")
  86 public class DistributedShardedDOMDataTreeTest extends AbstractTest {
  87
  88     private static final Logger LOG = LoggerFactory.getLogger(DistributedShardedDOMDataTreeRemotingTest.class);
  89
  90     private static final Address MEMBER_1_ADDRESS =
  91             AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558");
  92
  93     private static final DOMDataTreeIdentifier TEST_ID =
  94             new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, TestModel.TEST_PATH);
  95
  96     private static final DOMDataTreeIdentifier INNER_LIST_ID =
  97             new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION,
  98                     YangInstanceIdentifier.create(getOuterListIdFor(0).getPathArguments())
  99                             .node(TestModel.INNER_LIST_QNAME));
 100     private static final Set<MemberName> SINGLE_MEMBER = Collections.singleton(AbstractTest.MEMBER_NAME);
 101
 102     private ActorSystem leaderSystem;
 103
 104     private final Builder leaderDatastoreContextBuilder =
 105             DatastoreContext.newBuilder()
 106                     .shardHeartbeatIntervalInMillis(100)
 107                     .shardElectionTimeoutFactor(2)
 108                     .logicalStoreType(
 109                             org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType.CONFIGURATION);
 110
 111     private DistributedDataStore leaderDistributedDataStore;
 112     private IntegrationTestKit leaderTestKit;
 113
 114     private DistributedShardedDOMDataTree leaderShardFactory;
 115
 116     @Captor
 117     private ArgumentCaptor<Collection<DataTreeCandidate>> captorForChanges;
 118     @Captor
 119     private ArgumentCaptor<Map<DOMDataTreeIdentifier, NormalizedNode<?, ?>>> captorForSubtrees;
 120
 121     private ActorSystemProvider leaderSystemProvider;
 122
 123     @Before
 124     public void setUp() {
 125         MockitoAnnotations.initMocks(this);
 126
 127         leaderSystem = ActorSystem.create("cluster-test", ConfigFactory.load().getConfig("Member1"));
 128         Cluster.get(leaderSystem).join(MEMBER_1_ADDRESS);
 129
 130         leaderSystemProvider = Mockito.mock(ActorSystemProvider.class);
 131         doReturn(leaderSystem).when(leaderSystemProvider).getActorSystem();
 132     }
 133
 134     @After
 135     public void tearDown() {
 136         if (leaderDistributedDataStore != null) {
 137             leaderDistributedDataStore.close();
 138         }
 139
 140         JavaTestKit.shutdownActorSystem(leaderSystem);
 141     }
 142
 143     private void initEmptyDatastore(final String type) {
 144         leaderTestKit = new IntegrationTestKit(leaderSystem, leaderDatastoreContextBuilder);
 145
 146         leaderDistributedDataStore =
 147                 leaderTestKit.setupDistributedDataStoreWithoutConfig(type, SchemaContextHelper.full());
 148
 149
 150         leaderShardFactory = new DistributedShardedDOMDataTree(leaderSystemProvider,
 151                 leaderDistributedDataStore,
 152                 leaderDistributedDataStore);
 153     }
 154
 155
 156     @Test
 157     public void testWritesIntoDefaultShard() throws Exception {
 158         initEmptyDatastore("config");
 159
 160         final DOMDataTreeIdentifier configRoot =
 161                 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, YangInstanceIdentifier.EMPTY);
 162
 163         final DOMDataTreeProducer producer = leaderShardFactory.createProducer(Collections.singleton(configRoot));
 164
 165         final DOMDataTreeCursorAwareTransaction tx = producer.createTransaction(true);
 166         final DOMDataTreeWriteCursor cursor =
 167                 tx.createCursor(new DOMDataTreeIdentifier(
 168                         LogicalDatastoreType.CONFIGURATION, YangInstanceIdentifier.EMPTY));
 169         Assert.assertNotNull(cursor);
 170
 171         final ContainerNode test =
 172                 ImmutableContainerNodeBuilder.create()
 173                         .withNodeIdentifier(new NodeIdentifier(TestModel.TEST_QNAME)).build();
 174
 175         cursor.write(test.getIdentifier(), test);
 176         cursor.close();
 177
 178         tx.submit().checkedGet();
 179     }
 180
 181     @Test
 182     public void testSingleNodeWrites() throws Exception {
 183         initEmptyDatastore("config");
 184
 185         final DistributedShardRegistration shardRegistration = waitOnAsyncTask(
 186                 leaderShardFactory.createDistributedShard(TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
 187                 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 188
 189         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 190                 ClusterUtils.getCleanShardName(TEST_ID.getRootIdentifier()));
 191
 192         final DOMDataTreeProducer producer = leaderShardFactory.createProducer(Collections.singleton(TEST_ID));
 193
 194         final DOMDataTreeCursorAwareTransaction tx = producer.createTransaction(true);
 195         final DOMDataTreeWriteCursor cursor = tx.createCursor(TEST_ID);
 196         Assert.assertNotNull(cursor);
 197         final YangInstanceIdentifier nameId =
 198                 YangInstanceIdentifier.builder(TestModel.TEST_PATH).node(TestModel.NAME_QNAME).build();
 199         final LeafNode<String> valueToCheck = ImmutableLeafNodeBuilder.<String>create().withNodeIdentifier(
 200                 new NodeIdentifier(TestModel.NAME_QNAME)).withValue("Test Value").build();
 201         cursor.write(nameId.getLastPathArgument(),
 202                 valueToCheck);
 203
 204         cursor.close();
 205         LOG.debug("Got to pre submit");
 206
 207         tx.submit().checkedGet();
 208
 209         final DOMDataTreeListener mockedDataTreeListener = mock(DOMDataTreeListener.class);
 210         doNothing().when(mockedDataTreeListener).onDataTreeChanged(anyCollection(), anyMap());
 211
 212         leaderShardFactory.registerListener(mockedDataTreeListener, Collections.singletonList(TEST_ID),
 213                 true, Collections.emptyList());
 214
 215         verify(mockedDataTreeListener, timeout(1000).times(1)).onDataTreeChanged(captorForChanges.capture(),
 216                 captorForSubtrees.capture());
 217         final List<Collection<DataTreeCandidate>> capturedValue = captorForChanges.getAllValues();
 218
 219         final Optional<NormalizedNode<?, ?>> dataAfter =
 220                 capturedValue.get(0).iterator().next().getRootNode().getDataAfter();
 221
 222         final NormalizedNode<?,?> expected = ImmutableContainerNodeBuilder.create()
 223                 .withNodeIdentifier(new NodeIdentifier(TestModel.TEST_QNAME)).withChild(valueToCheck).build();
 224         assertEquals(expected, dataAfter.get());
 225
 226         verifyNoMoreInteractions(mockedDataTreeListener);
 227
 228     }
 229
 230     @Test
 231     public void testMultipleWritesIntoSingleMapEntry() throws Exception {
 232         initEmptyDatastore("config");
 233
 234         final DistributedShardRegistration shardRegistration = waitOnAsyncTask(
 235                 leaderShardFactory.createDistributedShard(TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
 236                 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 237
 238         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 239                 ClusterUtils.getCleanShardName(TEST_ID.getRootIdentifier()));
 240
 241         LOG.warn("Got after waiting for nonleader");
 242         final ActorRef leaderShardManager = leaderDistributedDataStore.getActorContext().getShardManager();
 243
 244         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 245                 ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
 246
 247         final YangInstanceIdentifier oid1 = getOuterListIdFor(0);
 248         final DOMDataTreeIdentifier outerListPath = new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, oid1);
 249
 250         final DistributedShardRegistration outerListShardReg = waitOnAsyncTask(
 251                 leaderShardFactory.createDistributedShard(outerListPath, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
 252                 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 253
 254         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 255                 ClusterUtils.getCleanShardName(outerListPath.getRootIdentifier()));
 256
 257         final DOMDataTreeProducer shardProducer = leaderShardFactory.createProducer(
 258                 Collections.singletonList(outerListPath));
 259
 260         final DOMDataTreeCursorAwareTransaction tx = shardProducer.createTransaction(false);
 261         final DOMDataTreeWriteCursor cursor =
 262                 tx.createCursor(new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, oid1));
 263         assertNotNull(cursor);
 264
 265         MapNode innerList = ImmutableMapNodeBuilder
 266                 .create()
 267                 .withNodeIdentifier(new NodeIdentifier(TestModel.INNER_LIST_QNAME))
 268                 .build();
 269
 270         cursor.write(new NodeIdentifier(TestModel.INNER_LIST_QNAME), innerList);
 271         cursor.close();
 272         tx.submit().checkedGet();
 273
 274         final ArrayList<CheckedFuture<Void, TransactionCommitFailedException>> futures = new ArrayList<>();
 275         for (int i = 0; i < 1000; i++) {
 276             final Collection<MapEntryNode> innerListMapEntries = createInnerListMapEntries(1000, "run-" + i);
 277             for (final MapEntryNode innerListMapEntry : innerListMapEntries) {
 278                 final DOMDataTreeCursorAwareTransaction tx1 = shardProducer.createTransaction(false);
 279                 final DOMDataTreeWriteCursor cursor1 = tx1.createCursor(
 280                         new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION,
 281                                 oid1.node(new NodeIdentifier(TestModel.INNER_LIST_QNAME))));
 282                 cursor1.write(innerListMapEntry.getIdentifier(), innerListMapEntry);
 283                 cursor1.close();
 284                 futures.add(tx1.submit());
 285             }
 286         }
 287
 288         futures.get(futures.size() - 1).checkedGet();
 289
 290         final DOMDataTreeListener mockedDataTreeListener = mock(DOMDataTreeListener.class);
 291         doNothing().when(mockedDataTreeListener).onDataTreeChanged(anyCollection(), anyMap());
 292
 293         leaderShardFactory.registerListener(mockedDataTreeListener, Collections.singletonList(INNER_LIST_ID),
 294                 true, Collections.emptyList());
 295
 296         verify(mockedDataTreeListener, timeout(1000).times(1)).onDataTreeChanged(captorForChanges.capture(),
 297                 captorForSubtrees.capture());
 298         verifyNoMoreInteractions(mockedDataTreeListener);
 299         final List<Collection<DataTreeCandidate>> capturedValue = captorForChanges.getAllValues();
 300
 301         final NormalizedNode<?,?> expected =
 302                 ImmutableMapNodeBuilder
 303                         .create()
 304                         .withNodeIdentifier(new NodeIdentifier(TestModel.INNER_LIST_QNAME))
 305                                 // only the values from the last run should be present
 306                         .withValue(createInnerListMapEntries(1000, "run-999"))
 307                         .build();
 308
 309         assertEquals("List values dont match the expected values from the last run",
 310                 expected, capturedValue.get(0).iterator().next().getRootNode().getDataAfter().get());
 311
 312     }
 313
 314     // top level shard at TEST element, with subshards on each outer-list map entry
 315     @Test
 316     public void testMultipleShardLevels() throws Exception {
 317         initEmptyDatastore("config");
 318
 319         final DistributedShardRegistration testShardId = waitOnAsyncTask(
 320                 leaderShardFactory.createDistributedShard(TEST_ID, SINGLE_MEMBER),
 321                 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 322
 323         final ArrayList<DistributedShardRegistration> registrations = new ArrayList<>();
 324         final int listSize = 5;
 325         for (int i = 0; i < listSize; i++) {
 326             final YangInstanceIdentifier entryYID = getOuterListIdFor(i);
 327             final CompletionStage<DistributedShardRegistration> future = leaderShardFactory.createDistributedShard(
 328                     new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, entryYID), SINGLE_MEMBER);
 329
 330             registrations.add(waitOnAsyncTask(future, DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION));
 331         }
 332
 333         final DOMDataTreeIdentifier rootId =
 334                 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, YangInstanceIdentifier.EMPTY);
 335         final DOMDataTreeProducer producer = leaderShardFactory.createProducer(Collections.singletonList(
 336                 rootId));
 337
 338         DOMDataTreeCursorAwareTransaction transaction = producer.createTransaction(false);
 339
 340         DOMDataTreeWriteCursor cursor = transaction.createCursor(rootId);
 341         assertNotNull(cursor);
 342
 343         final MapNode outerList =
 344                 ImmutableMapNodeBuilder.create()
 345                         .withNodeIdentifier(new NodeIdentifier(TestModel.OUTER_LIST_QNAME)).build();
 346
 347         final ContainerNode testNode =
 348                 ImmutableContainerNodeBuilder.create()
 349                         .withNodeIdentifier(new NodeIdentifier(TestModel.TEST_QNAME))
 350                         .withChild(outerList)
 351                         .build();
 352
 353         cursor.write(testNode.getIdentifier(), testNode);
 354
 355         cursor.close();
 356         transaction.submit().checkedGet();
 357
 358         final DOMDataTreeListener mockedDataTreeListener = mock(DOMDataTreeListener.class);
 359         doNothing().when(mockedDataTreeListener).onDataTreeChanged(anyCollection(), anyMap());
 360
 361         final MapNode wholeList = ImmutableMapNodeBuilder.create(outerList)
 362                 .withValue(createOuterEntries(listSize, "testing-values")).build();
 363
 364         transaction = producer.createTransaction(false);
 365         cursor = transaction.createCursor(TEST_ID);
 366         assertNotNull(cursor);
 367
 368         cursor.write(wholeList.getIdentifier(), wholeList);
 369         cursor.close();
 370
 371         transaction.submit().checkedGet();
 372
 373         leaderShardFactory.registerListener(mockedDataTreeListener, Collections.singletonList(TEST_ID),
 374                 true, Collections.emptyList());
 375
 376         // need 6 invocations, first initial thats from the parent shard, and then each individual subshard
 377         verify(mockedDataTreeListener, timeout(10000).times(6)).onDataTreeChanged(captorForChanges.capture(),
 378                 captorForSubtrees.capture());
 379         verifyNoMoreInteractions(mockedDataTreeListener);
 380         final List<Map<DOMDataTreeIdentifier, NormalizedNode<?, ?>>> allSubtrees = captorForSubtrees.getAllValues();
 381
 382         final Map<DOMDataTreeIdentifier, NormalizedNode<?, ?>> lastSubtree = allSubtrees.get(allSubtrees.size() - 1);
 383
 384         final NormalizedNode<?, ?> actual = lastSubtree.get(TEST_ID);
 385         assertNotNull(actual);
 386
 387         final NormalizedNode<?, ?> expected =
 388                 ImmutableContainerNodeBuilder.create()
 389                         .withNodeIdentifier(new NodeIdentifier(TestModel.TEST_QNAME))
 390                         .withChild(ImmutableMapNodeBuilder.create(outerList)
 391                                 .withValue(createOuterEntries(listSize, "testing-values")).build())
 392                         .build();
 393
 394         assertEquals(expected, actual);
 395     }
 396
 397     @Test
 398     public void testDistributedData() throws Exception {
 399         initEmptyDatastore("config");
 400
 401         waitOnAsyncTask(
 402                 leaderShardFactory.createDistributedShard(TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
 403                 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 404
 405         waitOnAsyncTask(
 406                 leaderShardFactory.createDistributedShard(
 407                         new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, TestModel.OUTER_CONTAINER_PATH),
 408                         Lists.newArrayList(AbstractTest.MEMBER_NAME)),
 409                 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 410
 411         waitOnAsyncTask(
 412                 leaderShardFactory.createDistributedShard(
 413                         new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, TestModel.INNER_LIST_PATH),
 414                         Lists.newArrayList(AbstractTest.MEMBER_NAME)),
 415                 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 416
 417         waitOnAsyncTask(
 418                 leaderShardFactory.createDistributedShard(
 419                         new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, TestModel.JUNK_PATH),
 420                         Lists.newArrayList(AbstractTest.MEMBER_NAME)),
 421                 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 422
 423         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 424                 ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
 425         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 426                 ClusterUtils.getCleanShardName(TestModel.OUTER_CONTAINER_PATH));
 427         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 428                 ClusterUtils.getCleanShardName(TestModel.INNER_LIST_PATH));
 429         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 430                 ClusterUtils.getCleanShardName(TestModel.JUNK_PATH));
 431
 432     }
 433
 434     @Test
 435     public void testMultipleRegistrationsAtOnePrefix() throws Exception {
 436         initEmptyDatastore("config");
 437
 438         for (int i = 0; i < 10; i++) {
 439             LOG.debug("Round {}", i);
 440             final DistributedShardRegistration reg1 = waitOnAsyncTask(leaderShardFactory.createDistributedShard(
 441                     TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
 442                     DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 443
 444             leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 445                     ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
 446
 447             assertNotNull(findLocalShard(leaderDistributedDataStore.getActorContext(),
 448                     ClusterUtils.getCleanShardName(TestModel.TEST_PATH)));
 449
 450             waitOnAsyncTask(reg1.close(), DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 451
 452             waitUntilShardIsDown(leaderDistributedDataStore.getActorContext(),
 453                     ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
 454         }
 455     }
 456
 457     private static Collection<MapEntryNode> createOuterEntries(final int amount, final String valuePrefix) {
 458         final Collection<MapEntryNode> ret = new ArrayList<>();
 459         for (int i = 0; i < amount; i++) {
 460             ret.add(ImmutableNodes.mapEntryBuilder()
 461                     .withNodeIdentifier(new NodeIdentifierWithPredicates(TestModel.OUTER_LIST_QNAME,
 462                             QName.create(TestModel.OUTER_LIST_QNAME, "id"), i))
 463                     .withChild(ImmutableNodes
 464                             .leafNode(QName.create(TestModel.OUTER_LIST_QNAME, "id"), i))
 465                     .withChild(createWholeInnerList(amount, "outer id: " + i + " " + valuePrefix))
 466                     .build());
 467         }
 468
 469         return ret;
 470     }
 471
 472     private static MapNode createWholeInnerList(final int amount, final String valuePrefix) {
 473         return ImmutableMapNodeBuilder.create().withNodeIdentifier(new NodeIdentifier(TestModel.INNER_LIST_QNAME))
 474                 .withValue(createInnerListMapEntries(amount, valuePrefix)).build();
 475     }
 476
 477     private static Collection<MapEntryNode> createInnerListMapEntries(final int amount, final String valuePrefix) {
 478         final Collection<MapEntryNode> ret = new ArrayList<>();
 479         for (int i = 0; i < amount; i++) {
 480             ret.add(ImmutableNodes.mapEntryBuilder()
 481                     .withNodeIdentifier(new NodeIdentifierWithPredicates(TestModel.INNER_LIST_QNAME,
 482                             QName.create(TestModel.INNER_LIST_QNAME, "name"), Integer.toString(i)))
 483                     .withChild(ImmutableNodes
 484                             .leafNode(QName.create(TestModel.INNER_LIST_QNAME, "value"), valuePrefix + "-" + i))
 485                     .build());
 486         }
 487
 488         return ret;
 489     }
 490
 491     private static YangInstanceIdentifier getOuterListIdFor(final int id) {
 492         return TestModel.OUTER_LIST_PATH.node(new NodeIdentifierWithPredicates(
 493                 TestModel.OUTER_LIST_QNAME, QName.create(TestModel.OUTER_LIST_QNAME, "id"), id));
 494     }
 495 }