opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/sharding/DistributedShardedDOMDataTreeTest.java

   1 /*
   2  * Copyright (c) 2016 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.controller.cluster.sharding;
  10
  11 import static org.junit.Assert.assertEquals;
  12 import static org.junit.Assert.assertNotNull;
  13 import static org.mockito.Matchers.anyCollection;
  14 import static org.mockito.Matchers.anyMap;
  15 import static org.mockito.Mockito.doNothing;
  16 import static org.mockito.Mockito.doReturn;
  17 import static org.mockito.Mockito.mock;
  18 import static org.mockito.Mockito.timeout;
  19 import static org.mockito.Mockito.verify;
  20 import static org.mockito.Mockito.verifyNoMoreInteractions;
  21 import static org.opendaylight.controller.cluster.datastore.IntegrationTestKit.findLocalShard;
  22 import static org.opendaylight.controller.cluster.datastore.IntegrationTestKit.waitUntilShardIsDown;
  23
  24 import akka.actor.ActorRef;
  25 import akka.actor.ActorSystem;
  26 import akka.actor.Address;
  27 import akka.actor.AddressFromURIString;
  28 import akka.cluster.Cluster;
  29 import akka.testkit.JavaTestKit;
  30 import com.google.common.base.Optional;
  31 import com.google.common.collect.Lists;
  32 import com.google.common.util.concurrent.CheckedFuture;
  33 import com.typesafe.config.ConfigFactory;
  34 import java.util.ArrayList;
  35 import java.util.Collection;
  36 import java.util.Collections;
  37 import java.util.List;
  38 import java.util.Map;
  39 import org.junit.After;
  40 import org.junit.Assert;
  41 import org.junit.Before;
  42 import org.junit.Ignore;
  43 import org.junit.Test;
  44 import org.mockito.ArgumentCaptor;
  45 import org.mockito.Captor;
  46 import org.mockito.Mockito;
  47 import org.mockito.MockitoAnnotations;
  48 import org.opendaylight.controller.cluster.ActorSystemProvider;
  49 import org.opendaylight.controller.cluster.datastore.AbstractTest;
  50 import org.opendaylight.controller.cluster.datastore.DatastoreContext;
  51 import org.opendaylight.controller.cluster.datastore.DatastoreContext.Builder;
  52 import org.opendaylight.controller.cluster.datastore.DistributedDataStore;
  53 import org.opendaylight.controller.cluster.datastore.IntegrationTestKit;
  54 import org.opendaylight.controller.cluster.datastore.utils.ClusterUtils;
  55 import org.opendaylight.controller.cluster.sharding.DistributedShardFactory.DistributedShardRegistration;
  56 import org.opendaylight.controller.md.cluster.datastore.model.SchemaContextHelper;
  57 import org.opendaylight.controller.md.cluster.datastore.model.TestModel;
  58 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  59 import org.opendaylight.mdsal.common.api.TransactionCommitFailedException;
  60 import org.opendaylight.mdsal.dom.api.DOMDataTreeCursorAwareTransaction;
  61 import org.opendaylight.mdsal.dom.api.DOMDataTreeIdentifier;
  62 import org.opendaylight.mdsal.dom.api.DOMDataTreeListener;
  63 import org.opendaylight.mdsal.dom.api.DOMDataTreeProducer;
  64 import org.opendaylight.mdsal.dom.api.DOMDataTreeWriteCursor;
  65 import org.opendaylight.yangtools.yang.common.QName;
  66 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
  67 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifier;
  68 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifierWithPredicates;
  69 import org.opendaylight.yangtools.yang.data.api.schema.ContainerNode;
  70 import org.opendaylight.yangtools.yang.data.api.schema.LeafNode;
  71 import org.opendaylight.yangtools.yang.data.api.schema.MapEntryNode;
  72 import org.opendaylight.yangtools.yang.data.api.schema.MapNode;
  73 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
  74 import org.opendaylight.yangtools.yang.data.api.schema.tree.DataTreeCandidate;
  75 import org.opendaylight.yangtools.yang.data.impl.schema.ImmutableNodes;
  76 import org.opendaylight.yangtools.yang.data.impl.schema.builder.impl.ImmutableContainerNodeBuilder;
  77 import org.opendaylight.yangtools.yang.data.impl.schema.builder.impl.ImmutableLeafNodeBuilder;
  78 import org.opendaylight.yangtools.yang.data.impl.schema.builder.impl.ImmutableMapNodeBuilder;
  79 import org.slf4j.Logger;
  80 import org.slf4j.LoggerFactory;
  81
  82 @Ignore("distributed-data is broken needs to be removed")
  83 public class DistributedShardedDOMDataTreeTest extends AbstractTest {
  84
  85     private static final Logger LOG = LoggerFactory.getLogger(DistributedShardedDOMDataTreeRemotingTest.class);
  86
  87     private static final Address MEMBER_1_ADDRESS =
  88             AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558");
  89
  90     private static final DOMDataTreeIdentifier TEST_ID =
  91             new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, TestModel.TEST_PATH);
  92
  93     private static final DOMDataTreeIdentifier INNER_LIST_ID =
  94             new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION,
  95                     YangInstanceIdentifier.create(getOuterListIdFor(0).getPathArguments())
  96                             .node(TestModel.INNER_LIST_QNAME));
  97
  98     private ActorSystem leaderSystem;
  99
 100     private final Builder leaderDatastoreContextBuilder =
 101             DatastoreContext.newBuilder()
 102                     .shardHeartbeatIntervalInMillis(100)
 103                     .shardElectionTimeoutFactor(2)
 104                     .logicalStoreType(
 105                             org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType.CONFIGURATION);
 106
 107     private DistributedDataStore leaderDistributedDataStore;
 108     private IntegrationTestKit leaderTestKit;
 109
 110     private DistributedShardedDOMDataTree leaderShardFactory;
 111
 112     @Captor
 113     private ArgumentCaptor<Collection<DataTreeCandidate>> captorForChanges;
 114     @Captor
 115     private ArgumentCaptor<Map<DOMDataTreeIdentifier, NormalizedNode<?, ?>>> captorForSubtrees;
 116
 117     private ActorSystemProvider leaderSystemProvider;
 118
 119     @Before
 120     public void setUp() {
 121         MockitoAnnotations.initMocks(this);
 122
 123         leaderSystem = ActorSystem.create("cluster-test", ConfigFactory.load().getConfig("Member1"));
 124         Cluster.get(leaderSystem).join(MEMBER_1_ADDRESS);
 125
 126         leaderSystemProvider = Mockito.mock(ActorSystemProvider.class);
 127         doReturn(leaderSystem).when(leaderSystemProvider).getActorSystem();
 128     }
 129
 130     @After
 131     public void tearDown() {
 132         if (leaderDistributedDataStore != null) {
 133             leaderDistributedDataStore.close();
 134         }
 135
 136         JavaTestKit.shutdownActorSystem(leaderSystem);
 137     }
 138
 139     private void initEmptyDatastore(final String type) {
 140         leaderTestKit = new IntegrationTestKit(leaderSystem, leaderDatastoreContextBuilder);
 141
 142         leaderDistributedDataStore =
 143                 leaderTestKit.setupDistributedDataStoreWithoutConfig(type, SchemaContextHelper.full());
 144
 145
 146         leaderShardFactory = new DistributedShardedDOMDataTree(leaderSystemProvider,
 147                 leaderDistributedDataStore,
 148                 leaderDistributedDataStore);
 149     }
 150
 151
 152     @Test
 153     public void testWritesIntoDefaultShard() throws Exception {
 154         initEmptyDatastore("config");
 155
 156         final DOMDataTreeIdentifier configRoot =
 157                 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, YangInstanceIdentifier.EMPTY);
 158
 159         final DOMDataTreeProducer producer = leaderShardFactory.createProducer(Collections.singleton(configRoot));
 160
 161         final DOMDataTreeCursorAwareTransaction tx = producer.createTransaction(true);
 162         final DOMDataTreeWriteCursor cursor =
 163                 tx.createCursor(new DOMDataTreeIdentifier(
 164                         LogicalDatastoreType.CONFIGURATION, YangInstanceIdentifier.EMPTY));
 165         Assert.assertNotNull(cursor);
 166
 167         final ContainerNode test =
 168                 ImmutableContainerNodeBuilder.create()
 169                         .withNodeIdentifier(new NodeIdentifier(TestModel.TEST_QNAME)).build();
 170
 171         cursor.write(test.getIdentifier(), test);
 172         cursor.close();
 173
 174         tx.submit().checkedGet();
 175     }
 176
 177     @Test
 178     public void testSingleNodeWrites() throws Exception {
 179         initEmptyDatastore("config");
 180
 181         final DistributedShardRegistration shardRegistration = waitOnAsyncTask(
 182                 leaderShardFactory.createDistributedShard(TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
 183                 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 184
 185         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 186                 ClusterUtils.getCleanShardName(TEST_ID.getRootIdentifier()));
 187
 188         final DOMDataTreeProducer producer = leaderShardFactory.createProducer(Collections.singleton(TEST_ID));
 189
 190         final DOMDataTreeCursorAwareTransaction tx = producer.createTransaction(true);
 191         final DOMDataTreeWriteCursor cursor = tx.createCursor(TEST_ID);
 192         Assert.assertNotNull(cursor);
 193         final YangInstanceIdentifier nameId =
 194                 YangInstanceIdentifier.builder(TestModel.TEST_PATH).node(TestModel.NAME_QNAME).build();
 195         final LeafNode<String> valueToCheck = ImmutableLeafNodeBuilder.<String>create().withNodeIdentifier(
 196                 new NodeIdentifier(TestModel.NAME_QNAME)).withValue("Test Value").build();
 197         cursor.write(nameId.getLastPathArgument(),
 198                 valueToCheck);
 199
 200         cursor.close();
 201         LOG.debug("Got to pre submit");
 202
 203         tx.submit().checkedGet();
 204
 205         final DOMDataTreeListener mockedDataTreeListener = mock(DOMDataTreeListener.class);
 206         doNothing().when(mockedDataTreeListener).onDataTreeChanged(anyCollection(), anyMap());
 207
 208         leaderShardFactory.registerListener(mockedDataTreeListener, Collections.singletonList(TEST_ID),
 209                 true, Collections.emptyList());
 210
 211         verify(mockedDataTreeListener, timeout(1000).times(1)).onDataTreeChanged(captorForChanges.capture(),
 212                 captorForSubtrees.capture());
 213         final List<Collection<DataTreeCandidate>> capturedValue = captorForChanges.getAllValues();
 214
 215         final Optional<NormalizedNode<?, ?>> dataAfter =
 216                 capturedValue.get(0).iterator().next().getRootNode().getDataAfter();
 217
 218         final NormalizedNode<?,?> expected = ImmutableContainerNodeBuilder.create()
 219                 .withNodeIdentifier(new NodeIdentifier(TestModel.TEST_QNAME)).withChild(valueToCheck).build();
 220         assertEquals(expected, dataAfter.get());
 221
 222         verifyNoMoreInteractions(mockedDataTreeListener);
 223
 224     }
 225
 226     @Test
 227     public void testMultipleWritesIntoSingleMapEntry() throws Exception {
 228         initEmptyDatastore("config");
 229
 230         final DistributedShardRegistration shardRegistration = waitOnAsyncTask(
 231                 leaderShardFactory.createDistributedShard(TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
 232                 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 233
 234         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 235                 ClusterUtils.getCleanShardName(TEST_ID.getRootIdentifier()));
 236
 237         LOG.warn("Got after waiting for nonleader");
 238         final ActorRef leaderShardManager = leaderDistributedDataStore.getActorContext().getShardManager();
 239
 240         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 241                 ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
 242
 243         final YangInstanceIdentifier oid1 = getOuterListIdFor(0);
 244         final DOMDataTreeIdentifier outerListPath = new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, oid1);
 245
 246         final DistributedShardRegistration outerListShardReg = waitOnAsyncTask(
 247                 leaderShardFactory.createDistributedShard(outerListPath, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
 248                 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 249
 250         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 251                 ClusterUtils.getCleanShardName(outerListPath.getRootIdentifier()));
 252
 253         final DOMDataTreeProducer shardProducer = leaderShardFactory.createProducer(
 254                 Collections.singletonList(outerListPath));
 255
 256         final DOMDataTreeCursorAwareTransaction tx = shardProducer.createTransaction(false);
 257         final DOMDataTreeWriteCursor cursor =
 258                 tx.createCursor(new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, oid1));
 259         assertNotNull(cursor);
 260
 261         MapNode innerList = ImmutableMapNodeBuilder
 262                 .create()
 263                 .withNodeIdentifier(new NodeIdentifier(TestModel.INNER_LIST_QNAME))
 264                 .build();
 265
 266         cursor.write(new NodeIdentifier(TestModel.INNER_LIST_QNAME), innerList);
 267         cursor.close();
 268         tx.submit().checkedGet();
 269
 270         final ArrayList<CheckedFuture<Void, TransactionCommitFailedException>> futures = new ArrayList<>();
 271         for (int i = 0; i < 1000; i++) {
 272             final Collection<MapEntryNode> innerListMapEntries = createInnerListMapEntries(1000, "run-" + i);
 273             for (final MapEntryNode innerListMapEntry : innerListMapEntries) {
 274                 final DOMDataTreeCursorAwareTransaction tx1 = shardProducer.createTransaction(false);
 275                 final DOMDataTreeWriteCursor cursor1 = tx1.createCursor(
 276                         new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION,
 277                                 oid1.node(new NodeIdentifier(TestModel.INNER_LIST_QNAME))));
 278                 cursor1.write(innerListMapEntry.getIdentifier(), innerListMapEntry);
 279                 cursor1.close();
 280                 futures.add(tx1.submit());
 281             }
 282         }
 283
 284         futures.get(futures.size() - 1).checkedGet();
 285
 286         final DOMDataTreeListener mockedDataTreeListener = mock(DOMDataTreeListener.class);
 287         doNothing().when(mockedDataTreeListener).onDataTreeChanged(anyCollection(), anyMap());
 288
 289         leaderShardFactory.registerListener(mockedDataTreeListener, Collections.singletonList(INNER_LIST_ID),
 290                 true, Collections.emptyList());
 291
 292         verify(mockedDataTreeListener, timeout(1000).times(1)).onDataTreeChanged(captorForChanges.capture(),
 293                 captorForSubtrees.capture());
 294         verifyNoMoreInteractions(mockedDataTreeListener);
 295         final List<Collection<DataTreeCandidate>> capturedValue = captorForChanges.getAllValues();
 296
 297         final NormalizedNode<?,?> expected =
 298                 ImmutableMapNodeBuilder
 299                         .create()
 300                         .withNodeIdentifier(new NodeIdentifier(TestModel.INNER_LIST_QNAME))
 301                                 // only the values from the last run should be present
 302                         .withValue(createInnerListMapEntries(1000, "run-999"))
 303                         .build();
 304
 305         assertEquals("List values dont match the expected values from the last run",
 306                 expected, capturedValue.get(0).iterator().next().getRootNode().getDataAfter().get());
 307
 308     }
 309
 310     private static Collection<MapEntryNode> createInnerListMapEntries(final int amount, final String valuePrefix) {
 311         final Collection<MapEntryNode> ret = new ArrayList<>();
 312         for (int i = 0; i < amount; i++) {
 313             ret.add(ImmutableNodes.mapEntryBuilder()
 314                     .withNodeIdentifier(new NodeIdentifierWithPredicates(TestModel.INNER_LIST_QNAME,
 315                             QName.create(TestModel.INNER_LIST_QNAME, "name"), Integer.toString(i)))
 316                     .withChild(ImmutableNodes
 317                             .leafNode(QName.create(TestModel.INNER_LIST_QNAME, "value"), valuePrefix + "-" + i))
 318                     .build());
 319         }
 320
 321         return ret;
 322     }
 323
 324     @Test
 325     public void testDistributedData() throws Exception {
 326         initEmptyDatastore("config");
 327
 328         waitOnAsyncTask(
 329                 leaderShardFactory.createDistributedShard(TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
 330                 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 331
 332         waitOnAsyncTask(
 333                 leaderShardFactory.createDistributedShard(
 334                         new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, TestModel.OUTER_CONTAINER_PATH),
 335                         Lists.newArrayList(AbstractTest.MEMBER_NAME)),
 336                 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 337
 338         waitOnAsyncTask(
 339                 leaderShardFactory.createDistributedShard(
 340                         new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, TestModel.INNER_LIST_PATH),
 341                         Lists.newArrayList(AbstractTest.MEMBER_NAME)),
 342                 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 343
 344         waitOnAsyncTask(
 345                 leaderShardFactory.createDistributedShard(
 346                         new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, TestModel.JUNK_PATH),
 347                         Lists.newArrayList(AbstractTest.MEMBER_NAME)),
 348                 DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 349
 350         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 351                 ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
 352         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 353                 ClusterUtils.getCleanShardName(TestModel.OUTER_CONTAINER_PATH));
 354         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 355                 ClusterUtils.getCleanShardName(TestModel.INNER_LIST_PATH));
 356         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 357                 ClusterUtils.getCleanShardName(TestModel.JUNK_PATH));
 358
 359     }
 360
 361     @Test
 362     public void testMultipleRegistrationsAtOnePrefix() throws Exception {
 363         initEmptyDatastore("config");
 364
 365         for (int i = 0; i < 10; i++) {
 366             LOG.debug("Round {}", i);
 367             final DistributedShardRegistration reg1 = waitOnAsyncTask(leaderShardFactory.createDistributedShard(
 368                     TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME)),
 369                     DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 370
 371             leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 372                     ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
 373
 374             assertNotNull(findLocalShard(leaderDistributedDataStore.getActorContext(),
 375                     ClusterUtils.getCleanShardName(TestModel.TEST_PATH)));
 376
 377             waitOnAsyncTask(reg1.close(), DistributedShardedDOMDataTree.SHARD_FUTURE_TIMEOUT_DURATION);
 378
 379             waitUntilShardIsDown(leaderDistributedDataStore.getActorContext(),
 380                     ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
 381         }
 382     }
 383
 384     private static YangInstanceIdentifier getOuterListIdFor(final int id) {
 385         return TestModel.OUTER_LIST_PATH.node(new NodeIdentifierWithPredicates(
 386                 TestModel.OUTER_LIST_QNAME, QName.create(TestModel.OUTER_LIST_QNAME, "id"), id));
 387     }
 388 }