opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/sharding/DistributedShardedDOMDataTreeTest.java

   1 /*
   2  * Copyright (c) 2016 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.controller.cluster.sharding;
  10
  11 import static org.junit.Assert.assertNotNull;
  12 import static org.opendaylight.controller.cluster.datastore.IntegrationTestKit.findLocalShard;
  13 import static org.opendaylight.controller.cluster.datastore.IntegrationTestKit.waitUntilShardIsDown;
  14
  15 import akka.actor.ActorRef;
  16 import akka.actor.ActorSystem;
  17 import akka.actor.Address;
  18 import akka.actor.AddressFromURIString;
  19 import akka.cluster.Cluster;
  20 import akka.testkit.JavaTestKit;
  21 import com.google.common.collect.Lists;
  22 import com.google.common.util.concurrent.CheckedFuture;
  23 import com.typesafe.config.ConfigFactory;
  24 import java.util.ArrayList;
  25 import java.util.Collection;
  26 import java.util.Collections;
  27 import org.junit.After;
  28 import org.junit.Assert;
  29 import org.junit.Before;
  30 import org.junit.Ignore;
  31 import org.junit.Test;
  32 import org.opendaylight.controller.cluster.datastore.AbstractTest;
  33 import org.opendaylight.controller.cluster.datastore.DatastoreContext;
  34 import org.opendaylight.controller.cluster.datastore.DatastoreContext.Builder;
  35 import org.opendaylight.controller.cluster.datastore.DistributedDataStore;
  36 import org.opendaylight.controller.cluster.datastore.IntegrationTestKit;
  37 import org.opendaylight.controller.cluster.datastore.messages.FindLocalShard;
  38 import org.opendaylight.controller.cluster.datastore.messages.FindPrimary;
  39 import org.opendaylight.controller.cluster.datastore.messages.LocalPrimaryShardFound;
  40 import org.opendaylight.controller.cluster.datastore.messages.LocalShardFound;
  41 import org.opendaylight.controller.cluster.datastore.utils.ClusterUtils;
  42 import org.opendaylight.controller.cluster.sharding.DistributedShardFactory.DistributedShardRegistration;
  43 import org.opendaylight.controller.md.cluster.datastore.model.SchemaContextHelper;
  44 import org.opendaylight.controller.md.cluster.datastore.model.TestModel;
  45 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  46 import org.opendaylight.mdsal.common.api.TransactionCommitFailedException;
  47 import org.opendaylight.mdsal.dom.api.DOMDataTreeCursorAwareTransaction;
  48 import org.opendaylight.mdsal.dom.api.DOMDataTreeIdentifier;
  49 import org.opendaylight.mdsal.dom.api.DOMDataTreeProducer;
  50 import org.opendaylight.mdsal.dom.api.DOMDataTreeWriteCursor;
  51 import org.opendaylight.yangtools.yang.common.QName;
  52 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
  53 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifier;
  54 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier.NodeIdentifierWithPredicates;
  55 import org.opendaylight.yangtools.yang.data.api.schema.MapEntryNode;
  56 import org.opendaylight.yangtools.yang.data.api.schema.MapNode;
  57 import org.opendaylight.yangtools.yang.data.impl.schema.ImmutableNodes;
  58 import org.opendaylight.yangtools.yang.data.impl.schema.builder.impl.ImmutableLeafNodeBuilder;
  59 import org.opendaylight.yangtools.yang.data.impl.schema.builder.impl.ImmutableMapNodeBuilder;
  60 import org.slf4j.Logger;
  61 import org.slf4j.LoggerFactory;
  62
  63 @Ignore("distributed-data is broken needs to be removed")
  64 public class DistributedShardedDOMDataTreeTest extends AbstractTest {
  65
  66     private static final Logger LOG = LoggerFactory.getLogger(DistributedShardedDOMDataTreeRemotingTest.class);
  67
  68     private static final Address MEMBER_1_ADDRESS =
  69             AddressFromURIString.parse("akka.tcp://cluster-test@127.0.0.1:2558");
  70
  71     private static final DOMDataTreeIdentifier TEST_ID =
  72             new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, TestModel.TEST_PATH);
  73
  74     private ActorSystem leaderSystem;
  75
  76     private final Builder leaderDatastoreContextBuilder =
  77             DatastoreContext.newBuilder()
  78                     .shardHeartbeatIntervalInMillis(100)
  79                     .shardElectionTimeoutFactor(2)
  80                     .logicalStoreType(
  81                             org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType.CONFIGURATION);
  82
  83     private DistributedDataStore leaderDistributedDataStore;
  84     private IntegrationTestKit leaderTestKit;
  85
  86     private DistributedShardedDOMDataTree leaderShardFactory;
  87
  88     @Before
  89     public void setUp() {
  90
  91         leaderSystem = ActorSystem.create("cluster-test", ConfigFactory.load().getConfig("Member1"));
  92         Cluster.get(leaderSystem).join(MEMBER_1_ADDRESS);
  93
  94     }
  95
  96     @After
  97     public void tearDown() {
  98         if (leaderDistributedDataStore != null) {
  99             leaderDistributedDataStore.close();
 100         }
 101
 102         JavaTestKit.shutdownActorSystem(leaderSystem);
 103     }
 104
 105     private void initEmptyDatastore(final String type) {
 106         leaderTestKit = new IntegrationTestKit(leaderSystem, leaderDatastoreContextBuilder);
 107
 108         leaderDistributedDataStore =
 109                 leaderTestKit.setupDistributedDataStoreWithoutConfig(type, SchemaContextHelper.full());
 110
 111         leaderShardFactory = new DistributedShardedDOMDataTree(leaderSystem,
 112                 leaderDistributedDataStore,
 113                 leaderDistributedDataStore);
 114     }
 115
 116     @Test
 117     public void testWritesIntoDefaultShard() throws Exception {
 118         initEmptyDatastore("config");
 119
 120         leaderShardFactory.createDistributedShard(TEST_ID,
 121                 Lists.newArrayList(AbstractTest.MEMBER_NAME, AbstractTest.MEMBER_2_NAME));
 122
 123         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 124                 ClusterUtils.getCleanShardName(YangInstanceIdentifier.EMPTY));
 125
 126         final DOMDataTreeIdentifier configRoot =
 127                 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, YangInstanceIdentifier.EMPTY);
 128
 129         final DOMDataTreeProducer producer = leaderShardFactory.createProducer(Collections.singleton(configRoot));
 130
 131         final DOMDataTreeCursorAwareTransaction tx = producer.createTransaction(true);
 132         final DOMDataTreeWriteCursor cursor = tx.createCursor(TEST_ID);
 133         Assert.assertNotNull(cursor);
 134     }
 135
 136     @Test
 137     public void testSingleNodeWrites() throws Exception {
 138         initEmptyDatastore("config");
 139
 140         leaderShardFactory.createDistributedShard(TEST_ID,
 141                 Lists.newArrayList(AbstractTest.MEMBER_NAME, AbstractTest.MEMBER_2_NAME));
 142
 143         final DistributedShardRegistration shardRegistration =
 144                 leaderShardFactory.createDistributedShard(TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME));
 145         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 146                 ClusterUtils.getCleanShardName(TEST_ID.getRootIdentifier()));
 147
 148         LOG.warn("Got after waiting for nonleader");
 149         final ActorRef leaderShardManager = leaderDistributedDataStore.getActorContext().getShardManager();
 150
 151         new JavaTestKit(leaderSystem) {
 152             {
 153                 leaderShardManager.tell(
 154                         new FindLocalShard(ClusterUtils.getCleanShardName(TestModel.TEST_PATH), true), getRef());
 155                 expectMsgClass(duration("5 seconds"), LocalShardFound.class);
 156
 157                 leaderDistributedDataStore.getActorContext().getShardManager().tell(
 158                         new FindPrimary(ClusterUtils.getCleanShardName(TestModel.TEST_PATH), true), getRef());
 159                 expectMsgClass(duration("5 seconds"), LocalPrimaryShardFound.class);
 160             }
 161         };
 162
 163         final DOMDataTreeProducer producer = leaderShardFactory.createProducer(Collections.singleton(TEST_ID));
 164
 165         final DOMDataTreeCursorAwareTransaction tx = producer.createTransaction(true);
 166         final DOMDataTreeWriteCursor cursor = tx.createCursor(TEST_ID);
 167         Assert.assertNotNull(cursor);
 168         final YangInstanceIdentifier nameId =
 169                 YangInstanceIdentifier.builder(TestModel.TEST_PATH).node(TestModel.NAME_QNAME).build();
 170         cursor.write(nameId.getLastPathArgument(),
 171                 ImmutableLeafNodeBuilder.<String>create().withNodeIdentifier(
 172                         new NodeIdentifier(TestModel.NAME_QNAME)).withValue("Test Value").build());
 173
 174         cursor.close();
 175         LOG.warn("Got to pre submit");
 176
 177         tx.submit().checkedGet();
 178     }
 179
 180     @Test
 181     public void testMultipleWritesIntoSingleMapEntry() throws Exception {
 182         initEmptyDatastore("config");
 183
 184         final DistributedShardRegistration shardRegistration =
 185                 leaderShardFactory.createDistributedShard(TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME));
 186         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 187                 ClusterUtils.getCleanShardName(TEST_ID.getRootIdentifier()));
 188
 189         LOG.warn("Got after waiting for nonleader");
 190         final ActorRef leaderShardManager = leaderDistributedDataStore.getActorContext().getShardManager();
 191
 192         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 193                 ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
 194
 195         final YangInstanceIdentifier oid1 = TestModel.OUTER_LIST_PATH.node(new NodeIdentifierWithPredicates(
 196                 TestModel.OUTER_LIST_QNAME, QName.create(TestModel.OUTER_LIST_QNAME, "id"), 0));
 197         final DOMDataTreeIdentifier outerListPath = new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, oid1);
 198
 199         final DistributedShardRegistration outerListShardReg = leaderShardFactory.createDistributedShard(outerListPath,
 200                 Lists.newArrayList(AbstractTest.MEMBER_NAME));
 201
 202         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 203                 ClusterUtils.getCleanShardName(outerListPath.getRootIdentifier()));
 204
 205         final DOMDataTreeProducer shardProducer = leaderShardFactory.createProducer(
 206                 Collections.singletonList(outerListPath));
 207
 208         final DOMDataTreeCursorAwareTransaction tx = shardProducer.createTransaction(false);
 209         final DOMDataTreeWriteCursor cursor =
 210                 tx.createCursor(new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, oid1));
 211         assertNotNull(cursor);
 212
 213         MapNode innerList = ImmutableMapNodeBuilder
 214                 .create()
 215                 .withNodeIdentifier(new NodeIdentifier(TestModel.INNER_LIST_QNAME))
 216                 .build();
 217
 218         cursor.write(new NodeIdentifier(TestModel.INNER_LIST_QNAME), innerList);
 219         cursor.close();
 220         tx.submit().checkedGet();
 221
 222         final ArrayList<CheckedFuture<Void, TransactionCommitFailedException>> futures = new ArrayList<>();
 223         for (int i = 0; i < 1000; i++) {
 224             final Collection<MapEntryNode> innerListMapEntries = createInnerListMapEntries(1000, "run-" + i);
 225             for (final MapEntryNode innerListMapEntry : innerListMapEntries) {
 226                 final DOMDataTreeCursorAwareTransaction tx1 = shardProducer.createTransaction(false);
 227                 final DOMDataTreeWriteCursor cursor1 = tx1.createCursor(
 228                         new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION,
 229                                 oid1.node(new NodeIdentifier(TestModel.INNER_LIST_QNAME))));
 230                 cursor1.write(innerListMapEntry.getIdentifier(), innerListMapEntry);
 231                 cursor1.close();
 232                 futures.add(tx1.submit());
 233             }
 234         }
 235
 236         futures.get(futures.size() - 1).checkedGet();
 237
 238     }
 239
 240     private static Collection<MapEntryNode> createInnerListMapEntries(final int amount, final String valuePrefix) {
 241         final Collection<MapEntryNode> ret = new ArrayList<>();
 242         for (int i = 0; i < amount; i++) {
 243             ret.add(ImmutableNodes.mapEntryBuilder()
 244                     .withNodeIdentifier(new NodeIdentifierWithPredicates(TestModel.INNER_LIST_QNAME,
 245                             QName.create(TestModel.INNER_LIST_QNAME, "name"), Integer.toString(i)))
 246                     .withChild(ImmutableNodes
 247                             .leafNode(QName.create(TestModel.INNER_LIST_QNAME, "value"), valuePrefix + "-" + i))
 248                     .build());
 249         }
 250
 251         return ret;
 252     }
 253
 254     @Test
 255     public void testDistributedData() throws Exception {
 256         initEmptyDatastore("config");
 257
 258         leaderShardFactory.createDistributedShard(TEST_ID, Lists.newArrayList(AbstractTest.MEMBER_NAME));
 259         leaderShardFactory.createDistributedShard(
 260                 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, TestModel.OUTER_CONTAINER_PATH),
 261                 Lists.newArrayList(AbstractTest.MEMBER_NAME));
 262         leaderShardFactory.createDistributedShard(
 263                 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, TestModel.INNER_LIST_PATH),
 264                 Lists.newArrayList(AbstractTest.MEMBER_NAME));
 265         leaderShardFactory.createDistributedShard(
 266                 new DOMDataTreeIdentifier(LogicalDatastoreType.CONFIGURATION, TestModel.JUNK_PATH),
 267                 Lists.newArrayList(AbstractTest.MEMBER_NAME));
 268
 269         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 270                 ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
 271         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 272                 ClusterUtils.getCleanShardName(TestModel.OUTER_CONTAINER_PATH));
 273         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 274                 ClusterUtils.getCleanShardName(TestModel.INNER_LIST_PATH));
 275         leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 276                 ClusterUtils.getCleanShardName(TestModel.JUNK_PATH));
 277
 278     }
 279
 280     @Test
 281     public void testMultipleRegistrationsAtOnePrefix() throws Exception {
 282         initEmptyDatastore("config");
 283
 284         for (int i = 0; i < 10; i++) {
 285             LOG.debug("Round {}", i);
 286             final DistributedShardRegistration reg1 = leaderShardFactory
 287                     .createDistributedShard(TEST_ID,
 288                             Lists.newArrayList(AbstractTest.MEMBER_NAME));
 289
 290             leaderTestKit.waitUntilLeader(leaderDistributedDataStore.getActorContext(),
 291                     ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
 292
 293             assertNotNull(findLocalShard(leaderDistributedDataStore.getActorContext(),
 294                     ClusterUtils.getCleanShardName(TestModel.TEST_PATH)));
 295
 296             reg1.close();
 297
 298             waitUntilShardIsDown(leaderDistributedDataStore.getActorContext(),
 299                     ClusterUtils.getCleanShardName(TestModel.TEST_PATH));
 300
 301         }
 302     }
 303 }