opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/IntegrationTestKit.java

   1 /*
   2  * Copyright (c) 2015 Brocade Communications Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.controller.cluster.datastore;
   9
  10 import static org.junit.Assert.assertEquals;
  11 import static org.junit.Assert.assertNotNull;
  12 import static org.junit.Assert.fail;
  13
  14 import akka.actor.ActorRef;
  15 import akka.actor.ActorSystem;
  16 import akka.cluster.Cluster;
  17 import akka.cluster.ClusterEvent.CurrentClusterState;
  18 import akka.cluster.Member;
  19 import akka.cluster.MemberStatus;
  20 import com.google.common.base.Optional;
  21 import com.google.common.base.Stopwatch;
  22 import com.google.common.collect.Sets;
  23 import com.google.common.util.concurrent.ListenableFuture;
  24 import com.google.common.util.concurrent.Uninterruptibles;
  25 import java.lang.reflect.Constructor;
  26 import java.util.Set;
  27 import java.util.concurrent.Callable;
  28 import java.util.concurrent.TimeUnit;
  29 import java.util.function.Consumer;
  30 import org.mockito.Mockito;
  31 import org.opendaylight.controller.cluster.datastore.DatastoreContext.Builder;
  32 import org.opendaylight.controller.cluster.datastore.config.Configuration;
  33 import org.opendaylight.controller.cluster.datastore.config.ConfigurationImpl;
  34 import org.opendaylight.controller.cluster.datastore.config.EmptyModuleShardConfigProvider;
  35 import org.opendaylight.controller.cluster.datastore.jmx.mbeans.shard.ShardStats;
  36 import org.opendaylight.controller.cluster.datastore.messages.OnDemandShardState;
  37 import org.opendaylight.controller.cluster.datastore.persisted.DatastoreSnapshot;
  38 import org.opendaylight.controller.cluster.datastore.utils.ActorContext;
  39 import org.opendaylight.controller.cluster.raft.client.messages.GetOnDemandRaftState;
  40 import org.opendaylight.controller.md.cluster.datastore.model.SchemaContextHelper;
  41 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  42 import org.opendaylight.controller.sal.core.spi.data.DOMStoreReadTransaction;
  43 import org.opendaylight.controller.sal.core.spi.data.DOMStoreThreePhaseCommitCohort;
  44 import org.opendaylight.controller.sal.core.spi.data.DOMStoreTransactionChain;
  45 import org.opendaylight.controller.sal.core.spi.data.DOMStoreWriteTransaction;
  46 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
  47 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
  48 import org.opendaylight.yangtools.yang.model.api.SchemaContext;
  49 import org.slf4j.Logger;
  50 import org.slf4j.LoggerFactory;
  51 import scala.concurrent.Await;
  52 import scala.concurrent.Future;
  53 import scala.concurrent.duration.Duration;
  54
  55 public class IntegrationTestKit extends ShardTestKit {
  56
  57     private static final Logger LOG = LoggerFactory.getLogger(IntegrationTestKit.class);
  58
  59     protected DatastoreContext.Builder datastoreContextBuilder;
  60     protected DatastoreSnapshot restoreFromSnapshot;
  61
  62     public IntegrationTestKit(final ActorSystem actorSystem, final Builder datastoreContextBuilder) {
  63         super(actorSystem);
  64         this.datastoreContextBuilder = datastoreContextBuilder;
  65     }
  66
  67     public DatastoreContext.Builder getDatastoreContextBuilder() {
  68         return datastoreContextBuilder;
  69     }
  70
  71     public DistributedDataStore setupDistributedDataStore(final String typeName, final String moduleShardsConfig,
  72                                                           final boolean waitUntilLeader,
  73                                                           final SchemaContext schemaContext) throws Exception {
  74         return setupDistributedDataStore(typeName, moduleShardsConfig, "modules.conf", waitUntilLeader, schemaContext);
  75     }
  76
  77     public DistributedDataStore setupDistributedDataStore(final String typeName, final String moduleShardsConfig,
  78                                                           final String modulesConfig,
  79                                                           final boolean waitUntilLeader,
  80                                                           final SchemaContext schemaContext,
  81                                                           final String... shardNames) throws Exception {
  82         return (DistributedDataStore) setupAbstractDataStore(DistributedDataStore.class, typeName, moduleShardsConfig,
  83                 modulesConfig, waitUntilLeader, schemaContext, shardNames);
  84     }
  85
  86     public AbstractDataStore setupAbstractDataStore(final Class<? extends AbstractDataStore> implementation,
  87                                                     final String typeName, final String... shardNames)
  88             throws Exception {
  89         return setupAbstractDataStore(implementation, typeName, "module-shards.conf", true,
  90                 SchemaContextHelper.full(), shardNames);
  91     }
  92
  93     public AbstractDataStore setupAbstractDataStore(final Class<? extends AbstractDataStore> implementation,
  94                                                     final String typeName, final boolean waitUntilLeader,
  95                                                     final String... shardNames) throws Exception {
  96         return setupAbstractDataStore(implementation, typeName, "module-shards.conf", waitUntilLeader,
  97                 SchemaContextHelper.full(), shardNames);
  98     }
  99
 100     public AbstractDataStore setupAbstractDataStore(final Class<? extends AbstractDataStore> implementation,
 101                                                     final String typeName, final String moduleShardsConfig,
 102                                                     final boolean waitUntilLeader, final String... shardNames)
 103             throws Exception {
 104         return setupAbstractDataStore(implementation, typeName, moduleShardsConfig, waitUntilLeader,
 105                 SchemaContextHelper.full(), shardNames);
 106     }
 107
 108     public AbstractDataStore setupAbstractDataStore(final Class<? extends AbstractDataStore> implementation,
 109                                                     final String typeName, final String moduleShardsConfig,
 110                                                     final boolean waitUntilLeader,
 111                                                     final SchemaContext schemaContext,
 112                                                     final String... shardNames) throws Exception {
 113         return setupAbstractDataStore(implementation, typeName, moduleShardsConfig, "modules.conf", waitUntilLeader,
 114                 schemaContext, shardNames);
 115     }
 116
 117     private AbstractDataStore setupAbstractDataStore(final Class<? extends AbstractDataStore> implementation,
 118                                                      final String typeName, final String moduleShardsConfig,
 119                                                      final String modulesConfig, final boolean waitUntilLeader,
 120                                                      final SchemaContext schemaContext, final String... shardNames)
 121             throws Exception {
 122         final ClusterWrapper cluster = new ClusterWrapperImpl(getSystem());
 123         final Configuration config = new ConfigurationImpl(moduleShardsConfig, modulesConfig);
 124
 125         datastoreContextBuilder.dataStoreName(typeName);
 126
 127         final DatastoreContext datastoreContext = datastoreContextBuilder.build();
 128         final DatastoreContextFactory mockContextFactory = Mockito.mock(DatastoreContextFactory.class);
 129         Mockito.doReturn(datastoreContext).when(mockContextFactory).getBaseDatastoreContext();
 130         Mockito.doReturn(datastoreContext).when(mockContextFactory).getShardDatastoreContext(Mockito.anyString());
 131
 132         final Constructor constructor = implementation.getDeclaredConstructor(
 133                 ActorSystem.class, ClusterWrapper.class, Configuration.class,
 134                 DatastoreContextFactory.class, DatastoreSnapshot.class);
 135
 136         final AbstractDataStore dataStore = (AbstractDataStore) constructor.newInstance(
 137                 getSystem(), cluster, config, mockContextFactory, restoreFromSnapshot);
 138
 139         dataStore.onGlobalContextUpdated(schemaContext);
 140
 141         if (waitUntilLeader) {
 142             waitUntilLeader(dataStore.getActorContext(), shardNames);
 143         }
 144
 145         datastoreContextBuilder = DatastoreContext.newBuilderFrom(datastoreContext);
 146         return dataStore;
 147     }
 148
 149     public DistributedDataStore setupDistributedDataStoreWithoutConfig(final String typeName,
 150                                                                        final SchemaContext schemaContext) {
 151         final ClusterWrapper cluster = new ClusterWrapperImpl(getSystem());
 152         final ConfigurationImpl configuration = new ConfigurationImpl(new EmptyModuleShardConfigProvider());
 153
 154         getDatastoreContextBuilder().dataStoreName(typeName);
 155
 156         final DatastoreContext datastoreContext = getDatastoreContextBuilder().build();
 157
 158         final DatastoreContextFactory mockContextFactory = Mockito.mock(DatastoreContextFactory.class);
 159         Mockito.doReturn(datastoreContext).when(mockContextFactory).getBaseDatastoreContext();
 160         Mockito.doReturn(datastoreContext).when(mockContextFactory).getShardDatastoreContext(Mockito.anyString());
 161
 162         final DistributedDataStore dataStore = new DistributedDataStore(getSystem(), cluster,
 163                 configuration, mockContextFactory, restoreFromSnapshot);
 164
 165         dataStore.onGlobalContextUpdated(schemaContext);
 166
 167         datastoreContextBuilder = DatastoreContext.newBuilderFrom(datastoreContext);
 168         return dataStore;
 169     }
 170
 171     public DistributedDataStore setupDistributedDataStoreWithoutConfig(final String typeName,
 172                                                                        final SchemaContext schemaContext,
 173                                                                        final LogicalDatastoreType storeType) {
 174         final ClusterWrapper cluster = new ClusterWrapperImpl(getSystem());
 175         final ConfigurationImpl configuration = new ConfigurationImpl(new EmptyModuleShardConfigProvider());
 176
 177         getDatastoreContextBuilder().dataStoreName(typeName);
 178
 179         final DatastoreContext datastoreContext =
 180                 getDatastoreContextBuilder().logicalStoreType(storeType).build();
 181
 182         final DatastoreContextFactory mockContextFactory = Mockito.mock(DatastoreContextFactory.class);
 183         Mockito.doReturn(datastoreContext).when(mockContextFactory).getBaseDatastoreContext();
 184         Mockito.doReturn(datastoreContext).when(mockContextFactory).getShardDatastoreContext(Mockito.anyString());
 185
 186         final DistributedDataStore dataStore = new DistributedDataStore(getSystem(), cluster,
 187                 configuration, mockContextFactory, restoreFromSnapshot);
 188
 189         dataStore.onGlobalContextUpdated(schemaContext);
 190
 191         datastoreContextBuilder = DatastoreContext.newBuilderFrom(datastoreContext);
 192         return dataStore;
 193     }
 194
 195     public void waitUntilLeader(final ActorContext actorContext, final String... shardNames) {
 196         for (String shardName: shardNames) {
 197             ActorRef shard = findLocalShard(actorContext, shardName);
 198
 199             assertNotNull("Shard was not created for " + shardName, shard);
 200
 201             waitUntilLeader(shard);
 202         }
 203     }
 204
 205     public void waitUntilNoLeader(final ActorContext actorContext, final String... shardNames) {
 206         for (String shardName: shardNames) {
 207             ActorRef shard = findLocalShard(actorContext, shardName);
 208             assertNotNull("No local shard found for " + shardName, shard);
 209
 210             waitUntilNoLeader(shard);
 211         }
 212     }
 213
 214     public void waitForMembersUp(final String... otherMembers) {
 215         Set<String> otherMembersSet = Sets.newHashSet(otherMembers);
 216         Stopwatch sw = Stopwatch.createStarted();
 217         while (sw.elapsed(TimeUnit.SECONDS) <= 10) {
 218             CurrentClusterState state = Cluster.get(getSystem()).state();
 219             for (Member m: state.getMembers()) {
 220                 if (m.status() == MemberStatus.up() && otherMembersSet.remove(m.getRoles().iterator().next())
 221                         && otherMembersSet.isEmpty()) {
 222                     return;
 223                 }
 224             }
 225
 226             Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
 227         }
 228
 229         fail("Member(s) " + otherMembersSet + " are not Up");
 230     }
 231
 232     public static ActorRef findLocalShard(final ActorContext actorContext, final String shardName) {
 233         ActorRef shard = null;
 234         for (int i = 0; i < 20 * 5 && shard == null; i++) {
 235             Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
 236             Optional<ActorRef> shardReply = actorContext.findLocalShard(shardName);
 237             if (shardReply.isPresent()) {
 238                 shard = shardReply.get();
 239             }
 240         }
 241         return shard;
 242     }
 243
 244     public static void waitUntilShardIsDown(final ActorContext actorContext, final String shardName) {
 245         for (int i = 0; i < 20 * 5 ; i++) {
 246             LOG.debug("Waiting for shard down {}", shardName);
 247             Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
 248             Optional<ActorRef> shardReply = actorContext.findLocalShard(shardName);
 249             if (!shardReply.isPresent()) {
 250                 return;
 251             }
 252         }
 253
 254         throw new IllegalStateException("Shard[" + shardName + " did not shutdown in time");
 255     }
 256
 257     public static void verifyShardStats(final AbstractDataStore datastore, final String shardName,
 258             final ShardStatsVerifier verifier) throws Exception {
 259         ActorContext actorContext = datastore.getActorContext();
 260
 261         Future<ActorRef> future = actorContext.findLocalShardAsync(shardName);
 262         ActorRef shardActor = Await.result(future, Duration.create(10, TimeUnit.SECONDS));
 263
 264         AssertionError lastError = null;
 265         Stopwatch sw = Stopwatch.createStarted();
 266         while (sw.elapsed(TimeUnit.SECONDS) <= 5) {
 267             ShardStats shardStats = (ShardStats)actorContext
 268                     .executeOperation(shardActor, Shard.GET_SHARD_MBEAN_MESSAGE);
 269
 270             try {
 271                 verifier.verify(shardStats);
 272                 return;
 273             } catch (AssertionError e) {
 274                 lastError = e;
 275                 Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
 276             }
 277         }
 278
 279         throw lastError;
 280     }
 281
 282     public static void verifyShardState(final AbstractDataStore datastore, final String shardName,
 283             final Consumer<OnDemandShardState> verifier) throws Exception {
 284         ActorContext actorContext = datastore.getActorContext();
 285
 286         Future<ActorRef> future = actorContext.findLocalShardAsync(shardName);
 287         ActorRef shardActor = Await.result(future, Duration.create(10, TimeUnit.SECONDS));
 288
 289         AssertionError lastError = null;
 290         Stopwatch sw = Stopwatch.createStarted();
 291         while (sw.elapsed(TimeUnit.SECONDS) <= 5) {
 292             OnDemandShardState shardState = (OnDemandShardState)actorContext
 293                     .executeOperation(shardActor, GetOnDemandRaftState.INSTANCE);
 294
 295             try {
 296                 verifier.accept(shardState);
 297                 return;
 298             } catch (AssertionError e) {
 299                 lastError = e;
 300                 Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
 301             }
 302         }
 303
 304         throw lastError;
 305     }
 306
 307     void testWriteTransaction(final AbstractDataStore dataStore, final YangInstanceIdentifier nodePath,
 308             final NormalizedNode<?, ?> nodeToWrite) throws Exception {
 309
 310         // 1. Create a write-only Tx
 311
 312         DOMStoreWriteTransaction writeTx = dataStore.newWriteOnlyTransaction();
 313         assertNotNull("newWriteOnlyTransaction returned null", writeTx);
 314
 315         // 2. Write some data
 316
 317         writeTx.write(nodePath, nodeToWrite);
 318
 319         // 3. Ready the Tx for commit
 320
 321         DOMStoreThreePhaseCommitCohort cohort = writeTx.ready();
 322
 323         // 4. Commit the Tx
 324
 325         doCommit(cohort);
 326
 327         // 5. Verify the data in the store
 328
 329         DOMStoreReadTransaction readTx = dataStore.newReadOnlyTransaction();
 330
 331         Optional<NormalizedNode<?, ?>> optional = readTx.read(nodePath).get(5, TimeUnit.SECONDS);
 332         assertEquals("isPresent", true, optional.isPresent());
 333         assertEquals("Data node", nodeToWrite, optional.get());
 334     }
 335
 336     public void doCommit(final DOMStoreThreePhaseCommitCohort cohort) throws Exception {
 337         Boolean canCommit = cohort.canCommit().get(7, TimeUnit.SECONDS);
 338         assertEquals("canCommit", true, canCommit);
 339         cohort.preCommit().get(5, TimeUnit.SECONDS);
 340         cohort.commit().get(5, TimeUnit.SECONDS);
 341     }
 342
 343     void doCommit(final ListenableFuture<Boolean> canCommitFuture, final DOMStoreThreePhaseCommitCohort cohort)
 344             throws Exception {
 345         Boolean canCommit = canCommitFuture.get(7, TimeUnit.SECONDS);
 346         assertEquals("canCommit", true, canCommit);
 347         cohort.preCommit().get(5, TimeUnit.SECONDS);
 348         cohort.commit().get(5, TimeUnit.SECONDS);
 349     }
 350
 351     @SuppressWarnings("checkstyle:IllegalCatch")
 352     void assertExceptionOnCall(final Callable<Void> callable, final Class<? extends Exception> expType)
 353             throws Exception {
 354         try {
 355             callable.call();
 356             fail("Expected " + expType.getSimpleName());
 357         } catch (Exception e) {
 358             assertEquals("Exception type", expType, e.getClass());
 359         }
 360     }
 361
 362     void assertExceptionOnTxChainCreates(final DOMStoreTransactionChain txChain,
 363             final Class<? extends Exception> expType) throws Exception {
 364         assertExceptionOnCall(() -> {
 365             txChain.newWriteOnlyTransaction();
 366             return null;
 367         }, expType);
 368
 369         assertExceptionOnCall(() -> {
 370             txChain.newReadWriteTransaction();
 371             return null;
 372         }, expType);
 373
 374         assertExceptionOnCall(() -> {
 375             txChain.newReadOnlyTransaction();
 376             return null;
 377         }, expType);
 378     }
 379
 380     public interface ShardStatsVerifier {
 381         void verify(ShardStats stats);
 382     }
 383 }