opendaylight/md-sal/sal-distributed-datastore/src/test/java/org/opendaylight/controller/cluster/datastore/IntegrationTestKit.java

   1 /*
   2  * Copyright (c) 2015 Brocade Communications Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.controller.cluster.datastore;
   9
  10 import static org.junit.Assert.assertEquals;
  11 import static org.junit.Assert.assertNotNull;
  12 import static org.junit.Assert.assertTrue;
  13 import static org.junit.Assert.fail;
  14
  15 import akka.actor.ActorRef;
  16 import akka.actor.ActorSystem;
  17 import akka.cluster.Cluster;
  18 import akka.cluster.ClusterEvent.CurrentClusterState;
  19 import akka.cluster.Member;
  20 import akka.cluster.MemberStatus;
  21 import com.google.common.base.Stopwatch;
  22 import com.google.common.collect.Sets;
  23 import com.google.common.util.concurrent.ListenableFuture;
  24 import com.google.common.util.concurrent.Uninterruptibles;
  25 import java.lang.reflect.Constructor;
  26 import java.util.Optional;
  27 import java.util.Set;
  28 import java.util.concurrent.Callable;
  29 import java.util.concurrent.TimeUnit;
  30 import java.util.function.Consumer;
  31 import org.mockito.Mockito;
  32 import org.opendaylight.controller.cluster.databroker.ClientBackedDataStore;
  33 import org.opendaylight.controller.cluster.datastore.DatastoreContext.Builder;
  34 import org.opendaylight.controller.cluster.datastore.config.Configuration;
  35 import org.opendaylight.controller.cluster.datastore.config.ConfigurationImpl;
  36 import org.opendaylight.controller.cluster.datastore.config.EmptyModuleShardConfigProvider;
  37 import org.opendaylight.controller.cluster.datastore.messages.OnDemandShardState;
  38 import org.opendaylight.controller.cluster.datastore.persisted.DatastoreSnapshot;
  39 import org.opendaylight.controller.cluster.datastore.utils.ActorUtils;
  40 import org.opendaylight.controller.cluster.raft.client.messages.GetOnDemandRaftState;
  41 import org.opendaylight.controller.md.cluster.datastore.model.SchemaContextHelper;
  42 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  43 import org.opendaylight.mdsal.dom.spi.store.DOMStoreReadTransaction;
  44 import org.opendaylight.mdsal.dom.spi.store.DOMStoreThreePhaseCommitCohort;
  45 import org.opendaylight.mdsal.dom.spi.store.DOMStoreTransactionChain;
  46 import org.opendaylight.mdsal.dom.spi.store.DOMStoreWriteTransaction;
  47 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
  48 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
  49 import org.opendaylight.yangtools.yang.model.api.EffectiveModelContext;
  50 import org.slf4j.Logger;
  51 import org.slf4j.LoggerFactory;
  52 import scala.concurrent.Await;
  53 import scala.concurrent.Future;
  54 import scala.concurrent.duration.FiniteDuration;
  55
  56 public class IntegrationTestKit extends ShardTestKit {
  57
  58     private static final Logger LOG = LoggerFactory.getLogger(IntegrationTestKit.class);
  59
  60     protected DatastoreContext.Builder datastoreContextBuilder;
  61     protected DatastoreSnapshot restoreFromSnapshot;
  62     private final int commitTimeout;
  63
  64     public IntegrationTestKit(final ActorSystem actorSystem, final Builder datastoreContextBuilder) {
  65         this(actorSystem, datastoreContextBuilder, 7);
  66     }
  67
  68     public IntegrationTestKit(final ActorSystem actorSystem, final Builder datastoreContextBuilder,
  69             final int commitTimeout) {
  70         super(actorSystem);
  71         this.datastoreContextBuilder = datastoreContextBuilder;
  72         this.commitTimeout = commitTimeout;
  73     }
  74
  75     public DatastoreContext.Builder getDatastoreContextBuilder() {
  76         return datastoreContextBuilder;
  77     }
  78
  79     public DistributedDataStore setupDistributedDataStore(final String typeName, final String moduleShardsConfig,
  80                                                           final boolean waitUntilLeader,
  81                                                           final EffectiveModelContext schemaContext) throws Exception {
  82         return setupDistributedDataStore(typeName, moduleShardsConfig, "modules.conf", waitUntilLeader, schemaContext);
  83     }
  84
  85     public DistributedDataStore setupDistributedDataStore(final String typeName, final String moduleShardsConfig,
  86                                                           final String modulesConfig,
  87                                                           final boolean waitUntilLeader,
  88                                                           final EffectiveModelContext schemaContext,
  89                                                           final String... shardNames) throws Exception {
  90         return (DistributedDataStore) setupAbstractDataStore(DistributedDataStore.class, typeName, moduleShardsConfig,
  91                 modulesConfig, waitUntilLeader, schemaContext, shardNames);
  92     }
  93
  94     public AbstractDataStore setupAbstractDataStore(final Class<? extends AbstractDataStore> implementation,
  95                                                     final String typeName, final String... shardNames)
  96             throws Exception {
  97         return setupAbstractDataStore(implementation, typeName, "module-shards.conf", true,
  98                 SchemaContextHelper.full(), shardNames);
  99     }
 100
 101     public AbstractDataStore setupAbstractDataStore(final Class<? extends AbstractDataStore> implementation,
 102                                                     final String typeName, final boolean waitUntilLeader,
 103                                                     final String... shardNames) throws Exception {
 104         return setupAbstractDataStore(implementation, typeName, "module-shards.conf", waitUntilLeader,
 105                 SchemaContextHelper.full(), shardNames);
 106     }
 107
 108     public AbstractDataStore setupAbstractDataStore(final Class<? extends AbstractDataStore> implementation,
 109                                                     final String typeName, final String moduleShardsConfig,
 110                                                     final boolean waitUntilLeader, final String... shardNames)
 111             throws Exception {
 112         return setupAbstractDataStore(implementation, typeName, moduleShardsConfig, waitUntilLeader,
 113                 SchemaContextHelper.full(), shardNames);
 114     }
 115
 116     public AbstractDataStore setupAbstractDataStore(final Class<? extends AbstractDataStore> implementation,
 117                                                     final String typeName, final String moduleShardsConfig,
 118                                                     final boolean waitUntilLeader,
 119                                                     final EffectiveModelContext schemaContext,
 120                                                     final String... shardNames) throws Exception {
 121         return setupAbstractDataStore(implementation, typeName, moduleShardsConfig, "modules.conf", waitUntilLeader,
 122                 schemaContext, shardNames);
 123     }
 124
 125     private AbstractDataStore setupAbstractDataStore(final Class<? extends AbstractDataStore> implementation,
 126                                                      final String typeName, final String moduleShardsConfig,
 127                                                      final String modulesConfig, final boolean waitUntilLeader,
 128                                                      final EffectiveModelContext schemaContext,
 129                                                      final String... shardNames)
 130             throws Exception {
 131         final ClusterWrapper cluster = new ClusterWrapperImpl(getSystem());
 132         final Configuration config = new ConfigurationImpl(moduleShardsConfig, modulesConfig);
 133
 134         setDataStoreName(typeName);
 135
 136         // Make sure we set up datastore context correctly
 137         datastoreContextBuilder.useTellBasedProtocol(ClientBackedDataStore.class.isAssignableFrom(implementation));
 138
 139         final DatastoreContext datastoreContext = datastoreContextBuilder.build();
 140         final DatastoreContextFactory mockContextFactory = Mockito.mock(DatastoreContextFactory.class);
 141         Mockito.doReturn(datastoreContext).when(mockContextFactory).getBaseDatastoreContext();
 142         Mockito.doReturn(datastoreContext).when(mockContextFactory).getShardDatastoreContext(Mockito.anyString());
 143
 144         final Constructor<? extends AbstractDataStore> constructor = implementation.getDeclaredConstructor(
 145                 ActorSystem.class, ClusterWrapper.class, Configuration.class,
 146                 DatastoreContextFactory.class, DatastoreSnapshot.class);
 147
 148         final AbstractDataStore dataStore = constructor.newInstance(getSystem(), cluster, config, mockContextFactory,
 149             restoreFromSnapshot);
 150
 151         dataStore.onModelContextUpdated(schemaContext);
 152
 153         if (waitUntilLeader) {
 154             waitUntilLeader(dataStore.getActorUtils(), shardNames);
 155         }
 156
 157         datastoreContextBuilder = DatastoreContext.newBuilderFrom(datastoreContext);
 158         return dataStore;
 159     }
 160
 161     private void setDataStoreName(final String typeName) {
 162         if ("config".equals(typeName)) {
 163             datastoreContextBuilder.logicalStoreType(LogicalDatastoreType.CONFIGURATION);
 164         } else if ("operational".equals(typeName)) {
 165             datastoreContextBuilder.logicalStoreType(LogicalDatastoreType.OPERATIONAL);
 166         } else {
 167             datastoreContextBuilder.dataStoreName(typeName);
 168         }
 169     }
 170
 171     public DistributedDataStore setupDistributedDataStoreWithoutConfig(final String typeName,
 172                                                                        final EffectiveModelContext schemaContext) {
 173         final ClusterWrapper cluster = new ClusterWrapperImpl(getSystem());
 174         final ConfigurationImpl configuration = new ConfigurationImpl(new EmptyModuleShardConfigProvider());
 175
 176         setDataStoreName(typeName);
 177
 178         final DatastoreContext datastoreContext = getDatastoreContextBuilder().build();
 179
 180         final DatastoreContextFactory mockContextFactory = Mockito.mock(DatastoreContextFactory.class);
 181         Mockito.doReturn(datastoreContext).when(mockContextFactory).getBaseDatastoreContext();
 182         Mockito.doReturn(datastoreContext).when(mockContextFactory).getShardDatastoreContext(Mockito.anyString());
 183
 184         final DistributedDataStore dataStore = new DistributedDataStore(getSystem(), cluster,
 185                 configuration, mockContextFactory, restoreFromSnapshot);
 186
 187         dataStore.onModelContextUpdated(schemaContext);
 188
 189         datastoreContextBuilder = DatastoreContext.newBuilderFrom(datastoreContext);
 190         return dataStore;
 191     }
 192
 193     public DistributedDataStore setupDistributedDataStoreWithoutConfig(final String typeName,
 194                                                                        final EffectiveModelContext schemaContext,
 195                                                                        final LogicalDatastoreType storeType) {
 196         final ClusterWrapper cluster = new ClusterWrapperImpl(getSystem());
 197         final ConfigurationImpl configuration = new ConfigurationImpl(new EmptyModuleShardConfigProvider());
 198
 199         setDataStoreName(typeName);
 200
 201         final DatastoreContext datastoreContext =
 202                 getDatastoreContextBuilder().logicalStoreType(storeType).build();
 203
 204         final DatastoreContextFactory mockContextFactory = Mockito.mock(DatastoreContextFactory.class);
 205         Mockito.doReturn(datastoreContext).when(mockContextFactory).getBaseDatastoreContext();
 206         Mockito.doReturn(datastoreContext).when(mockContextFactory).getShardDatastoreContext(Mockito.anyString());
 207
 208         final DistributedDataStore dataStore = new DistributedDataStore(getSystem(), cluster,
 209                 configuration, mockContextFactory, restoreFromSnapshot);
 210
 211         dataStore.onModelContextUpdated(schemaContext);
 212
 213         datastoreContextBuilder = DatastoreContext.newBuilderFrom(datastoreContext);
 214         return dataStore;
 215     }
 216
 217     public void waitUntilLeader(final ActorUtils actorUtils, final String... shardNames) {
 218         for (String shardName: shardNames) {
 219             ActorRef shard = findLocalShard(actorUtils, shardName);
 220
 221             assertNotNull("Shard was not created for " + shardName, shard);
 222
 223             waitUntilLeader(shard);
 224         }
 225     }
 226
 227     public void waitUntilNoLeader(final ActorUtils actorUtils, final String... shardNames) {
 228         for (String shardName: shardNames) {
 229             ActorRef shard = findLocalShard(actorUtils, shardName);
 230             assertNotNull("No local shard found for " + shardName, shard);
 231
 232             waitUntilNoLeader(shard);
 233         }
 234     }
 235
 236     public void waitForMembersUp(final String... otherMembers) {
 237         Set<String> otherMembersSet = Sets.newHashSet(otherMembers);
 238         Stopwatch sw = Stopwatch.createStarted();
 239         while (sw.elapsed(TimeUnit.SECONDS) <= 10) {
 240             CurrentClusterState state = Cluster.get(getSystem()).state();
 241             for (Member m: state.getMembers()) {
 242                 if (m.status() == MemberStatus.up() && otherMembersSet.remove(m.getRoles().iterator().next())
 243                         && otherMembersSet.isEmpty()) {
 244                     return;
 245                 }
 246             }
 247
 248             Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
 249         }
 250
 251         fail("Member(s) " + otherMembersSet + " are not Up");
 252     }
 253
 254     public static ActorRef findLocalShard(final ActorUtils actorUtils, final String shardName) {
 255         ActorRef shard = null;
 256         for (int i = 0; i < 20 * 5 && shard == null; i++) {
 257             Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
 258             Optional<ActorRef> shardReply = actorUtils.findLocalShard(shardName);
 259             if (shardReply.isPresent()) {
 260                 shard = shardReply.get();
 261             }
 262         }
 263         return shard;
 264     }
 265
 266     public static void waitUntilShardIsDown(final ActorUtils actorUtils, final String shardName) {
 267         for (int i = 0; i < 20 * 5 ; i++) {
 268             LOG.debug("Waiting for shard down {}", shardName);
 269             Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
 270             Optional<ActorRef> shardReply = actorUtils.findLocalShard(shardName);
 271             if (!shardReply.isPresent()) {
 272                 return;
 273             }
 274         }
 275
 276         throw new IllegalStateException("Shard[" + shardName + " did not shutdown in time");
 277     }
 278
 279     public static void verifyShardStats(final AbstractDataStore datastore, final String shardName,
 280             final ShardStatsVerifier verifier) throws Exception {
 281         ActorUtils actorUtils = datastore.getActorUtils();
 282
 283         Future<ActorRef> future = actorUtils.findLocalShardAsync(shardName);
 284         ActorRef shardActor = Await.result(future, FiniteDuration.create(10, TimeUnit.SECONDS));
 285
 286         AssertionError lastError = null;
 287         Stopwatch sw = Stopwatch.createStarted();
 288         while (sw.elapsed(TimeUnit.SECONDS) <= 5) {
 289             ShardStats shardStats = (ShardStats)actorUtils
 290                     .executeOperation(shardActor, Shard.GET_SHARD_MBEAN_MESSAGE);
 291
 292             try {
 293                 verifier.verify(shardStats);
 294                 return;
 295             } catch (AssertionError e) {
 296                 lastError = e;
 297                 Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
 298             }
 299         }
 300
 301         throw lastError;
 302     }
 303
 304     public static void verifyShardState(final AbstractDataStore datastore, final String shardName,
 305             final Consumer<OnDemandShardState> verifier) throws Exception {
 306         ActorUtils actorUtils = datastore.getActorUtils();
 307
 308         Future<ActorRef> future = actorUtils.findLocalShardAsync(shardName);
 309         ActorRef shardActor = Await.result(future, FiniteDuration.create(10, TimeUnit.SECONDS));
 310
 311         AssertionError lastError = null;
 312         Stopwatch sw = Stopwatch.createStarted();
 313         while (sw.elapsed(TimeUnit.SECONDS) <= 5) {
 314             OnDemandShardState shardState = (OnDemandShardState)actorUtils
 315                     .executeOperation(shardActor, GetOnDemandRaftState.INSTANCE);
 316
 317             try {
 318                 verifier.accept(shardState);
 319                 return;
 320             } catch (AssertionError e) {
 321                 lastError = e;
 322                 Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
 323             }
 324         }
 325
 326         throw lastError;
 327     }
 328
 329     void testWriteTransaction(final AbstractDataStore dataStore, final YangInstanceIdentifier nodePath,
 330             final NormalizedNode nodeToWrite) throws Exception {
 331
 332         // 1. Create a write-only Tx
 333
 334         DOMStoreWriteTransaction writeTx = dataStore.newWriteOnlyTransaction();
 335         assertNotNull("newWriteOnlyTransaction returned null", writeTx);
 336
 337         // 2. Write some data
 338
 339         writeTx.write(nodePath, nodeToWrite);
 340
 341         // 3. Ready the Tx for commit
 342
 343         DOMStoreThreePhaseCommitCohort cohort = writeTx.ready();
 344
 345         // 4. Commit the Tx
 346
 347         doCommit(cohort);
 348
 349         // 5. Verify the data in the store
 350
 351         DOMStoreReadTransaction readTx = dataStore.newReadOnlyTransaction();
 352
 353         Optional<NormalizedNode> optional = readTx.read(nodePath).get(5, TimeUnit.SECONDS);
 354         assertTrue("isPresent", optional.isPresent());
 355         assertEquals("Data node", nodeToWrite, optional.get());
 356     }
 357
 358     public void doCommit(final DOMStoreThreePhaseCommitCohort cohort) throws Exception {
 359         Boolean canCommit = cohort.canCommit().get(commitTimeout, TimeUnit.SECONDS);
 360         assertEquals("canCommit", Boolean.TRUE, canCommit);
 361         cohort.preCommit().get(5, TimeUnit.SECONDS);
 362         cohort.commit().get(5, TimeUnit.SECONDS);
 363     }
 364
 365     void doCommit(final ListenableFuture<Boolean> canCommitFuture, final DOMStoreThreePhaseCommitCohort cohort)
 366             throws Exception {
 367         Boolean canCommit = canCommitFuture.get(commitTimeout, TimeUnit.SECONDS);
 368         assertEquals("canCommit", Boolean.TRUE, canCommit);
 369         cohort.preCommit().get(5, TimeUnit.SECONDS);
 370         cohort.commit().get(5, TimeUnit.SECONDS);
 371     }
 372
 373     @SuppressWarnings("checkstyle:IllegalCatch")
 374     void assertExceptionOnCall(final Callable<Void> callable, final Class<? extends Exception> expType) {
 375         try {
 376             callable.call();
 377             fail("Expected " + expType.getSimpleName());
 378         } catch (Exception e) {
 379             assertEquals("Exception type", expType, e.getClass());
 380         }
 381     }
 382
 383     void assertExceptionOnTxChainCreates(final DOMStoreTransactionChain txChain,
 384             final Class<? extends Exception> expType) {
 385         assertExceptionOnCall(() -> {
 386             txChain.newWriteOnlyTransaction();
 387             return null;
 388         }, expType);
 389
 390         assertExceptionOnCall(() -> {
 391             txChain.newReadWriteTransaction();
 392             return null;
 393         }, expType);
 394
 395         assertExceptionOnCall(() -> {
 396             txChain.newReadOnlyTransaction();
 397             return null;
 398         }, expType);
 399     }
 400
 401     public interface ShardStatsVerifier {
 402         void verify(ShardStats stats);
 403     }
 404 }