2 * Copyright (c) 2015 Brocade Communications Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.controller.cluster.datastore;
10 import static org.junit.Assert.assertEquals;
11 import static org.junit.Assert.assertNotNull;
12 import static org.junit.Assert.assertTrue;
13 import static org.junit.Assert.fail;
15 import akka.actor.ActorRef;
16 import akka.actor.ActorSystem;
17 import akka.cluster.Cluster;
18 import akka.cluster.ClusterEvent.CurrentClusterState;
19 import akka.cluster.Member;
20 import akka.cluster.MemberStatus;
21 import com.google.common.base.Stopwatch;
22 import com.google.common.collect.Sets;
23 import com.google.common.util.concurrent.ListenableFuture;
24 import com.google.common.util.concurrent.Uninterruptibles;
25 import java.lang.reflect.Constructor;
26 import java.util.Optional;
28 import java.util.concurrent.Callable;
29 import java.util.concurrent.TimeUnit;
30 import java.util.function.Consumer;
31 import org.mockito.Mockito;
32 import org.opendaylight.controller.cluster.databroker.ClientBackedDataStore;
33 import org.opendaylight.controller.cluster.datastore.DatastoreContext.Builder;
34 import org.opendaylight.controller.cluster.datastore.config.Configuration;
35 import org.opendaylight.controller.cluster.datastore.config.ConfigurationImpl;
36 import org.opendaylight.controller.cluster.datastore.config.EmptyModuleShardConfigProvider;
37 import org.opendaylight.controller.cluster.datastore.messages.OnDemandShardState;
38 import org.opendaylight.controller.cluster.datastore.persisted.DatastoreSnapshot;
39 import org.opendaylight.controller.cluster.datastore.utils.ActorUtils;
40 import org.opendaylight.controller.cluster.raft.client.messages.GetOnDemandRaftState;
41 import org.opendaylight.controller.md.cluster.datastore.model.SchemaContextHelper;
42 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
43 import org.opendaylight.mdsal.dom.spi.store.DOMStoreReadTransaction;
44 import org.opendaylight.mdsal.dom.spi.store.DOMStoreThreePhaseCommitCohort;
45 import org.opendaylight.mdsal.dom.spi.store.DOMStoreTransactionChain;
46 import org.opendaylight.mdsal.dom.spi.store.DOMStoreWriteTransaction;
47 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
48 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
49 import org.opendaylight.yangtools.yang.model.api.EffectiveModelContext;
50 import org.slf4j.Logger;
51 import org.slf4j.LoggerFactory;
52 import scala.concurrent.Await;
53 import scala.concurrent.Future;
54 import scala.concurrent.duration.FiniteDuration;
56 public class IntegrationTestKit extends ShardTestKit {
58 private static final Logger LOG = LoggerFactory.getLogger(IntegrationTestKit.class);
60 protected DatastoreContext.Builder datastoreContextBuilder;
61 protected DatastoreSnapshot restoreFromSnapshot;
62 private final int commitTimeout;
64 public IntegrationTestKit(final ActorSystem actorSystem, final Builder datastoreContextBuilder) {
65 this(actorSystem, datastoreContextBuilder, 7);
68 public IntegrationTestKit(final ActorSystem actorSystem, final Builder datastoreContextBuilder,
69 final int commitTimeout) {
71 this.datastoreContextBuilder = datastoreContextBuilder;
72 this.commitTimeout = commitTimeout;
75 public DatastoreContext.Builder getDatastoreContextBuilder() {
76 return datastoreContextBuilder;
79 public DistributedDataStore setupDistributedDataStore(final String typeName, final String moduleShardsConfig,
80 final boolean waitUntilLeader,
81 final EffectiveModelContext schemaContext) throws Exception {
82 return setupDistributedDataStore(typeName, moduleShardsConfig, "modules.conf", waitUntilLeader, schemaContext);
85 public DistributedDataStore setupDistributedDataStore(final String typeName, final String moduleShardsConfig,
86 final String modulesConfig,
87 final boolean waitUntilLeader,
88 final EffectiveModelContext schemaContext,
89 final String... shardNames) throws Exception {
90 return (DistributedDataStore) setupAbstractDataStore(DistributedDataStore.class, typeName, moduleShardsConfig,
91 modulesConfig, waitUntilLeader, schemaContext, shardNames);
94 public AbstractDataStore setupAbstractDataStore(final Class<? extends AbstractDataStore> implementation,
95 final String typeName, final String... shardNames)
97 return setupAbstractDataStore(implementation, typeName, "module-shards.conf", true,
98 SchemaContextHelper.full(), shardNames);
101 public AbstractDataStore setupAbstractDataStore(final Class<? extends AbstractDataStore> implementation,
102 final String typeName, final boolean waitUntilLeader,
103 final String... shardNames) throws Exception {
104 return setupAbstractDataStore(implementation, typeName, "module-shards.conf", waitUntilLeader,
105 SchemaContextHelper.full(), shardNames);
108 public AbstractDataStore setupAbstractDataStore(final Class<? extends AbstractDataStore> implementation,
109 final String typeName, final String moduleShardsConfig,
110 final boolean waitUntilLeader, final String... shardNames)
112 return setupAbstractDataStore(implementation, typeName, moduleShardsConfig, waitUntilLeader,
113 SchemaContextHelper.full(), shardNames);
116 public AbstractDataStore setupAbstractDataStore(final Class<? extends AbstractDataStore> implementation,
117 final String typeName, final String moduleShardsConfig,
118 final boolean waitUntilLeader,
119 final EffectiveModelContext schemaContext,
120 final String... shardNames) throws Exception {
121 return setupAbstractDataStore(implementation, typeName, moduleShardsConfig, "modules.conf", waitUntilLeader,
122 schemaContext, shardNames);
125 private AbstractDataStore setupAbstractDataStore(final Class<? extends AbstractDataStore> implementation,
126 final String typeName, final String moduleShardsConfig,
127 final String modulesConfig, final boolean waitUntilLeader,
128 final EffectiveModelContext schemaContext,
129 final String... shardNames)
131 final ClusterWrapper cluster = new ClusterWrapperImpl(getSystem());
132 final Configuration config = new ConfigurationImpl(moduleShardsConfig, modulesConfig);
134 setDataStoreName(typeName);
136 // Make sure we set up datastore context correctly
137 datastoreContextBuilder.useTellBasedProtocol(ClientBackedDataStore.class.isAssignableFrom(implementation));
139 final DatastoreContext datastoreContext = datastoreContextBuilder.build();
140 final DatastoreContextFactory mockContextFactory = Mockito.mock(DatastoreContextFactory.class);
141 Mockito.doReturn(datastoreContext).when(mockContextFactory).getBaseDatastoreContext();
142 Mockito.doReturn(datastoreContext).when(mockContextFactory).getShardDatastoreContext(Mockito.anyString());
144 final Constructor<? extends AbstractDataStore> constructor = implementation.getDeclaredConstructor(
145 ActorSystem.class, ClusterWrapper.class, Configuration.class,
146 DatastoreContextFactory.class, DatastoreSnapshot.class);
148 final AbstractDataStore dataStore = constructor.newInstance(getSystem(), cluster, config, mockContextFactory,
149 restoreFromSnapshot);
151 dataStore.onModelContextUpdated(schemaContext);
153 if (waitUntilLeader) {
154 waitUntilLeader(dataStore.getActorUtils(), shardNames);
157 datastoreContextBuilder = DatastoreContext.newBuilderFrom(datastoreContext);
161 private void setDataStoreName(final String typeName) {
162 if ("config".equals(typeName)) {
163 datastoreContextBuilder.logicalStoreType(LogicalDatastoreType.CONFIGURATION);
164 } else if ("operational".equals(typeName)) {
165 datastoreContextBuilder.logicalStoreType(LogicalDatastoreType.OPERATIONAL);
167 datastoreContextBuilder.dataStoreName(typeName);
171 public DistributedDataStore setupDistributedDataStoreWithoutConfig(final String typeName,
172 final EffectiveModelContext schemaContext) {
173 final ClusterWrapper cluster = new ClusterWrapperImpl(getSystem());
174 final ConfigurationImpl configuration = new ConfigurationImpl(new EmptyModuleShardConfigProvider());
176 setDataStoreName(typeName);
178 final DatastoreContext datastoreContext = getDatastoreContextBuilder().build();
180 final DatastoreContextFactory mockContextFactory = Mockito.mock(DatastoreContextFactory.class);
181 Mockito.doReturn(datastoreContext).when(mockContextFactory).getBaseDatastoreContext();
182 Mockito.doReturn(datastoreContext).when(mockContextFactory).getShardDatastoreContext(Mockito.anyString());
184 final DistributedDataStore dataStore = new DistributedDataStore(getSystem(), cluster,
185 configuration, mockContextFactory, restoreFromSnapshot);
187 dataStore.onModelContextUpdated(schemaContext);
189 datastoreContextBuilder = DatastoreContext.newBuilderFrom(datastoreContext);
193 public DistributedDataStore setupDistributedDataStoreWithoutConfig(final String typeName,
194 final EffectiveModelContext schemaContext,
195 final LogicalDatastoreType storeType) {
196 final ClusterWrapper cluster = new ClusterWrapperImpl(getSystem());
197 final ConfigurationImpl configuration = new ConfigurationImpl(new EmptyModuleShardConfigProvider());
199 setDataStoreName(typeName);
201 final DatastoreContext datastoreContext =
202 getDatastoreContextBuilder().logicalStoreType(storeType).build();
204 final DatastoreContextFactory mockContextFactory = Mockito.mock(DatastoreContextFactory.class);
205 Mockito.doReturn(datastoreContext).when(mockContextFactory).getBaseDatastoreContext();
206 Mockito.doReturn(datastoreContext).when(mockContextFactory).getShardDatastoreContext(Mockito.anyString());
208 final DistributedDataStore dataStore = new DistributedDataStore(getSystem(), cluster,
209 configuration, mockContextFactory, restoreFromSnapshot);
211 dataStore.onModelContextUpdated(schemaContext);
213 datastoreContextBuilder = DatastoreContext.newBuilderFrom(datastoreContext);
217 public void waitUntilLeader(final ActorUtils actorUtils, final String... shardNames) {
218 for (String shardName: shardNames) {
219 ActorRef shard = findLocalShard(actorUtils, shardName);
221 assertNotNull("Shard was not created for " + shardName, shard);
223 waitUntilLeader(shard);
227 public void waitUntilNoLeader(final ActorUtils actorUtils, final String... shardNames) {
228 for (String shardName: shardNames) {
229 ActorRef shard = findLocalShard(actorUtils, shardName);
230 assertNotNull("No local shard found for " + shardName, shard);
232 waitUntilNoLeader(shard);
236 public void waitForMembersUp(final String... otherMembers) {
237 Set<String> otherMembersSet = Sets.newHashSet(otherMembers);
238 Stopwatch sw = Stopwatch.createStarted();
239 while (sw.elapsed(TimeUnit.SECONDS) <= 10) {
240 CurrentClusterState state = Cluster.get(getSystem()).state();
241 for (Member m: state.getMembers()) {
242 if (m.status() == MemberStatus.up() && otherMembersSet.remove(m.getRoles().iterator().next())
243 && otherMembersSet.isEmpty()) {
248 Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
251 fail("Member(s) " + otherMembersSet + " are not Up");
254 public static ActorRef findLocalShard(final ActorUtils actorUtils, final String shardName) {
255 ActorRef shard = null;
256 for (int i = 0; i < 20 * 5 && shard == null; i++) {
257 Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
258 Optional<ActorRef> shardReply = actorUtils.findLocalShard(shardName);
259 if (shardReply.isPresent()) {
260 shard = shardReply.get();
266 public static void waitUntilShardIsDown(final ActorUtils actorUtils, final String shardName) {
267 for (int i = 0; i < 20 * 5 ; i++) {
268 LOG.debug("Waiting for shard down {}", shardName);
269 Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
270 Optional<ActorRef> shardReply = actorUtils.findLocalShard(shardName);
271 if (!shardReply.isPresent()) {
276 throw new IllegalStateException("Shard[" + shardName + " did not shutdown in time");
279 public static void verifyShardStats(final AbstractDataStore datastore, final String shardName,
280 final ShardStatsVerifier verifier) throws Exception {
281 ActorUtils actorUtils = datastore.getActorUtils();
283 Future<ActorRef> future = actorUtils.findLocalShardAsync(shardName);
284 ActorRef shardActor = Await.result(future, FiniteDuration.create(10, TimeUnit.SECONDS));
286 AssertionError lastError = null;
287 Stopwatch sw = Stopwatch.createStarted();
288 while (sw.elapsed(TimeUnit.SECONDS) <= 5) {
289 ShardStats shardStats = (ShardStats)actorUtils
290 .executeOperation(shardActor, Shard.GET_SHARD_MBEAN_MESSAGE);
293 verifier.verify(shardStats);
295 } catch (AssertionError e) {
297 Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
304 public static void verifyShardState(final AbstractDataStore datastore, final String shardName,
305 final Consumer<OnDemandShardState> verifier) throws Exception {
306 ActorUtils actorUtils = datastore.getActorUtils();
308 Future<ActorRef> future = actorUtils.findLocalShardAsync(shardName);
309 ActorRef shardActor = Await.result(future, FiniteDuration.create(10, TimeUnit.SECONDS));
311 AssertionError lastError = null;
312 Stopwatch sw = Stopwatch.createStarted();
313 while (sw.elapsed(TimeUnit.SECONDS) <= 5) {
314 OnDemandShardState shardState = (OnDemandShardState)actorUtils
315 .executeOperation(shardActor, GetOnDemandRaftState.INSTANCE);
318 verifier.accept(shardState);
320 } catch (AssertionError e) {
322 Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
329 void testWriteTransaction(final AbstractDataStore dataStore, final YangInstanceIdentifier nodePath,
330 final NormalizedNode<?, ?> nodeToWrite) throws Exception {
332 // 1. Create a write-only Tx
334 DOMStoreWriteTransaction writeTx = dataStore.newWriteOnlyTransaction();
335 assertNotNull("newWriteOnlyTransaction returned null", writeTx);
337 // 2. Write some data
339 writeTx.write(nodePath, nodeToWrite);
341 // 3. Ready the Tx for commit
343 DOMStoreThreePhaseCommitCohort cohort = writeTx.ready();
349 // 5. Verify the data in the store
351 DOMStoreReadTransaction readTx = dataStore.newReadOnlyTransaction();
353 Optional<NormalizedNode<?, ?>> optional = readTx.read(nodePath).get(5, TimeUnit.SECONDS);
354 assertTrue("isPresent", optional.isPresent());
355 assertEquals("Data node", nodeToWrite, optional.get());
358 public void doCommit(final DOMStoreThreePhaseCommitCohort cohort) throws Exception {
359 Boolean canCommit = cohort.canCommit().get(commitTimeout, TimeUnit.SECONDS);
360 assertEquals("canCommit", Boolean.TRUE, canCommit);
361 cohort.preCommit().get(5, TimeUnit.SECONDS);
362 cohort.commit().get(5, TimeUnit.SECONDS);
365 void doCommit(final ListenableFuture<Boolean> canCommitFuture, final DOMStoreThreePhaseCommitCohort cohort)
367 Boolean canCommit = canCommitFuture.get(commitTimeout, TimeUnit.SECONDS);
368 assertEquals("canCommit", Boolean.TRUE, canCommit);
369 cohort.preCommit().get(5, TimeUnit.SECONDS);
370 cohort.commit().get(5, TimeUnit.SECONDS);
373 @SuppressWarnings("checkstyle:IllegalCatch")
374 void assertExceptionOnCall(final Callable<Void> callable, final Class<? extends Exception> expType) {
377 fail("Expected " + expType.getSimpleName());
378 } catch (Exception e) {
379 assertEquals("Exception type", expType, e.getClass());
383 void assertExceptionOnTxChainCreates(final DOMStoreTransactionChain txChain,
384 final Class<? extends Exception> expType) {
385 assertExceptionOnCall(() -> {
386 txChain.newWriteOnlyTransaction();
390 assertExceptionOnCall(() -> {
391 txChain.newReadWriteTransaction();
395 assertExceptionOnCall(() -> {
396 txChain.newReadOnlyTransaction();
401 public interface ShardStatsVerifier {
402 void verify(ShardStats stats);