2 * Copyright (c) 2015 Brocade Communications Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.controller.cluster.datastore;
10 import static org.junit.Assert.assertEquals;
11 import static org.junit.Assert.assertNotNull;
12 import static org.junit.Assert.assertThrows;
13 import static org.junit.Assert.fail;
14 import static org.mockito.ArgumentMatchers.anyString;
15 import static org.mockito.Mockito.doReturn;
16 import static org.mockito.Mockito.mock;
18 import akka.actor.ActorRef;
19 import akka.actor.ActorSystem;
20 import akka.cluster.Cluster;
21 import akka.cluster.ClusterEvent.CurrentClusterState;
22 import akka.cluster.Member;
23 import akka.cluster.MemberStatus;
24 import com.google.common.base.Stopwatch;
25 import com.google.common.collect.Sets;
26 import com.google.common.util.concurrent.ListenableFuture;
27 import com.google.common.util.concurrent.Uninterruptibles;
28 import java.util.Optional;
30 import java.util.concurrent.TimeUnit;
31 import java.util.function.Consumer;
32 import org.opendaylight.controller.cluster.databroker.ClientBackedDataStore;
33 import org.opendaylight.controller.cluster.datastore.DatastoreContext.Builder;
34 import org.opendaylight.controller.cluster.datastore.config.Configuration;
35 import org.opendaylight.controller.cluster.datastore.config.ConfigurationImpl;
36 import org.opendaylight.controller.cluster.datastore.messages.OnDemandShardState;
37 import org.opendaylight.controller.cluster.datastore.persisted.DatastoreSnapshot;
38 import org.opendaylight.controller.cluster.datastore.utils.ActorUtils;
39 import org.opendaylight.controller.cluster.raft.client.messages.GetOnDemandRaftState;
40 import org.opendaylight.controller.md.cluster.datastore.model.SchemaContextHelper;
41 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
42 import org.opendaylight.mdsal.dom.spi.store.DOMStoreReadTransaction;
43 import org.opendaylight.mdsal.dom.spi.store.DOMStoreThreePhaseCommitCohort;
44 import org.opendaylight.mdsal.dom.spi.store.DOMStoreTransactionChain;
45 import org.opendaylight.mdsal.dom.spi.store.DOMStoreWriteTransaction;
46 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
47 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
48 import org.opendaylight.yangtools.yang.model.api.EffectiveModelContext;
49 import org.slf4j.Logger;
50 import org.slf4j.LoggerFactory;
51 import scala.concurrent.Await;
52 import scala.concurrent.Future;
53 import scala.concurrent.duration.FiniteDuration;
55 public class IntegrationTestKit extends ShardTestKit {
57 private static final Logger LOG = LoggerFactory.getLogger(IntegrationTestKit.class);
59 protected DatastoreContext.Builder datastoreContextBuilder;
60 protected DatastoreSnapshot restoreFromSnapshot;
61 private final int commitTimeout;
63 public IntegrationTestKit(final ActorSystem actorSystem, final Builder datastoreContextBuilder) {
64 this(actorSystem, datastoreContextBuilder, 7);
67 public IntegrationTestKit(final ActorSystem actorSystem, final Builder datastoreContextBuilder,
68 final int commitTimeout) {
70 this.datastoreContextBuilder = datastoreContextBuilder;
71 this.commitTimeout = commitTimeout;
74 public DatastoreContext.Builder getDatastoreContextBuilder() {
75 return datastoreContextBuilder;
78 public ClientBackedDataStore setupDataStore(final Class<? extends ClientBackedDataStore> implementation,
79 final String typeName, final String... shardNames) throws Exception {
80 return setupDataStore(implementation, typeName, "module-shards.conf", true, SchemaContextHelper.full(),
84 public ClientBackedDataStore setupDataStore(final Class<? extends ClientBackedDataStore> implementation,
85 final String typeName, final boolean waitUntilLeader, final String... shardNames) throws Exception {
86 return setupDataStore(implementation, typeName, "module-shards.conf", waitUntilLeader,
87 SchemaContextHelper.full(), shardNames);
90 public ClientBackedDataStore setupDataStore(final Class<? extends ClientBackedDataStore> implementation,
91 final String typeName, final String moduleShardsConfig, final boolean waitUntilLeader,
92 final String... shardNames) throws Exception {
93 return setupDataStore(implementation, typeName, moduleShardsConfig, waitUntilLeader,
94 SchemaContextHelper.full(), shardNames);
97 public ClientBackedDataStore setupDataStore(final Class<? extends ClientBackedDataStore> implementation,
98 final String typeName, final String moduleShardsConfig, final boolean waitUntilLeader,
99 final EffectiveModelContext schemaContext, final String... shardNames) throws Exception {
100 return setupDataStore(implementation, typeName, moduleShardsConfig, "modules.conf", waitUntilLeader,
101 schemaContext, shardNames);
104 private ClientBackedDataStore setupDataStore(final Class<? extends ClientBackedDataStore> implementation,
105 final String typeName, final String moduleShardsConfig, final String modulesConfig,
106 final boolean waitUntilLeader, final EffectiveModelContext schemaContext, final String... shardNames)
108 final ClusterWrapper cluster = new ClusterWrapperImpl(getSystem());
109 final Configuration config = new ConfigurationImpl(moduleShardsConfig, modulesConfig);
111 setDataStoreName(typeName);
113 final DatastoreContext datastoreContext = datastoreContextBuilder.build();
114 final DatastoreContextFactory mockContextFactory = mock(DatastoreContextFactory.class);
115 doReturn(datastoreContext).when(mockContextFactory).getBaseDatastoreContext();
116 doReturn(datastoreContext).when(mockContextFactory).getShardDatastoreContext(anyString());
118 final var constructor = implementation.getDeclaredConstructor(ActorSystem.class, ClusterWrapper.class,
119 Configuration.class, DatastoreContextFactory.class, DatastoreSnapshot.class);
121 final var dataStore = constructor.newInstance(getSystem(), cluster, config, mockContextFactory,
122 restoreFromSnapshot);
124 dataStore.onModelContextUpdated(schemaContext);
126 if (waitUntilLeader) {
127 waitUntilLeader(dataStore.getActorUtils(), shardNames);
130 datastoreContextBuilder = DatastoreContext.newBuilderFrom(datastoreContext);
134 private void setDataStoreName(final String typeName) {
135 if ("config".equals(typeName)) {
136 datastoreContextBuilder.logicalStoreType(LogicalDatastoreType.CONFIGURATION);
137 } else if ("operational".equals(typeName)) {
138 datastoreContextBuilder.logicalStoreType(LogicalDatastoreType.OPERATIONAL);
140 datastoreContextBuilder.dataStoreName(typeName);
144 public void waitUntilLeader(final ActorUtils actorUtils, final String... shardNames) {
145 for (String shardName: shardNames) {
146 ActorRef shard = findLocalShard(actorUtils, shardName);
148 assertNotNull("Shard was not created for " + shardName, shard);
150 waitUntilLeader(shard);
154 public void waitUntilNoLeader(final ActorUtils actorUtils, final String... shardNames) {
155 for (String shardName: shardNames) {
156 ActorRef shard = findLocalShard(actorUtils, shardName);
157 assertNotNull("No local shard found for " + shardName, shard);
159 waitUntilNoLeader(shard);
163 public void waitForMembersUp(final String... otherMembers) {
164 Set<String> otherMembersSet = Sets.newHashSet(otherMembers);
165 Stopwatch sw = Stopwatch.createStarted();
166 while (sw.elapsed(TimeUnit.SECONDS) <= 10) {
167 CurrentClusterState state = Cluster.get(getSystem()).state();
168 for (Member m: state.getMembers()) {
169 if (m.status() == MemberStatus.up() && otherMembersSet.remove(m.getRoles().iterator().next())
170 && otherMembersSet.isEmpty()) {
175 Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
178 fail("Member(s) " + otherMembersSet + " are not Up");
181 public static ActorRef findLocalShard(final ActorUtils actorUtils, final String shardName) {
182 for (int i = 0; i < 20 * 5; i++) {
183 Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
184 Optional<ActorRef> shardReply = actorUtils.findLocalShard(shardName);
185 if (shardReply.isPresent()) {
186 return shardReply.orElseThrow();
192 public static void waitUntilShardIsDown(final ActorUtils actorUtils, final String shardName) {
193 for (int i = 0; i < 20 * 5 ; i++) {
194 LOG.debug("Waiting for shard down {}", shardName);
195 Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
196 Optional<ActorRef> shardReply = actorUtils.findLocalShard(shardName);
197 if (!shardReply.isPresent()) {
202 throw new IllegalStateException("Shard[" + shardName + " did not shutdown in time");
205 public static void verifyShardStats(final ClientBackedDataStore datastore, final String shardName,
206 final ShardStatsVerifier verifier) throws Exception {
207 ActorUtils actorUtils = datastore.getActorUtils();
209 Future<ActorRef> future = actorUtils.findLocalShardAsync(shardName);
210 ActorRef shardActor = Await.result(future, FiniteDuration.create(10, TimeUnit.SECONDS));
212 AssertionError lastError = null;
213 Stopwatch sw = Stopwatch.createStarted();
214 while (sw.elapsed(TimeUnit.SECONDS) <= 5) {
215 ShardStats shardStats = (ShardStats)actorUtils
216 .executeOperation(shardActor, Shard.GET_SHARD_MBEAN_MESSAGE);
219 verifier.verify(shardStats);
221 } catch (AssertionError e) {
223 Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
230 public static void verifyShardState(final ClientBackedDataStore datastore, final String shardName,
231 final Consumer<OnDemandShardState> verifier) throws Exception {
232 ActorUtils actorUtils = datastore.getActorUtils();
234 Future<ActorRef> future = actorUtils.findLocalShardAsync(shardName);
235 ActorRef shardActor = Await.result(future, FiniteDuration.create(10, TimeUnit.SECONDS));
237 AssertionError lastError = null;
238 Stopwatch sw = Stopwatch.createStarted();
239 while (sw.elapsed(TimeUnit.SECONDS) <= 5) {
240 OnDemandShardState shardState = (OnDemandShardState)actorUtils
241 .executeOperation(shardActor, GetOnDemandRaftState.INSTANCE);
244 verifier.accept(shardState);
246 } catch (AssertionError e) {
248 Uninterruptibles.sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
255 void testWriteTransaction(final ClientBackedDataStore dataStore, final YangInstanceIdentifier nodePath,
256 final NormalizedNode nodeToWrite) throws Exception {
258 // 1. Create a write-only Tx
260 DOMStoreWriteTransaction writeTx = dataStore.newWriteOnlyTransaction();
261 assertNotNull("newWriteOnlyTransaction returned null", writeTx);
263 // 2. Write some data
265 writeTx.write(nodePath, nodeToWrite);
267 // 3. Ready the Tx for commit
269 DOMStoreThreePhaseCommitCohort cohort = writeTx.ready();
275 // 5. Verify the data in the store
277 DOMStoreReadTransaction readTx = dataStore.newReadOnlyTransaction();
278 assertEquals(Optional.of(nodeToWrite), readTx.read(nodePath).get(5, TimeUnit.SECONDS));
281 public void doCommit(final DOMStoreThreePhaseCommitCohort cohort) throws Exception {
282 Boolean canCommit = cohort.canCommit().get(commitTimeout, TimeUnit.SECONDS);
283 assertEquals("canCommit", Boolean.TRUE, canCommit);
284 cohort.preCommit().get(5, TimeUnit.SECONDS);
285 cohort.commit().get(5, TimeUnit.SECONDS);
288 void doCommit(final ListenableFuture<Boolean> canCommitFuture, final DOMStoreThreePhaseCommitCohort cohort)
290 Boolean canCommit = canCommitFuture.get(commitTimeout, TimeUnit.SECONDS);
291 assertEquals("canCommit", Boolean.TRUE, canCommit);
292 cohort.preCommit().get(5, TimeUnit.SECONDS);
293 cohort.commit().get(5, TimeUnit.SECONDS);
296 void assertExceptionOnTxChainCreates(final DOMStoreTransactionChain txChain,
297 final Class<? extends Exception> expType) {
298 assertThrows(expType, () -> txChain.newWriteOnlyTransaction());
299 assertThrows(expType, () -> txChain.newReadWriteTransaction());
300 assertThrows(expType, () -> txChain.newReadOnlyTransaction());
303 public interface ShardStatsVerifier {
304 void verify(ShardStats stats);