2 * Copyright (c) 2015 Brocade Communications Systems, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.controller.cluster.datastore.admin;
10 import akka.actor.ActorRef;
11 import akka.actor.Status.Success;
12 import akka.dispatch.OnComplete;
13 import akka.pattern.Patterns;
14 import akka.util.Timeout;
15 import com.google.common.base.Function;
16 import com.google.common.base.Strings;
17 import com.google.common.base.Throwables;
18 import com.google.common.util.concurrent.FutureCallback;
19 import com.google.common.util.concurrent.Futures;
20 import com.google.common.util.concurrent.ListenableFuture;
21 import com.google.common.util.concurrent.SettableFuture;
22 import java.io.FileOutputStream;
23 import java.io.IOException;
24 import java.util.AbstractMap.SimpleEntry;
25 import java.util.ArrayList;
26 import java.util.HashMap;
27 import java.util.List;
29 import java.util.Map.Entry;
31 import java.util.concurrent.Future;
32 import java.util.concurrent.TimeUnit;
33 import org.apache.commons.lang3.SerializationUtils;
34 import org.opendaylight.controller.cluster.access.concepts.MemberName;
35 import org.opendaylight.controller.cluster.datastore.DistributedDataStoreInterface;
36 import org.opendaylight.controller.cluster.datastore.messages.AddShardReplica;
37 import org.opendaylight.controller.cluster.datastore.messages.ChangeShardMembersVotingStatus;
38 import org.opendaylight.controller.cluster.datastore.messages.DatastoreSnapshot;
39 import org.opendaylight.controller.cluster.datastore.messages.DatastoreSnapshotList;
40 import org.opendaylight.controller.cluster.datastore.messages.FlipShardMembersVotingStatus;
41 import org.opendaylight.controller.cluster.datastore.messages.RemoveShardReplica;
42 import org.opendaylight.controller.cluster.datastore.utils.ActorContext;
43 import org.opendaylight.controller.cluster.raft.client.messages.GetSnapshot;
44 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.AddReplicasForAllShardsOutput;
45 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.AddReplicasForAllShardsOutputBuilder;
46 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.AddShardReplicaInput;
47 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.BackupDatastoreInput;
48 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.ChangeMemberVotingStatesForAllShardsInput;
49 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.ChangeMemberVotingStatesForAllShardsOutput;
50 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.ChangeMemberVotingStatesForAllShardsOutputBuilder;
51 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.ChangeMemberVotingStatesForShardInput;
52 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.ClusterAdminService;
53 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.DataStoreType;
54 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.FlipMemberVotingStatesForAllShardsOutput;
55 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.FlipMemberVotingStatesForAllShardsOutputBuilder;
56 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.RemoveAllShardReplicasInput;
57 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.RemoveAllShardReplicasOutput;
58 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.RemoveAllShardReplicasOutputBuilder;
59 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.RemoveShardReplicaInput;
60 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.member.voting.states.input.MemberVotingState;
61 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.shard.result.output.ShardResult;
62 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.shard.result.output.ShardResultBuilder;
63 import org.opendaylight.yangtools.yang.common.RpcError.ErrorType;
64 import org.opendaylight.yangtools.yang.common.RpcResult;
65 import org.opendaylight.yangtools.yang.common.RpcResultBuilder;
66 import org.slf4j.Logger;
67 import org.slf4j.LoggerFactory;
70 * Implements the yang RPCs defined in the generated ClusterAdminService interface.
72 * @author Thomas Pantelis
74 public class ClusterAdminRpcService implements ClusterAdminService {
75 private static final Timeout SHARD_MGR_TIMEOUT = new Timeout(1, TimeUnit.MINUTES);
77 private static final Logger LOG = LoggerFactory.getLogger(ClusterAdminRpcService.class);
79 private final DistributedDataStoreInterface configDataStore;
80 private final DistributedDataStoreInterface operDataStore;
82 public ClusterAdminRpcService(DistributedDataStoreInterface configDataStore,
83 DistributedDataStoreInterface operDataStore) {
84 this.configDataStore = configDataStore;
85 this.operDataStore = operDataStore;
89 public Future<RpcResult<Void>> addShardReplica(final AddShardReplicaInput input) {
90 final String shardName = input.getShardName();
91 if (Strings.isNullOrEmpty(shardName)) {
92 return newFailedRpcResultFuture("A valid shard name must be specified");
95 DataStoreType dataStoreType = input.getDataStoreType();
96 if (dataStoreType == null) {
97 return newFailedRpcResultFuture("A valid DataStoreType must be specified");
100 LOG.info("Adding replica for shard {}", shardName);
102 final SettableFuture<RpcResult<Void>> returnFuture = SettableFuture.create();
103 ListenableFuture<Success> future = sendMessageToShardManager(dataStoreType, new AddShardReplica(shardName));
104 Futures.addCallback(future, new FutureCallback<Success>() {
106 public void onSuccess(Success success) {
107 LOG.info("Successfully added replica for shard {}", shardName);
108 returnFuture.set(newSuccessfulResult());
112 public void onFailure(Throwable failure) {
113 onMessageFailure(String.format("Failed to add replica for shard %s", shardName),
114 returnFuture, failure);
122 public Future<RpcResult<Void>> removeShardReplica(RemoveShardReplicaInput input) {
123 final String shardName = input.getShardName();
124 if (Strings.isNullOrEmpty(shardName)) {
125 return newFailedRpcResultFuture("A valid shard name must be specified");
128 DataStoreType dataStoreType = input.getDataStoreType();
129 if (dataStoreType == null) {
130 return newFailedRpcResultFuture("A valid DataStoreType must be specified");
133 final String memberName = input.getMemberName();
134 if (Strings.isNullOrEmpty(memberName)) {
135 return newFailedRpcResultFuture("A valid member name must be specified");
138 LOG.info("Removing replica for shard {} memberName {}, datastoreType {}", shardName, memberName, dataStoreType);
140 final SettableFuture<RpcResult<Void>> returnFuture = SettableFuture.create();
141 ListenableFuture<Success> future = sendMessageToShardManager(dataStoreType,
142 new RemoveShardReplica(shardName, MemberName.forName(memberName)));
143 Futures.addCallback(future, new FutureCallback<Success>() {
145 public void onSuccess(Success success) {
146 LOG.info("Successfully removed replica for shard {}", shardName);
147 returnFuture.set(newSuccessfulResult());
151 public void onFailure(Throwable failure) {
152 onMessageFailure(String.format("Failed to remove replica for shard %s", shardName),
153 returnFuture, failure);
161 public Future<RpcResult<AddReplicasForAllShardsOutput>> addReplicasForAllShards() {
162 LOG.info("Adding replicas for all shards");
164 final List<Entry<ListenableFuture<Success>, ShardResultBuilder>> shardResultData = new ArrayList<>();
165 Function<String, Object> messageSupplier = shardName -> new AddShardReplica(shardName);
167 sendMessageToManagerForConfiguredShards(DataStoreType.Config, shardResultData, messageSupplier);
168 sendMessageToManagerForConfiguredShards(DataStoreType.Operational, shardResultData, messageSupplier);
170 return waitForShardResults(shardResultData, shardResults ->
171 new AddReplicasForAllShardsOutputBuilder().setShardResult(shardResults).build(),
172 "Failed to add replica");
177 public Future<RpcResult<RemoveAllShardReplicasOutput>> removeAllShardReplicas(RemoveAllShardReplicasInput input) {
178 LOG.info("Removing replicas for all shards");
180 final String memberName = input.getMemberName();
181 if (Strings.isNullOrEmpty(memberName)) {
182 return newFailedRpcResultFuture("A valid member name must be specified");
185 final List<Entry<ListenableFuture<Success>, ShardResultBuilder>> shardResultData = new ArrayList<>();
186 Function<String, Object> messageSupplier = shardName ->
187 new RemoveShardReplica(shardName, MemberName.forName(memberName));
189 sendMessageToManagerForConfiguredShards(DataStoreType.Config, shardResultData, messageSupplier);
190 sendMessageToManagerForConfiguredShards(DataStoreType.Operational, shardResultData, messageSupplier);
192 return waitForShardResults(shardResultData, shardResults ->
193 new RemoveAllShardReplicasOutputBuilder().setShardResult(shardResults).build(),
194 " Failed to remove replica");
198 public Future<RpcResult<Void>> changeMemberVotingStatesForShard(ChangeMemberVotingStatesForShardInput input) {
199 final String shardName = input.getShardName();
200 if (Strings.isNullOrEmpty(shardName)) {
201 return newFailedRpcResultFuture("A valid shard name must be specified");
204 DataStoreType dataStoreType = input.getDataStoreType();
205 if (dataStoreType == null) {
206 return newFailedRpcResultFuture("A valid DataStoreType must be specified");
209 List<MemberVotingState> memberVotingStates = input.getMemberVotingState();
210 if (memberVotingStates == null || memberVotingStates.isEmpty()) {
211 return newFailedRpcResultFuture("No member voting state input was specified");
214 ChangeShardMembersVotingStatus changeVotingStatus = toChangeShardMembersVotingStatus(shardName,
217 LOG.info("Change member voting states for shard {}: {}", shardName,
218 changeVotingStatus.getMeberVotingStatusMap());
220 final SettableFuture<RpcResult<Void>> returnFuture = SettableFuture.create();
221 ListenableFuture<Success> future = sendMessageToShardManager(dataStoreType, changeVotingStatus);
222 Futures.addCallback(future, new FutureCallback<Success>() {
224 public void onSuccess(Success success) {
225 LOG.info("Successfully changed member voting states for shard {}", shardName);
226 returnFuture.set(newSuccessfulResult());
230 public void onFailure(Throwable failure) {
231 onMessageFailure(String.format("Failed to change member voting states for shard %s", shardName),
232 returnFuture, failure);
240 public Future<RpcResult<ChangeMemberVotingStatesForAllShardsOutput>> changeMemberVotingStatesForAllShards(
241 final ChangeMemberVotingStatesForAllShardsInput input) {
242 List<MemberVotingState> memberVotingStates = input.getMemberVotingState();
243 if (memberVotingStates == null || memberVotingStates.isEmpty()) {
244 return newFailedRpcResultFuture("No member voting state input was specified");
247 final List<Entry<ListenableFuture<Success>, ShardResultBuilder>> shardResultData = new ArrayList<>();
248 Function<String, Object> messageSupplier = shardName ->
249 toChangeShardMembersVotingStatus(shardName, memberVotingStates);
251 LOG.info("Change member voting states for all shards");
253 sendMessageToManagerForConfiguredShards(DataStoreType.Config, shardResultData, messageSupplier);
254 sendMessageToManagerForConfiguredShards(DataStoreType.Operational, shardResultData, messageSupplier);
256 return waitForShardResults(shardResultData, shardResults ->
257 new ChangeMemberVotingStatesForAllShardsOutputBuilder().setShardResult(shardResults).build(),
258 "Failed to change member voting states");
262 public Future<RpcResult<FlipMemberVotingStatesForAllShardsOutput>> flipMemberVotingStatesForAllShards() {
263 final List<Entry<ListenableFuture<Success>, ShardResultBuilder>> shardResultData = new ArrayList<>();
264 Function<String, Object> messageSupplier = shardName ->
265 new FlipShardMembersVotingStatus(shardName);
267 LOG.info("Flip member voting states for all shards");
269 sendMessageToManagerForConfiguredShards(DataStoreType.Config, shardResultData, messageSupplier);
270 sendMessageToManagerForConfiguredShards(DataStoreType.Operational, shardResultData, messageSupplier);
272 return waitForShardResults(shardResultData, shardResults ->
273 new FlipMemberVotingStatesForAllShardsOutputBuilder().setShardResult(shardResults).build(),
274 "Failed to change member voting states");
278 public Future<RpcResult<Void>> backupDatastore(final BackupDatastoreInput input) {
279 LOG.debug("backupDatastore: {}", input);
281 if (Strings.isNullOrEmpty(input.getFilePath())) {
282 return newFailedRpcResultFuture("A valid file path must be specified");
285 final SettableFuture<RpcResult<Void>> returnFuture = SettableFuture.create();
286 ListenableFuture<List<DatastoreSnapshot>> future = sendMessageToShardManagers(GetSnapshot.INSTANCE);
287 Futures.addCallback(future, new FutureCallback<List<DatastoreSnapshot>>() {
289 public void onSuccess(List<DatastoreSnapshot> snapshots) {
290 saveSnapshotsToFile(new DatastoreSnapshotList(snapshots), input.getFilePath(), returnFuture);
294 public void onFailure(Throwable failure) {
295 onDatastoreBackupFailure(input.getFilePath(), returnFuture, failure);
302 private ChangeShardMembersVotingStatus toChangeShardMembersVotingStatus(final String shardName,
303 List<MemberVotingState> memberVotingStatus) {
304 Map<String, Boolean> serverVotingStatusMap = new HashMap<>();
305 for (MemberVotingState memberStatus: memberVotingStatus) {
306 serverVotingStatusMap.put(memberStatus.getMemberName(), memberStatus.isVoting());
309 ChangeShardMembersVotingStatus changeVotingStatus = new ChangeShardMembersVotingStatus(shardName,
310 serverVotingStatusMap);
311 return changeVotingStatus;
314 private static <T> SettableFuture<RpcResult<T>> waitForShardResults(
315 final List<Entry<ListenableFuture<Success>, ShardResultBuilder>> shardResultData,
316 final Function<List<ShardResult>, T> resultDataSupplier,
317 final String failureLogMsgPrefix) {
318 final SettableFuture<RpcResult<T>> returnFuture = SettableFuture.create();
319 final List<ShardResult> shardResults = new ArrayList<>();
320 for (final Entry<ListenableFuture<Success>, ShardResultBuilder> entry : shardResultData) {
321 Futures.addCallback(entry.getKey(), new FutureCallback<Success>() {
323 public void onSuccess(Success result) {
324 synchronized (shardResults) {
325 ShardResultBuilder shardResult = entry.getValue();
326 LOG.debug("onSuccess for shard {}, type {}", shardResult.getShardName(),
327 shardResult.getDataStoreType());
328 shardResults.add(shardResult.setSucceeded(true).build());
334 public void onFailure(Throwable failure) {
335 synchronized (shardResults) {
336 ShardResultBuilder shardResult = entry.getValue();
337 LOG.warn("{} for shard {}, type {}", failureLogMsgPrefix, shardResult.getShardName(),
338 shardResult.getDataStoreType(), failure);
339 shardResults.add(shardResult.setSucceeded(false).setErrorMessage(
340 Throwables.getRootCause(failure).getMessage()).build());
345 void checkIfComplete() {
346 LOG.debug("checkIfComplete: expected {}, actual {}", shardResultData.size(), shardResults.size());
347 if (shardResults.size() == shardResultData.size()) {
348 returnFuture.set(newSuccessfulResult(resultDataSupplier.apply(shardResults)));
356 private <T> void sendMessageToManagerForConfiguredShards(DataStoreType dataStoreType,
357 List<Entry<ListenableFuture<T>, ShardResultBuilder>> shardResultData,
358 Function<String, Object> messageSupplier) {
359 ActorContext actorContext = dataStoreType == DataStoreType.Config ? configDataStore.getActorContext()
360 : operDataStore.getActorContext();
361 Set<String> allShardNames = actorContext.getConfiguration().getAllShardNames();
363 LOG.debug("Sending message to all shards {} for data store {}", allShardNames, actorContext.getDataStoreName());
365 for (String shardName: allShardNames) {
366 ListenableFuture<T> future = this.<T>ask(actorContext.getShardManager(), messageSupplier.apply(shardName),
368 shardResultData.add(new SimpleEntry<>(future,
369 new ShardResultBuilder().setShardName(shardName).setDataStoreType(dataStoreType)));
373 @SuppressWarnings("unchecked")
374 private <T> ListenableFuture<List<T>> sendMessageToShardManagers(Object message) {
375 Timeout timeout = SHARD_MGR_TIMEOUT;
376 ListenableFuture<T> configFuture = ask(configDataStore.getActorContext().getShardManager(), message, timeout);
377 ListenableFuture<T> operFuture = ask(operDataStore.getActorContext().getShardManager(), message, timeout);
379 return Futures.allAsList(configFuture, operFuture);
382 private <T> ListenableFuture<T> sendMessageToShardManager(DataStoreType dataStoreType, Object message) {
383 ActorRef shardManager = dataStoreType == DataStoreType.Config
384 ? configDataStore.getActorContext().getShardManager()
385 : operDataStore.getActorContext().getShardManager();
386 return ask(shardManager, message, SHARD_MGR_TIMEOUT);
389 @SuppressWarnings("checkstyle:IllegalCatch")
390 private static void saveSnapshotsToFile(DatastoreSnapshotList snapshots, String fileName,
391 SettableFuture<RpcResult<Void>> returnFuture) {
392 try (FileOutputStream fos = new FileOutputStream(fileName)) {
393 SerializationUtils.serialize(snapshots, fos);
395 returnFuture.set(newSuccessfulResult());
396 LOG.info("Successfully backed up datastore to file {}", fileName);
397 } catch (IOException | RuntimeException e) {
398 onDatastoreBackupFailure(fileName, returnFuture, e);
402 private static void onDatastoreBackupFailure(String fileName, SettableFuture<RpcResult<Void>> returnFuture,
404 onMessageFailure(String.format("Failed to back up datastore to file %s", fileName), returnFuture, failure);
407 private static void onMessageFailure(String msg, final SettableFuture<RpcResult<Void>> returnFuture,
409 LOG.error(msg, failure);
410 returnFuture.set(ClusterAdminRpcService.<Void>newFailedRpcResultBuilder(String.format("%s: %s", msg,
411 failure.getMessage())).build());
414 private <T> ListenableFuture<T> ask(ActorRef actor, Object message, Timeout timeout) {
415 final SettableFuture<T> returnFuture = SettableFuture.create();
417 @SuppressWarnings("unchecked")
418 scala.concurrent.Future<T> askFuture = (scala.concurrent.Future<T>) Patterns.ask(actor, message, timeout);
419 askFuture.onComplete(new OnComplete<T>() {
421 public void onComplete(Throwable failure, T resp) {
422 if (failure != null) {
423 returnFuture.setException(failure);
425 returnFuture.set(resp);
428 }, configDataStore.getActorContext().getClientDispatcher());
433 private static <T> ListenableFuture<RpcResult<T>> newFailedRpcResultFuture(String message) {
434 return ClusterAdminRpcService.<T>newFailedRpcResultBuilder(message).buildFuture();
437 private static <T> RpcResultBuilder<T> newFailedRpcResultBuilder(String message) {
438 return newFailedRpcResultBuilder(message, null);
441 private static <T> RpcResultBuilder<T> newFailedRpcResultBuilder(String message, Throwable cause) {
442 return RpcResultBuilder.<T>failed().withError(ErrorType.RPC, message, cause);
445 private static RpcResult<Void> newSuccessfulResult() {
446 return newSuccessfulResult((Void)null);
449 private static <T> RpcResult<T> newSuccessfulResult(T data) {
450 return RpcResultBuilder.<T>success(data).build();