opendaylight/md-sal/sal-cluster-admin/src/main/java/org/opendaylight/controller/cluster/datastore/admin/ClusterAdminRpcService.java

   1 /*
   2  * Copyright (c) 2015 Brocade Communications Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.controller.cluster.datastore.admin;
   9
  10 import akka.actor.ActorRef;
  11 import akka.actor.Status.Success;
  12 import akka.dispatch.OnComplete;
  13 import akka.pattern.Patterns;
  14 import akka.util.Timeout;
  15 import com.google.common.base.Function;
  16 import com.google.common.base.Strings;
  17 import com.google.common.base.Throwables;
  18 import com.google.common.util.concurrent.FutureCallback;
  19 import com.google.common.util.concurrent.Futures;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import com.google.common.util.concurrent.SettableFuture;
  22 import java.io.FileOutputStream;
  23 import java.io.IOException;
  24 import java.util.AbstractMap.SimpleEntry;
  25 import java.util.ArrayList;
  26 import java.util.HashMap;
  27 import java.util.List;
  28 import java.util.Map;
  29 import java.util.Map.Entry;
  30 import java.util.Set;
  31 import java.util.concurrent.Future;
  32 import java.util.concurrent.TimeUnit;
  33 import org.apache.commons.lang3.SerializationUtils;
  34 import org.opendaylight.controller.cluster.access.concepts.MemberName;
  35 import org.opendaylight.controller.cluster.datastore.DistributedDataStoreInterface;
  36 import org.opendaylight.controller.cluster.datastore.messages.AddShardReplica;
  37 import org.opendaylight.controller.cluster.datastore.messages.ChangeShardMembersVotingStatus;
  38 import org.opendaylight.controller.cluster.datastore.messages.DatastoreSnapshot;
  39 import org.opendaylight.controller.cluster.datastore.messages.DatastoreSnapshotList;
  40 import org.opendaylight.controller.cluster.datastore.messages.FlipShardMembersVotingStatus;
  41 import org.opendaylight.controller.cluster.datastore.messages.RemoveShardReplica;
  42 import org.opendaylight.controller.cluster.datastore.utils.ActorContext;
  43 import org.opendaylight.controller.cluster.raft.client.messages.GetSnapshot;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.AddReplicasForAllShardsOutput;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.AddReplicasForAllShardsOutputBuilder;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.AddShardReplicaInput;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.BackupDatastoreInput;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.ChangeMemberVotingStatesForAllShardsInput;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.ChangeMemberVotingStatesForAllShardsOutput;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.ChangeMemberVotingStatesForAllShardsOutputBuilder;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.ChangeMemberVotingStatesForShardInput;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.ClusterAdminService;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.DataStoreType;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.FlipMemberVotingStatesForAllShardsOutput;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.FlipMemberVotingStatesForAllShardsOutputBuilder;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.RemoveAllShardReplicasInput;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.RemoveAllShardReplicasOutput;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.RemoveAllShardReplicasOutputBuilder;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.RemoveShardReplicaInput;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.member.voting.states.input.MemberVotingState;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.shard.result.output.ShardResult;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.controller.md.sal.cluster.admin.rev151013.shard.result.output.ShardResultBuilder;
  63 import org.opendaylight.yangtools.yang.common.RpcError.ErrorType;
  64 import org.opendaylight.yangtools.yang.common.RpcResult;
  65 import org.opendaylight.yangtools.yang.common.RpcResultBuilder;
  66 import org.slf4j.Logger;
  67 import org.slf4j.LoggerFactory;
  68
  69 /**
  70  * Implements the yang RPCs defined in the generated ClusterAdminService interface.
  71  *
  72  * @author Thomas Pantelis
  73  */
  74 public class ClusterAdminRpcService implements ClusterAdminService {
  75     private static final Timeout SHARD_MGR_TIMEOUT = new Timeout(1, TimeUnit.MINUTES);
  76
  77     private static final Logger LOG = LoggerFactory.getLogger(ClusterAdminRpcService.class);
  78
  79     private final DistributedDataStoreInterface configDataStore;
  80     private final DistributedDataStoreInterface operDataStore;
  81
  82     public ClusterAdminRpcService(DistributedDataStoreInterface configDataStore,
  83             DistributedDataStoreInterface operDataStore) {
  84         this.configDataStore = configDataStore;
  85         this.operDataStore = operDataStore;
  86     }
  87
  88     @Override
  89     public Future<RpcResult<Void>> addShardReplica(final AddShardReplicaInput input) {
  90         final String shardName = input.getShardName();
  91         if (Strings.isNullOrEmpty(shardName)) {
  92             return newFailedRpcResultFuture("A valid shard name must be specified");
  93         }
  94
  95         DataStoreType dataStoreType = input.getDataStoreType();
  96         if (dataStoreType == null) {
  97             return newFailedRpcResultFuture("A valid DataStoreType must be specified");
  98         }
  99
 100         LOG.info("Adding replica for shard {}", shardName);
 101
 102         final SettableFuture<RpcResult<Void>> returnFuture = SettableFuture.create();
 103         ListenableFuture<Success> future = sendMessageToShardManager(dataStoreType, new AddShardReplica(shardName));
 104         Futures.addCallback(future, new FutureCallback<Success>() {
 105             @Override
 106             public void onSuccess(Success success) {
 107                 LOG.info("Successfully added replica for shard {}", shardName);
 108                 returnFuture.set(newSuccessfulResult());
 109             }
 110
 111             @Override
 112             public void onFailure(Throwable failure) {
 113                 onMessageFailure(String.format("Failed to add replica for shard %s", shardName),
 114                         returnFuture, failure);
 115             }
 116         });
 117
 118         return returnFuture;
 119     }
 120
 121     @Override
 122     public Future<RpcResult<Void>> removeShardReplica(RemoveShardReplicaInput input) {
 123         final String shardName = input.getShardName();
 124         if (Strings.isNullOrEmpty(shardName)) {
 125             return newFailedRpcResultFuture("A valid shard name must be specified");
 126         }
 127
 128         DataStoreType dataStoreType = input.getDataStoreType();
 129         if (dataStoreType == null) {
 130             return newFailedRpcResultFuture("A valid DataStoreType must be specified");
 131         }
 132
 133         final String memberName = input.getMemberName();
 134         if (Strings.isNullOrEmpty(memberName)) {
 135             return newFailedRpcResultFuture("A valid member name must be specified");
 136         }
 137
 138         LOG.info("Removing replica for shard {} memberName {}, datastoreType {}", shardName, memberName, dataStoreType);
 139
 140         final SettableFuture<RpcResult<Void>> returnFuture = SettableFuture.create();
 141         ListenableFuture<Success> future = sendMessageToShardManager(dataStoreType,
 142                 new RemoveShardReplica(shardName, MemberName.forName(memberName)));
 143         Futures.addCallback(future, new FutureCallback<Success>() {
 144             @Override
 145             public void onSuccess(Success success) {
 146                 LOG.info("Successfully removed replica for shard {}", shardName);
 147                 returnFuture.set(newSuccessfulResult());
 148             }
 149
 150             @Override
 151             public void onFailure(Throwable failure) {
 152                 onMessageFailure(String.format("Failed to remove replica for shard %s", shardName),
 153                         returnFuture, failure);
 154             }
 155         });
 156
 157         return returnFuture;
 158     }
 159
 160     @Override
 161     public Future<RpcResult<AddReplicasForAllShardsOutput>> addReplicasForAllShards() {
 162         LOG.info("Adding replicas for all shards");
 163
 164         final List<Entry<ListenableFuture<Success>, ShardResultBuilder>> shardResultData = new ArrayList<>();
 165         Function<String, Object> messageSupplier = shardName -> new AddShardReplica(shardName);
 166
 167         sendMessageToManagerForConfiguredShards(DataStoreType.Config, shardResultData, messageSupplier);
 168         sendMessageToManagerForConfiguredShards(DataStoreType.Operational, shardResultData, messageSupplier);
 169
 170         return waitForShardResults(shardResultData, shardResults ->
 171                 new AddReplicasForAllShardsOutputBuilder().setShardResult(shardResults).build(),
 172                 "Failed to add replica");
 173     }
 174
 175
 176     @Override
 177     public Future<RpcResult<RemoveAllShardReplicasOutput>> removeAllShardReplicas(RemoveAllShardReplicasInput input) {
 178         LOG.info("Removing replicas for all shards");
 179
 180         final String memberName = input.getMemberName();
 181         if (Strings.isNullOrEmpty(memberName)) {
 182             return newFailedRpcResultFuture("A valid member name must be specified");
 183         }
 184
 185         final List<Entry<ListenableFuture<Success>, ShardResultBuilder>> shardResultData = new ArrayList<>();
 186         Function<String, Object> messageSupplier = shardName ->
 187                 new RemoveShardReplica(shardName, MemberName.forName(memberName));
 188
 189         sendMessageToManagerForConfiguredShards(DataStoreType.Config, shardResultData, messageSupplier);
 190         sendMessageToManagerForConfiguredShards(DataStoreType.Operational, shardResultData, messageSupplier);
 191
 192         return waitForShardResults(shardResultData, shardResults ->
 193                 new RemoveAllShardReplicasOutputBuilder().setShardResult(shardResults).build(),
 194         "       Failed to remove replica");
 195     }
 196
 197     @Override
 198     public Future<RpcResult<Void>> changeMemberVotingStatesForShard(ChangeMemberVotingStatesForShardInput input) {
 199         final String shardName = input.getShardName();
 200         if (Strings.isNullOrEmpty(shardName)) {
 201             return newFailedRpcResultFuture("A valid shard name must be specified");
 202         }
 203
 204         DataStoreType dataStoreType = input.getDataStoreType();
 205         if (dataStoreType == null) {
 206             return newFailedRpcResultFuture("A valid DataStoreType must be specified");
 207         }
 208
 209         List<MemberVotingState> memberVotingStates = input.getMemberVotingState();
 210         if (memberVotingStates == null || memberVotingStates.isEmpty()) {
 211             return newFailedRpcResultFuture("No member voting state input was specified");
 212         }
 213
 214         ChangeShardMembersVotingStatus changeVotingStatus = toChangeShardMembersVotingStatus(shardName,
 215                 memberVotingStates);
 216
 217         LOG.info("Change member voting states for shard {}: {}", shardName,
 218                 changeVotingStatus.getMeberVotingStatusMap());
 219
 220         final SettableFuture<RpcResult<Void>> returnFuture = SettableFuture.create();
 221         ListenableFuture<Success> future = sendMessageToShardManager(dataStoreType, changeVotingStatus);
 222         Futures.addCallback(future, new FutureCallback<Success>() {
 223             @Override
 224             public void onSuccess(Success success) {
 225                 LOG.info("Successfully changed member voting states for shard {}", shardName);
 226                 returnFuture.set(newSuccessfulResult());
 227             }
 228
 229             @Override
 230             public void onFailure(Throwable failure) {
 231                 onMessageFailure(String.format("Failed to change member voting states for shard %s", shardName),
 232                         returnFuture, failure);
 233             }
 234         });
 235
 236         return returnFuture;
 237     }
 238
 239     @Override
 240     public Future<RpcResult<ChangeMemberVotingStatesForAllShardsOutput>> changeMemberVotingStatesForAllShards(
 241             final ChangeMemberVotingStatesForAllShardsInput input) {
 242         List<MemberVotingState> memberVotingStates = input.getMemberVotingState();
 243         if (memberVotingStates == null || memberVotingStates.isEmpty()) {
 244             return newFailedRpcResultFuture("No member voting state input was specified");
 245         }
 246
 247         final List<Entry<ListenableFuture<Success>, ShardResultBuilder>> shardResultData = new ArrayList<>();
 248         Function<String, Object> messageSupplier = shardName ->
 249                 toChangeShardMembersVotingStatus(shardName, memberVotingStates);
 250
 251         LOG.info("Change member voting states for all shards");
 252
 253         sendMessageToManagerForConfiguredShards(DataStoreType.Config, shardResultData, messageSupplier);
 254         sendMessageToManagerForConfiguredShards(DataStoreType.Operational, shardResultData, messageSupplier);
 255
 256         return waitForShardResults(shardResultData, shardResults ->
 257                 new ChangeMemberVotingStatesForAllShardsOutputBuilder().setShardResult(shardResults).build(),
 258                 "Failed to change member voting states");
 259     }
 260
 261     @Override
 262     public Future<RpcResult<FlipMemberVotingStatesForAllShardsOutput>> flipMemberVotingStatesForAllShards() {
 263         final List<Entry<ListenableFuture<Success>, ShardResultBuilder>> shardResultData = new ArrayList<>();
 264         Function<String, Object> messageSupplier = shardName ->
 265                 new FlipShardMembersVotingStatus(shardName);
 266
 267         LOG.info("Flip member voting states for all shards");
 268
 269         sendMessageToManagerForConfiguredShards(DataStoreType.Config, shardResultData, messageSupplier);
 270         sendMessageToManagerForConfiguredShards(DataStoreType.Operational, shardResultData, messageSupplier);
 271
 272         return waitForShardResults(shardResultData, shardResults ->
 273                 new FlipMemberVotingStatesForAllShardsOutputBuilder().setShardResult(shardResults).build(),
 274                 "Failed to change member voting states");
 275     }
 276
 277     @Override
 278     public Future<RpcResult<Void>> backupDatastore(final BackupDatastoreInput input) {
 279         LOG.debug("backupDatastore: {}", input);
 280
 281         if (Strings.isNullOrEmpty(input.getFilePath())) {
 282             return newFailedRpcResultFuture("A valid file path must be specified");
 283         }
 284
 285         final SettableFuture<RpcResult<Void>> returnFuture = SettableFuture.create();
 286         ListenableFuture<List<DatastoreSnapshot>> future = sendMessageToShardManagers(GetSnapshot.INSTANCE);
 287         Futures.addCallback(future, new FutureCallback<List<DatastoreSnapshot>>() {
 288             @Override
 289             public void onSuccess(List<DatastoreSnapshot> snapshots) {
 290                 saveSnapshotsToFile(new DatastoreSnapshotList(snapshots), input.getFilePath(), returnFuture);
 291             }
 292
 293             @Override
 294             public void onFailure(Throwable failure) {
 295                 onDatastoreBackupFailure(input.getFilePath(), returnFuture, failure);
 296             }
 297         });
 298
 299         return returnFuture;
 300     }
 301
 302     private ChangeShardMembersVotingStatus toChangeShardMembersVotingStatus(final String shardName,
 303             List<MemberVotingState> memberVotingStatus) {
 304         Map<String, Boolean> serverVotingStatusMap = new HashMap<>();
 305         for (MemberVotingState memberStatus: memberVotingStatus) {
 306             serverVotingStatusMap.put(memberStatus.getMemberName(), memberStatus.isVoting());
 307         }
 308
 309         ChangeShardMembersVotingStatus changeVotingStatus = new ChangeShardMembersVotingStatus(shardName,
 310                 serverVotingStatusMap);
 311         return changeVotingStatus;
 312     }
 313
 314     private static <T> SettableFuture<RpcResult<T>> waitForShardResults(
 315             final List<Entry<ListenableFuture<Success>, ShardResultBuilder>> shardResultData,
 316             final Function<List<ShardResult>, T> resultDataSupplier,
 317             final String failureLogMsgPrefix) {
 318         final SettableFuture<RpcResult<T>> returnFuture = SettableFuture.create();
 319         final List<ShardResult> shardResults = new ArrayList<>();
 320         for (final Entry<ListenableFuture<Success>, ShardResultBuilder> entry : shardResultData) {
 321             Futures.addCallback(entry.getKey(), new FutureCallback<Success>() {
 322                 @Override
 323                 public void onSuccess(Success result) {
 324                     synchronized (shardResults) {
 325                         ShardResultBuilder shardResult = entry.getValue();
 326                         LOG.debug("onSuccess for shard {}, type {}", shardResult.getShardName(),
 327                                 shardResult.getDataStoreType());
 328                         shardResults.add(shardResult.setSucceeded(true).build());
 329                         checkIfComplete();
 330                     }
 331                 }
 332
 333                 @Override
 334                 public void onFailure(Throwable failure) {
 335                     synchronized (shardResults) {
 336                         ShardResultBuilder shardResult = entry.getValue();
 337                         LOG.warn("{} for shard {}, type {}", failureLogMsgPrefix, shardResult.getShardName(),
 338                                 shardResult.getDataStoreType(), failure);
 339                         shardResults.add(shardResult.setSucceeded(false).setErrorMessage(
 340                                 Throwables.getRootCause(failure).getMessage()).build());
 341                         checkIfComplete();
 342                     }
 343                 }
 344
 345                 void checkIfComplete() {
 346                     LOG.debug("checkIfComplete: expected {}, actual {}", shardResultData.size(), shardResults.size());
 347                     if (shardResults.size() == shardResultData.size()) {
 348                         returnFuture.set(newSuccessfulResult(resultDataSupplier.apply(shardResults)));
 349                     }
 350                 }
 351             });
 352         }
 353         return returnFuture;
 354     }
 355
 356     private <T> void sendMessageToManagerForConfiguredShards(DataStoreType dataStoreType,
 357             List<Entry<ListenableFuture<T>, ShardResultBuilder>> shardResultData,
 358             Function<String, Object> messageSupplier) {
 359         ActorContext actorContext = dataStoreType == DataStoreType.Config ? configDataStore.getActorContext()
 360                 : operDataStore.getActorContext();
 361         Set<String> allShardNames = actorContext.getConfiguration().getAllShardNames();
 362
 363         LOG.debug("Sending message to all shards {} for data store {}", allShardNames, actorContext.getDataStoreName());
 364
 365         for (String shardName: allShardNames) {
 366             ListenableFuture<T> future = this.<T>ask(actorContext.getShardManager(), messageSupplier.apply(shardName),
 367                     SHARD_MGR_TIMEOUT);
 368             shardResultData.add(new SimpleEntry<>(future,
 369                     new ShardResultBuilder().setShardName(shardName).setDataStoreType(dataStoreType)));
 370         }
 371     }
 372
 373     @SuppressWarnings("unchecked")
 374     private <T> ListenableFuture<List<T>> sendMessageToShardManagers(Object message) {
 375         Timeout timeout = SHARD_MGR_TIMEOUT;
 376         ListenableFuture<T> configFuture = ask(configDataStore.getActorContext().getShardManager(), message, timeout);
 377         ListenableFuture<T> operFuture = ask(operDataStore.getActorContext().getShardManager(), message, timeout);
 378
 379         return Futures.allAsList(configFuture, operFuture);
 380     }
 381
 382     private <T> ListenableFuture<T> sendMessageToShardManager(DataStoreType dataStoreType, Object message) {
 383         ActorRef shardManager = dataStoreType == DataStoreType.Config
 384                 ? configDataStore.getActorContext().getShardManager()
 385                         : operDataStore.getActorContext().getShardManager();
 386         return ask(shardManager, message, SHARD_MGR_TIMEOUT);
 387     }
 388
 389     @SuppressWarnings("checkstyle:IllegalCatch")
 390     private static void saveSnapshotsToFile(DatastoreSnapshotList snapshots, String fileName,
 391             SettableFuture<RpcResult<Void>> returnFuture) {
 392         try (FileOutputStream fos = new FileOutputStream(fileName)) {
 393             SerializationUtils.serialize(snapshots, fos);
 394
 395             returnFuture.set(newSuccessfulResult());
 396             LOG.info("Successfully backed up datastore to file {}", fileName);
 397         } catch (IOException | RuntimeException e) {
 398             onDatastoreBackupFailure(fileName, returnFuture, e);
 399         }
 400     }
 401
 402     private static void onDatastoreBackupFailure(String fileName, SettableFuture<RpcResult<Void>> returnFuture,
 403             Throwable failure) {
 404         onMessageFailure(String.format("Failed to back up datastore to file %s", fileName), returnFuture, failure);
 405     }
 406
 407     private static void onMessageFailure(String msg, final SettableFuture<RpcResult<Void>> returnFuture,
 408             Throwable failure) {
 409         LOG.error(msg, failure);
 410         returnFuture.set(ClusterAdminRpcService.<Void>newFailedRpcResultBuilder(String.format("%s: %s", msg,
 411                 failure.getMessage())).build());
 412     }
 413
 414     private <T> ListenableFuture<T> ask(ActorRef actor, Object message, Timeout timeout) {
 415         final SettableFuture<T> returnFuture = SettableFuture.create();
 416
 417         @SuppressWarnings("unchecked")
 418         scala.concurrent.Future<T> askFuture = (scala.concurrent.Future<T>) Patterns.ask(actor, message, timeout);
 419         askFuture.onComplete(new OnComplete<T>() {
 420             @Override
 421             public void onComplete(Throwable failure, T resp) {
 422                 if (failure != null) {
 423                     returnFuture.setException(failure);
 424                 } else {
 425                     returnFuture.set(resp);
 426                 }
 427             }
 428         }, configDataStore.getActorContext().getClientDispatcher());
 429
 430         return returnFuture;
 431     }
 432
 433     private static <T> ListenableFuture<RpcResult<T>> newFailedRpcResultFuture(String message) {
 434         return ClusterAdminRpcService.<T>newFailedRpcResultBuilder(message).buildFuture();
 435     }
 436
 437     private static <T> RpcResultBuilder<T> newFailedRpcResultBuilder(String message) {
 438         return newFailedRpcResultBuilder(message, null);
 439     }
 440
 441     private static <T> RpcResultBuilder<T> newFailedRpcResultBuilder(String message, Throwable cause) {
 442         return RpcResultBuilder.<T>failed().withError(ErrorType.RPC, message, cause);
 443     }
 444
 445     private static RpcResult<Void> newSuccessfulResult() {
 446         return newSuccessfulResult((Void)null);
 447     }
 448
 449     private static <T> RpcResult<T> newSuccessfulResult(T data) {
 450         return RpcResultBuilder.<T>success(data).build();
 451     }
 452 }