X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?p=controller.git;a=blobdiff_plain;f=opendaylight%2Fmd-sal%2Fsal-distributed-datastore%2Fsrc%2Fmain%2Fjava%2Forg%2Fopendaylight%2Fcontroller%2Fcluster%2Fsharding%2FDistributedShardedDOMDataTree.java;h=df21b901da6676e29e2bf7352e5a734228b9d965;hp=df49e33cdd09d74f295c6922af03974081edb5c5;hb=f83b2d36fdd7e953ba72492ffb684cd112aa04a6;hpb=d69af1f79ae4630be8c4d65b98096aa27b1665b6 diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/sharding/DistributedShardedDOMDataTree.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/sharding/DistributedShardedDOMDataTree.java index df49e33cdd..df21b901da 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/sharding/DistributedShardedDOMDataTree.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/sharding/DistributedShardedDOMDataTree.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved. + * Copyright (c) 2016, 2017 Cisco Systems, Inc. and others. All rights reserved. * * This program and the accompanying materials are made available under the * terms of the Eclipse Public License v1.0 which accompanies this distribution, @@ -9,6 +9,9 @@ package org.opendaylight.controller.cluster.sharding; import static akka.actor.ActorRef.noSender; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; import akka.actor.ActorRef; import akka.actor.ActorSystem; @@ -21,16 +24,19 @@ import akka.util.Timeout; import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.base.Throwables; +import com.google.common.collect.ClassToInstanceMap; import com.google.common.collect.ForwardingObject; +import com.google.common.collect.ImmutableClassToInstanceMap; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; import com.google.common.util.concurrent.Uninterruptibles; import java.util.AbstractMap.SimpleEntry; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.EnumMap; import java.util.HashMap; import java.util.List; @@ -48,13 +54,13 @@ import org.opendaylight.controller.cluster.ActorSystemProvider; import org.opendaylight.controller.cluster.access.concepts.MemberName; import org.opendaylight.controller.cluster.databroker.actors.dds.DataStoreClient; import org.opendaylight.controller.cluster.databroker.actors.dds.SimpleDataStoreClientActor; -import org.opendaylight.controller.cluster.datastore.DistributedDataStore; +import org.opendaylight.controller.cluster.datastore.AbstractDataStore; import org.opendaylight.controller.cluster.datastore.Shard; import org.opendaylight.controller.cluster.datastore.config.Configuration; import org.opendaylight.controller.cluster.datastore.config.ModuleShardConfiguration; import org.opendaylight.controller.cluster.datastore.messages.CreateShard; import org.opendaylight.controller.cluster.datastore.shardstrategy.ModuleShardStrategy; -import org.opendaylight.controller.cluster.datastore.utils.ActorContext; +import org.opendaylight.controller.cluster.datastore.utils.ActorUtils; import org.opendaylight.controller.cluster.datastore.utils.ClusterUtils; import org.opendaylight.controller.cluster.dom.api.CDSDataTreeProducer; import org.opendaylight.controller.cluster.dom.api.CDSShardAccess; @@ -73,6 +79,7 @@ import org.opendaylight.mdsal.dom.api.DOMDataTreeLoopException; import org.opendaylight.mdsal.dom.api.DOMDataTreeProducer; import org.opendaylight.mdsal.dom.api.DOMDataTreeProducerException; import org.opendaylight.mdsal.dom.api.DOMDataTreeService; +import org.opendaylight.mdsal.dom.api.DOMDataTreeServiceExtension; import org.opendaylight.mdsal.dom.api.DOMDataTreeShard; import org.opendaylight.mdsal.dom.api.DOMDataTreeShardingConflictException; import org.opendaylight.mdsal.dom.api.DOMDataTreeShardingService; @@ -86,7 +93,6 @@ import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import scala.compat.java8.FutureConverters; -import scala.concurrent.Await; import scala.concurrent.Future; import scala.concurrent.Promise; import scala.concurrent.duration.FiniteDuration; @@ -112,8 +118,8 @@ public class DistributedShardedDOMDataTree implements DOMDataTreeService, DOMDat private final ShardedDOMDataTree shardedDOMDataTree; private final ActorSystem actorSystem; - private final DistributedDataStore distributedOperDatastore; - private final DistributedDataStore distributedConfigDatastore; + private final AbstractDataStore distributedOperDatastore; + private final AbstractDataStore distributedConfigDatastore; private final ActorRef shardedDataTreeActor; private final MemberName memberName; @@ -122,9 +128,6 @@ public class DistributedShardedDOMDataTree implements DOMDataTreeService, DOMDat private final DOMDataTreePrefixTable> shards = DOMDataTreePrefixTable.create(); - private final EnumMap defaultShardRegistrations = - new EnumMap<>(LogicalDatastoreType.class); - private final EnumMap> configurationShardMap = new EnumMap<>(LogicalDatastoreType.class); @@ -134,8 +137,8 @@ public class DistributedShardedDOMDataTree implements DOMDataTreeService, DOMDat private final PrefixedShardConfigUpdateHandler updateHandler; public DistributedShardedDOMDataTree(final ActorSystemProvider actorSystemProvider, - final DistributedDataStore distributedOperDatastore, - final DistributedDataStore distributedConfigDatastore) { + final AbstractDataStore distributedOperDatastore, + final AbstractDataStore distributedConfigDatastore) { this.actorSystem = Preconditions.checkNotNull(actorSystemProvider).getActorSystem(); this.distributedOperDatastore = Preconditions.checkNotNull(distributedOperDatastore); this.distributedConfigDatastore = Preconditions.checkNotNull(distributedConfigDatastore); @@ -145,32 +148,32 @@ public class DistributedShardedDOMDataTree implements DOMDataTreeService, DOMDat new ShardedDataTreeActorCreator() .setShardingService(this) .setActorSystem(actorSystem) - .setClusterWrapper(distributedConfigDatastore.getActorContext().getClusterWrapper()) + .setClusterWrapper(distributedConfigDatastore.getActorUtils().getClusterWrapper()) .setDistributedConfigDatastore(distributedConfigDatastore) .setDistributedOperDatastore(distributedOperDatastore) .setLookupTaskMaxRetries(LOOKUP_TASK_MAX_RETRIES), ACTOR_ID); - this.memberName = distributedConfigDatastore.getActorContext().getCurrentMemberName(); + this.memberName = distributedConfigDatastore.getActorUtils().getCurrentMemberName(); updateHandler = new PrefixedShardConfigUpdateHandler(shardedDataTreeActor, - distributedConfigDatastore.getActorContext().getCurrentMemberName()); + distributedConfigDatastore.getActorUtils().getCurrentMemberName()); LOG.debug("{} - Starting prefix configuration shards", memberName); createPrefixConfigShard(distributedConfigDatastore); createPrefixConfigShard(distributedOperDatastore); } - private void createPrefixConfigShard(final DistributedDataStore dataStore) { - Configuration configuration = dataStore.getActorContext().getConfiguration(); + private static void createPrefixConfigShard(final AbstractDataStore dataStore) { + Configuration configuration = dataStore.getActorUtils().getConfiguration(); Collection memberNames = configuration.getUniqueMemberNamesForAllShards(); CreateShard createShardMessage = new CreateShard(new ModuleShardConfiguration(PrefixShards.QNAME.getNamespace(), "prefix-shard-configuration", ClusterUtils.PREFIX_CONFIG_SHARD_ID, ModuleShardStrategy.NAME, memberNames), - Shard.builder(), dataStore.getActorContext().getDatastoreContext()); + Shard.builder(), dataStore.getActorUtils().getDatastoreContext()); - dataStore.getActorContext().getShardManager().tell(createShardMessage, noSender()); + dataStore.getActorUtils().getShardManager().tell(createShardMessage, noSender()); } /** @@ -195,24 +198,20 @@ public class DistributedShardedDOMDataTree implements DOMDataTreeService, DOMDat public void init() { // create our writers to the configuration try { - LOG.debug("{} - starting config shard lookup.", - distributedConfigDatastore.getActorContext().getCurrentMemberName()); + LOG.debug("{} - starting config shard lookup.", memberName); // We have to wait for prefix config shards to be up and running // so we can create datastore clients for them handleConfigShardLookup().get(SHARD_FUTURE_TIMEOUT_DURATION.length(), SHARD_FUTURE_TIMEOUT_DURATION.unit()); - - LOG.debug("Prefix configuration shards ready - creating clients"); - } catch (InterruptedException | ExecutionException | TimeoutException e) { throw new IllegalStateException("Prefix config shards not found", e); } try { - LOG.debug("Prefix configuration shards ready - creating clients"); + LOG.debug("{}: Prefix configuration shards ready - creating clients", memberName); configurationShardMap.put(LogicalDatastoreType.CONFIGURATION, createDatastoreClient(ClusterUtils.PREFIX_CONFIG_SHARD_ID, - distributedConfigDatastore.getActorContext())); + distributedConfigDatastore.getActorUtils())); } catch (final DOMDataTreeShardCreationFailedException e) { throw new IllegalStateException( "Unable to create datastoreClient for config DS prefix configuration shard.", e); @@ -221,7 +220,7 @@ public class DistributedShardedDOMDataTree implements DOMDataTreeService, DOMDat try { configurationShardMap.put(LogicalDatastoreType.OPERATIONAL, createDatastoreClient(ClusterUtils.PREFIX_CONFIG_SHARD_ID, - distributedOperDatastore.getActorContext())); + distributedOperDatastore.getActorUtils())); } catch (final DOMDataTreeShardCreationFailedException e) { throw new IllegalStateException( @@ -237,24 +236,13 @@ public class DistributedShardedDOMDataTree implements DOMDataTreeService, DOMDat updateHandler.initListener(distributedConfigDatastore, LogicalDatastoreType.CONFIGURATION); updateHandler.initListener(distributedOperDatastore, LogicalDatastoreType.OPERATIONAL); - distributedConfigDatastore.getActorContext().getShardManager().tell(InitConfigListener.INSTANCE, noSender()); - distributedOperDatastore.getActorContext().getShardManager().tell(InitConfigListener.INSTANCE, noSender()); + distributedConfigDatastore.getActorUtils().getShardManager().tell(InitConfigListener.INSTANCE, noSender()); + distributedOperDatastore.getActorUtils().getShardManager().tell(InitConfigListener.INSTANCE, noSender()); //create shard registration for DEFAULT_SHARD - try { - defaultShardRegistrations.put(LogicalDatastoreType.CONFIGURATION, - initDefaultShard(LogicalDatastoreType.CONFIGURATION)); - } catch (final InterruptedException | ExecutionException e) { - throw new IllegalStateException("Unable to create default shard frontend for config shard", e); - } - - try { - defaultShardRegistrations.put(LogicalDatastoreType.OPERATIONAL, - initDefaultShard(LogicalDatastoreType.OPERATIONAL)); - } catch (final InterruptedException | ExecutionException e) { - throw new IllegalStateException("Unable to create default shard frontend for operational shard", e); - } + initDefaultShard(LogicalDatastoreType.CONFIGURATION); + initDefaultShard(LogicalDatastoreType.OPERATIONAL); } private ListenableFuture> handleConfigShardLookup() { @@ -273,7 +261,7 @@ public class DistributedShardedDOMDataTree implements DOMDataTreeService, DOMDat ask.onComplete(new OnComplete() { @Override - public void onComplete(final Throwable throwable, final Object result) throws Throwable { + public void onComplete(final Throwable throwable, final Object result) { if (throwable != null) { future.setException(throwable); } else { @@ -294,27 +282,32 @@ public class DistributedShardedDOMDataTree implements DOMDataTreeService, DOMDat return shardedDOMDataTree.registerListener(listener, subtrees, allowRxMerges, producers); } + @Override + public ClassToInstanceMap getExtensions() { + return ImmutableClassToInstanceMap.of(); + } + @Nonnull @Override public DOMDataTreeProducer createProducer(@Nonnull final Collection subtrees) { - LOG.debug("{} - Creating producer for {}", - distributedConfigDatastore.getActorContext().getClusterWrapper().getCurrentMemberName(), subtrees); + LOG.debug("{} - Creating producer for {}", memberName, subtrees); final DOMDataTreeProducer producer = shardedDOMDataTree.createProducer(subtrees); - final Object response = distributedConfigDatastore.getActorContext() + final Object response = distributedConfigDatastore.getActorUtils() .executeOperation(shardedDataTreeActor, new ProducerCreated(subtrees)); if (response == null) { - LOG.debug("{} - Received success from remote nodes, creating producer:{}", - distributedConfigDatastore.getActorContext().getClusterWrapper().getCurrentMemberName(), subtrees); + LOG.debug("{} - Received success from remote nodes, creating producer:{}", memberName, subtrees); return new ProxyProducer(producer, subtrees, shardedDataTreeActor, - distributedConfigDatastore.getActorContext()); - } else if (response instanceof Exception) { - closeProducer(producer); - throw Throwables.propagate((Exception) response); - } else { - closeProducer(producer); - throw new RuntimeException("Unexpected response to create producer received." + response); + distributedConfigDatastore.getActorUtils(), shards); } + + closeProducer(producer); + + if (response instanceof Throwable) { + Throwables.throwIfUnchecked((Throwable) response); + throw new RuntimeException((Throwable) response); + } + throw new RuntimeException("Unexpected response to create producer received." + response); } @Override @@ -366,52 +359,43 @@ public class DistributedShardedDOMDataTree implements DOMDataTreeService, DOMDat shardRegistrationPromise.failure( new DOMDataTreeShardCreationFailedException("Unable to create a cds shard.", throwable)); } - }); + }, MoreExecutors.directExecutor()); return FutureConverters.toJava(shardRegistrationPromise.future()); } void resolveShardAdditions(final Set additions) { - LOG.debug("Member {}: Resolving additions : {}", memberName, additions); - final ArrayList list = new ArrayList<>(additions); + LOG.debug("{}: Resolving additions : {}", memberName, additions); // we need to register the shards from top to bottom, so we need to atleast make sure the ordering reflects that - Collections.sort(list, (o1, o2) -> { - if (o1.getRootIdentifier().getPathArguments().size() < o2.getRootIdentifier().getPathArguments().size()) { - return -1; - } else if (o1.getRootIdentifier().getPathArguments().size() - == o2.getRootIdentifier().getPathArguments().size()) { - return 0; - } else { - return 1; - } - }); - list.forEach(this::createShardFrontend); + additions + .stream() + .sorted(Comparator.comparingInt(o -> o.getRootIdentifier().getPathArguments().size())) + .forEachOrdered(this::createShardFrontend); } void resolveShardRemovals(final Set removals) { - LOG.debug("Member {}: Resolving removals : {}", memberName, removals); + LOG.debug("{}: Resolving removals : {}", memberName, removals); // do we need to go from bottom to top? removals.forEach(this::despawnShardFrontend); } private void createShardFrontend(final DOMDataTreeIdentifier prefix) { - LOG.debug("Member {}: Creating CDS shard for prefix: {}", memberName, prefix); + LOG.debug("{}: Creating CDS shard for prefix: {}", memberName, prefix); final String shardName = ClusterUtils.getCleanShardName(prefix.getRootIdentifier()); - final DistributedDataStore distributedDataStore = + final AbstractDataStore distributedDataStore = prefix.getDatastoreType().equals(org.opendaylight.mdsal.common.api.LogicalDatastoreType.CONFIGURATION) ? distributedConfigDatastore : distributedOperDatastore; - try (final DOMDataTreeProducer producer = localCreateProducer(Collections.singletonList(prefix))) { + try (DOMDataTreeProducer producer = localCreateProducer(Collections.singletonList(prefix))) { final Entry entry = - createDatastoreClient(shardName, distributedDataStore.getActorContext()); + createDatastoreClient(shardName, distributedDataStore.getActorUtils()); final DistributedShardFrontend shard = new DistributedShardFrontend(distributedDataStore, entry.getKey(), prefix); - @SuppressWarnings("unchecked") final DOMDataTreeShardRegistration reg = - (DOMDataTreeShardRegistration) shardedDOMDataTree.registerDataTreeShard(prefix, shard, producer); + shardedDOMDataTree.registerDataTreeShard(prefix, shard, producer); synchronized (shards) { shards.store(prefix, reg); @@ -419,7 +403,7 @@ public class DistributedShardedDOMDataTree implements DOMDataTreeService, DOMDat } catch (final DOMDataTreeShardingConflictException e) { LOG.error("{}: Prefix {} is already occupied by another shard", - distributedConfigDatastore.getActorContext().getClusterWrapper().getCurrentMemberName(), prefix, e); + distributedConfigDatastore.getActorUtils().getClusterWrapper().getCurrentMemberName(), prefix, e); } catch (DOMDataTreeProducerException e) { LOG.error("Unable to close producer", e); } catch (DOMDataTreeShardCreationFailedException e) { @@ -428,14 +412,14 @@ public class DistributedShardedDOMDataTree implements DOMDataTreeService, DOMDat } private void despawnShardFrontend(final DOMDataTreeIdentifier prefix) { - LOG.debug("Member {}: Removing CDS shard for prefix: {}", memberName, prefix); + LOG.debug("{}: Removing CDS shard for prefix: {}", memberName, prefix); final DOMDataTreePrefixTableEntry> lookup; synchronized (shards) { lookup = shards.lookup(prefix); } if (lookup == null || !lookup.getValue().getPrefix().equals(prefix)) { - LOG.debug("Member {}: Received despawn for non-existing CDS shard frontend, prefix: {}, ignoring..", + LOG.debug("{}: Received despawn for non-existing CDS shard frontend, prefix: {}, ignoring..", memberName, prefix); return; } @@ -451,21 +435,22 @@ public class DistributedShardedDOMDataTree implements DOMDataTreeService, DOMDat Futures.addCallback(future, new FutureCallback() { @Override - public void onSuccess(@Nullable Void result) { + public void onSuccess(@Nullable final Void result) { LOG.debug("{} - Succesfuly removed shard for {}", memberName, prefix); } @Override - public void onFailure(Throwable throwable) { + public void onFailure(final Throwable throwable) { LOG.error("Removal of shard {} from configuration failed.", prefix, throwable); } - }); + }, MoreExecutors.directExecutor()); } DOMDataTreePrefixTableEntry> lookupShardFrontend( final DOMDataTreeIdentifier prefix) { - return shards.lookup(prefix); - + synchronized (shards) { + return shards.lookup(prefix); + } } DOMDataTreeProducer localCreateProducer(final Collection prefix) { @@ -490,20 +475,19 @@ public class DistributedShardedDOMDataTree implements DOMDataTreeService, DOMDat } @SuppressWarnings("checkstyle:IllegalCatch") - private Entry createDatastoreClient( - final String shardName, final ActorContext actorContext) + private Entry createDatastoreClient(final String shardName, final ActorUtils actorUtils) throws DOMDataTreeShardCreationFailedException { - LOG.debug("Creating distributed datastore client for shard {}", shardName); + LOG.debug("{}: Creating distributed datastore client for shard {}", memberName, shardName); final Props distributedDataStoreClientProps = - SimpleDataStoreClientActor.props(memberName, "Shard-" + shardName, actorContext, shardName); + SimpleDataStoreClientActor.props(memberName, "Shard-" + shardName, actorUtils, shardName); final ActorRef clientActor = actorSystem.actorOf(distributedDataStoreClientProps); try { return new SimpleEntry<>(SimpleDataStoreClientActor .getDistributedDataStoreClient(clientActor, 30, TimeUnit.SECONDS), clientActor); } catch (final Exception e) { - LOG.error("Failed to get actor for {}", distributedDataStoreClientProps, e); + LOG.error("{}: Failed to get actor for {}", distributedDataStoreClientProps, memberName, e); clientActor.tell(PoisonPill.getInstance(), noSender()); throw new DOMDataTreeShardCreationFailedException( "Unable to create datastore client for shard{" + shardName + "}", e); @@ -511,51 +495,48 @@ public class DistributedShardedDOMDataTree implements DOMDataTreeService, DOMDat } @SuppressWarnings("checkstyle:IllegalCatch") - private DistributedShardRegistration initDefaultShard(final LogicalDatastoreType logicalDatastoreType) - throws ExecutionException, InterruptedException { - final Collection names = - distributedConfigDatastore.getActorContext().getConfiguration().getUniqueMemberNamesForAllShards(); + private void initDefaultShard(final LogicalDatastoreType logicalDatastoreType) { final PrefixedShardConfigWriter writer = writerMap.get(logicalDatastoreType); if (writer.checkDefaultIsPresent()) { - LOG.debug("Default shard for {} is already present in the config. Possibly saved in snapshot.", - logicalDatastoreType); - return new DistributedShardRegistrationImpl( - new DOMDataTreeIdentifier(logicalDatastoreType, YangInstanceIdentifier.EMPTY), - shardedDataTreeActor, this); + LOG.debug("{}: Default shard for {} is already present in the config. Possibly saved in snapshot.", + memberName, logicalDatastoreType); } else { try { - // There can be situation when there is already started default shard - // because it is present in modules.conf. In that case we have to create - // just frontend for default shard, but not shard itself - // TODO we don't have to do it for config and operational default shard - // separately. Just one of them should be enough - final ActorContext actorContext = logicalDatastoreType == LogicalDatastoreType.CONFIGURATION - ? distributedConfigDatastore.getActorContext() : distributedOperDatastore.getActorContext(); + // Currently the default shard configuration is present in the out-of-box modules.conf and is + // expected to be present. So look up the local default shard here and create the frontend. + + // TODO we don't have to do it for config and operational default shard separately. Just one of them + // should be enough + final ActorUtils actorUtils = logicalDatastoreType == LogicalDatastoreType.CONFIGURATION + ? distributedConfigDatastore.getActorUtils() : distributedOperDatastore.getActorUtils(); final Optional defaultLocalShardOptional = - actorContext.findLocalShard(ClusterUtils.getCleanShardName(YangInstanceIdentifier.EMPTY)); + actorUtils.findLocalShard(ClusterUtils.getCleanShardName(YangInstanceIdentifier.EMPTY)); if (defaultLocalShardOptional.isPresent()) { - LOG.debug("{} Default shard is already started, creating just frontend", logicalDatastoreType); + LOG.debug("{}: Default shard for {} is already started, creating just frontend", memberName, + logicalDatastoreType); createShardFrontend(new DOMDataTreeIdentifier(logicalDatastoreType, YangInstanceIdentifier.EMPTY)); - return new DistributedShardRegistrationImpl( - new DOMDataTreeIdentifier(logicalDatastoreType, YangInstanceIdentifier.EMPTY), - shardedDataTreeActor, this); } - // we should probably only have one node create the default shards - return Await.result(FutureConverters.toScala(createDistributedShard( - new DOMDataTreeIdentifier(logicalDatastoreType, YangInstanceIdentifier.EMPTY), names)), - SHARD_FUTURE_TIMEOUT_DURATION); - } catch (DOMDataTreeShardingConflictException e) { - LOG.debug("Default shard already registered, possibly due to other node doing it faster"); - return new DistributedShardRegistrationImpl( - new DOMDataTreeIdentifier(logicalDatastoreType, YangInstanceIdentifier.EMPTY), - shardedDataTreeActor, this); + // The local shard isn't present - we assume that means the local member isn't in the replica list + // and will be dynamically created later via an explicit add-shard-replica request. This is the + // bootstrapping mechanism to add a new node into an existing cluster. The following code to create + // the default shard as a prefix shard is problematic in this scenario so it is commented out. Since + // the default shard is a module-based shard by default, it makes sense to always treat it as such, + // ie bootstrap it in the same manner as the special prefix-configuration and EOS shards. +// final Collection names = distributedConfigDatastore.getActorUtils().getConfiguration() +// .getUniqueMemberNamesForAllShards(); +// Await.result(FutureConverters.toScala(createDistributedShard( +// new DOMDataTreeIdentifier(logicalDatastoreType, YangInstanceIdentifier.EMPTY), names)), +// SHARD_FUTURE_TIMEOUT_DURATION); +// } catch (DOMDataTreeShardingConflictException e) { +// LOG.debug("{}: Default shard for {} already registered, possibly due to other node doing it faster", +// memberName, logicalDatastoreType); } catch (Exception e) { - LOG.error("{} default shard initialization failed", logicalDatastoreType, e); + LOG.error("{}: Default shard initialization for {} failed", memberName, logicalDatastoreType, e); throw new RuntimeException(e); } } @@ -637,18 +618,25 @@ public class DistributedShardedDOMDataTree implements DOMDataTreeService, DOMDat private final DOMDataTreeProducer delegate; private final Collection subtrees; private final ActorRef shardDataTreeActor; - private final ActorContext actorContext; + private final ActorUtils actorUtils; @GuardedBy("shardAccessMap") private final Map shardAccessMap = new HashMap<>(); + // We don't have to guard access to shardTable in ProxyProducer. + // ShardTable's entries relevant to this ProxyProducer shouldn't + // change during producer's lifetime. + private final DOMDataTreePrefixTable> shardTable; + ProxyProducer(final DOMDataTreeProducer delegate, final Collection subtrees, final ActorRef shardDataTreeActor, - final ActorContext actorContext) { - this.delegate = Preconditions.checkNotNull(delegate); - this.subtrees = Preconditions.checkNotNull(subtrees); - this.shardDataTreeActor = Preconditions.checkNotNull(shardDataTreeActor); - this.actorContext = Preconditions.checkNotNull(actorContext); + final ActorUtils actorUtils, + final DOMDataTreePrefixTable> shardLayout) { + this.delegate = requireNonNull(delegate); + this.subtrees = requireNonNull(subtrees); + this.shardDataTreeActor = requireNonNull(shardDataTreeActor); + this.actorUtils = requireNonNull(actorUtils); + this.shardTable = requireNonNull(shardLayout); } @Nonnull @@ -659,6 +647,7 @@ public class DistributedShardedDOMDataTree implements DOMDataTreeService, DOMDat @Nonnull @Override + @SuppressWarnings("checkstyle:hiddenField") public DOMDataTreeProducer createProducer(@Nonnull final Collection subtrees) { // TODO we probably don't need to distribute this on the remote nodes since once we have this producer // open we surely have the rights to all the subtrees. @@ -669,13 +658,14 @@ public class DistributedShardedDOMDataTree implements DOMDataTreeService, DOMDat @SuppressWarnings("checkstyle:IllegalCatch") public void close() throws DOMDataTreeProducerException { delegate.close(); + synchronized (shardAccessMap) { shardAccessMap.values().forEach(CDSShardAccessImpl::close); } - final Object o = actorContext.executeOperation(shardDataTreeActor, new ProducerRemoved(subtrees)); + final Object o = actorUtils.executeOperation(shardDataTreeActor, new ProducerRemoved(subtrees)); if (o instanceof DOMDataTreeProducerException) { - throw ((DOMDataTreeProducerException) o); + throw (DOMDataTreeProducerException) o; } else if (o instanceof Throwable) { throw new DOMDataTreeProducerException("Unable to close producer", (Throwable) o); } @@ -689,19 +679,27 @@ public class DistributedShardedDOMDataTree implements DOMDataTreeService, DOMDat @Nonnull @Override public CDSShardAccess getShardAccess(@Nonnull final DOMDataTreeIdentifier subtree) { + checkArgument(subtrees.stream().anyMatch(dataTreeIdentifier -> dataTreeIdentifier.contains(subtree)), + "Subtree %s is not controlled by this producer %s", subtree, this); + + final DOMDataTreePrefixTableEntry> lookup = + shardTable.lookup(subtree); + checkState(lookup != null, "Subtree %s is not contained in any registered shard.", subtree); + + final DOMDataTreeIdentifier lookupId = lookup.getValue().getPrefix(); + synchronized (shardAccessMap) { - Preconditions.checkArgument(subtrees.contains(subtree), - "Subtree {} is not controlled by this producer {}", subtree, this); - if (shardAccessMap.get(subtree) != null) { - return shardAccessMap.get(subtree); + if (shardAccessMap.get(lookupId) != null) { + return shardAccessMap.get(lookupId); } // TODO Maybe we can have static factory method and return the same instance // for same subtrees. But maybe it is not needed since there can be only one // producer attached to some subtree at a time. And also how we can close ShardAccess // then - final CDSShardAccessImpl shardAccess = new CDSShardAccessImpl(subtree, actorContext); - return shardAccessMap.put(subtree, shardAccess); + final CDSShardAccessImpl shardAccess = new CDSShardAccessImpl(lookupId, actorUtils); + shardAccessMap.put(lookupId, shardAccess); + return shardAccess; } } }