X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=opendaylight%2Fmd-sal%2Fsal-distributed-datastore%2Fsrc%2Fmain%2Fjava%2Forg%2Fopendaylight%2Fcontroller%2Fcluster%2Fdatabroker%2Factors%2Fdds%2FModuleShardBackendResolver.java;h=f6452a19b43cbbf7b3b6b365f3e1da1b7f8d5362;hb=refs%2Fchanges%2F10%2F78310%2F5;hp=9e6485b296e0e87dc20febe7db12ab08991d4db3;hpb=320a4e5cd2d9d80468a3f82798744f2035488218;p=controller.git diff --git a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/databroker/actors/dds/ModuleShardBackendResolver.java b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/databroker/actors/dds/ModuleShardBackendResolver.java index 9e6485b296..f6452a19b4 100644 --- a/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/databroker/actors/dds/ModuleShardBackendResolver.java +++ b/opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/databroker/actors/dds/ModuleShardBackendResolver.java @@ -7,39 +7,31 @@ */ package org.opendaylight.controller.cluster.databroker.actors.dds; -import akka.actor.ActorRef; +import static akka.pattern.Patterns.ask; +import static com.google.common.base.Verify.verifyNotNull; + +import akka.dispatch.ExecutionContexts; +import akka.dispatch.OnComplete; import akka.util.Timeout; -import com.google.common.base.Preconditions; -import com.google.common.base.Throwables; import com.google.common.collect.BiMap; import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableBiMap.Builder; -import com.google.common.primitives.UnsignedLong; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.ThreadSafe; -import org.opendaylight.controller.cluster.access.ABIVersion; import org.opendaylight.controller.cluster.access.client.BackendInfoResolver; -import org.opendaylight.controller.cluster.access.commands.ConnectClientRequest; -import org.opendaylight.controller.cluster.access.commands.ConnectClientSuccess; import org.opendaylight.controller.cluster.access.concepts.ClientIdentifier; -import org.opendaylight.controller.cluster.access.concepts.RequestFailure; -import org.opendaylight.controller.cluster.common.actor.ExplicitAsk; +import org.opendaylight.controller.cluster.datastore.shardmanager.RegisterForShardAvailabilityChanges; import org.opendaylight.controller.cluster.datastore.shardstrategy.DefaultShardStrategy; import org.opendaylight.controller.cluster.datastore.utils.ActorContext; +import org.opendaylight.yangtools.concepts.Registration; import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import scala.Function1; -import scala.compat.java8.FutureConverters; +import scala.concurrent.Future; /** * {@link BackendInfoResolver} implementation for static shard configuration based on ShardManager. Each string-named @@ -48,52 +40,13 @@ import scala.compat.java8.FutureConverters; * * @author Robert Varga */ -@SuppressFBWarnings(value = "NP_NONNULL_PARAM_VIOLATION", - justification = "Pertains to the NULL_FUTURE field below. Null is allowed and is intended") @ThreadSafe -final class ModuleShardBackendResolver extends BackendInfoResolver { - private static final class Entry { - private final CompletionStage stage; - @GuardedBy("this") - private ShardBackendInfo result; - - Entry(final CompletionStage stage) { - this.stage = Preconditions.checkNotNull(stage); - stage.whenComplete(this::onStageResolved); - } - - @Nonnull CompletionStage getStage() { - return stage; - } - - synchronized @Nullable ShardBackendInfo getResult() { - return result; - } - - private synchronized void onStageResolved(final ShardBackendInfo result, final Throwable failure) { - if (failure == null) { - this.result = Preconditions.checkNotNull(result); - } else { - LOG.warn("Failed to resolve shard", failure); - } - } - } - - private static final CompletableFuture NULL_FUTURE = CompletableFuture.completedFuture(null); +final class ModuleShardBackendResolver extends AbstractShardBackendResolver { private static final Logger LOG = LoggerFactory.getLogger(ModuleShardBackendResolver.class); - /** - * Fall-over-dead timeout. If we do not make progress in this long, just fall over and propagate the failure. - * All users are expected to fail, possibly attempting to recover by restarting. It is fair to remain - * non-operational. - */ - // TODO: maybe make this configurable somehow? - private static final Timeout DEAD_TIMEOUT = Timeout.apply(15, TimeUnit.MINUTES); + private final ConcurrentMap backends = new ConcurrentHashMap<>(); - private final ConcurrentMap backends = new ConcurrentHashMap<>(); - private final AtomicLong nextSessionId = new AtomicLong(); - private final Function1 connectFunction; - private final ActorContext actorContext; + private final Future shardAvailabilityChangesRegFuture; @GuardedBy("this") private long nextShard = 1; @@ -102,13 +55,36 @@ final class ModuleShardBackendResolver extends BackendInfoResolver new ConnectClientRequest(clientId, t, ABIVersion.BORON, - ABIVersion.current())); + super(clientId, actorContext); + + shardAvailabilityChangesRegFuture = ask(actorContext.getShardManager(), new RegisterForShardAvailabilityChanges( + this::onShardAvailabilityChange), Timeout.apply(60, TimeUnit.MINUTES)) + .map(reply -> (Registration)reply, ExecutionContexts.global()); + + shardAvailabilityChangesRegFuture.onComplete(new OnComplete() { + @Override + public void onComplete(Throwable failure, Registration reply) { + if (failure != null) { + LOG.error("RegisterForShardAvailabilityChanges failed", failure); + } + } + }, ExecutionContexts.global()); + } + + private void onShardAvailabilityChange(String shardName) { + LOG.debug("onShardAvailabilityChange for {}", shardName); + + Long cookie = shards.get(shardName); + if (cookie == null) { + LOG.debug("No shard cookie found for {}", shardName); + return; + } + + notifyStaleBackendInfoCallbacks(cookie); } Long resolveShardForPath(final YangInstanceIdentifier path) { - final String shardName = actorContext.getShardStrategyFactory().getStrategy(path).findShard(path); + final String shardName = actorContext().getShardStrategyFactory().getStrategy(path).findShard(path); Long cookie = shards.get(shardName); if (cookie == null) { synchronized (this) { @@ -127,57 +103,84 @@ final class ModuleShardBackendResolver extends BackendInfoResolver resolveBackendInfo(final Long cookie) { + @Override + public CompletionStage getBackendInfo(final Long cookie) { + /* + * We cannot perform a simple computeIfAbsent() here because we need to control sequencing of when the state + * is inserted into the map and retired from it (based on the stage result). + * + * We do not want to hook another stage one processing completes and hooking a removal on failure from a compute + * method runs the inherent risk of stage completing before the insertion does (i.e. we have a removal of + * non-existent element. + */ + final ShardState existing = backends.get(cookie); + if (existing != null) { + return existing.getStage(); + } + final String shardName = shards.inverse().get(cookie); if (shardName == null) { LOG.warn("Failing request for non-existent cookie {}", cookie); - return NULL_FUTURE; + throw new IllegalArgumentException("Cookie " + cookie + " does not have a shard assigned"); } LOG.debug("Resolving cookie {} to shard {}", cookie, shardName); + final ShardState toInsert = resolveBackendInfo(shardName, cookie); - return FutureConverters.toJava(actorContext.findPrimaryShardAsync(shardName)).thenCompose(info -> { - LOG.debug("Looking up primary info for {} from {}", shardName, info); - return FutureConverters.toJava(ExplicitAsk.ask(info.getPrimaryShardActor(), connectFunction, DEAD_TIMEOUT)); - }).thenApply(response -> { - if (response instanceof RequestFailure) { - final RequestFailure failure = (RequestFailure) response; - LOG.debug("Connect request failed {}", failure, failure.getCause()); - throw Throwables.propagate(failure.getCause()); - } - - LOG.debug("Resolved backend information to {}", response); + final ShardState raced = backends.putIfAbsent(cookie, toInsert); + if (raced != null) { + // We have had a concurrent insertion, return that + LOG.debug("Race during insertion of state for cookie {} shard {}", cookie, shardName); + return raced.getStage(); + } - Preconditions.checkArgument(response instanceof ConnectClientSuccess, "Unhandled response {}", response); - final ConnectClientSuccess success = (ConnectClientSuccess) response; + // We have succeeded in populating the map, now we need to take care of pruning the entry if it fails to + // complete + final CompletionStage stage = toInsert.getStage(); + stage.whenComplete((info, failure) -> { + if (failure != null) { + LOG.debug("Resolution of cookie {} shard {} failed, removing state", cookie, shardName, failure); + backends.remove(cookie, toInsert); - return new ShardBackendInfo(success.getBackend(), - nextSessionId.getAndIncrement(), success.getVersion(), shardName, UnsignedLong.fromLongBits(cookie), - success.getDataTree(), success.getMaxMessages()); + // Remove cache state in case someone else forgot to invalidate it + flushCache(shardName); + } }); - } - @Override - public CompletionStage getBackendInfo(final Long cookie) { - return backends.computeIfAbsent(cookie, key -> new Entry(resolveBackendInfo(key))).getStage(); + return stage; } @Override - public CompletionStage refreshBackendInfo(final Long cookie, + public CompletionStage refreshBackendInfo(final Long cookie, final ShardBackendInfo staleInfo) { - final Entry existing = backends.get(cookie); + final ShardState existing = backends.get(cookie); if (existing != null) { if (!staleInfo.equals(existing.getResult())) { return existing.getStage(); } LOG.debug("Invalidating backend information {}", staleInfo); - actorContext.getPrimaryShardInfoCache().remove(staleInfo.getShardName()); + flushCache(staleInfo.getName()); - LOG.trace("Invalidated cache %s -> %s", Long.toUnsignedString(cookie), staleInfo); + LOG.trace("Invalidated cache {}", staleInfo); backends.remove(cookie, existing); } return getBackendInfo(cookie); } + + @Override + public void close() { + shardAvailabilityChangesRegFuture.onComplete(new OnComplete() { + @Override + public void onComplete(Throwable failure, Registration reply) { + reply.close(); + } + }, ExecutionContexts.global()); + } + + @Override + public String resolveCookieName(Long cookie) { + return verifyNotNull(shards.inverse().get(cookie), "Unexpected null cookie: %s", cookie); + } }