opendaylight/md-sal/sal-distributed-datastore/src/main/java/org/opendaylight/controller/cluster/databroker/actors/dds/AbstractProxyTransaction.java

   1 /*
   2  * Copyright (c) 2016 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.controller.cluster.databroker.actors.dds;
   9
  10 import akka.actor.ActorRef;
  11 import com.google.common.base.Optional;
  12 import com.google.common.base.Preconditions;
  13 import com.google.common.base.Throwables;
  14 import com.google.common.base.Verify;
  15 import com.google.common.util.concurrent.CheckedFuture;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.SettableFuture;
  18 import java.util.ArrayDeque;
  19 import java.util.Deque;
  20 import java.util.Iterator;
  21 import java.util.concurrent.CountDownLatch;
  22 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
  23 import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
  24 import java.util.function.Consumer;
  25 import javax.annotation.Nonnull;
  26 import javax.annotation.Nullable;
  27 import javax.annotation.concurrent.GuardedBy;
  28 import javax.annotation.concurrent.NotThreadSafe;
  29 import org.opendaylight.controller.cluster.access.client.ConnectionEntry;
  30 import org.opendaylight.controller.cluster.access.commands.TransactionAbortRequest;
  31 import org.opendaylight.controller.cluster.access.commands.TransactionAbortSuccess;
  32 import org.opendaylight.controller.cluster.access.commands.TransactionCanCommitSuccess;
  33 import org.opendaylight.controller.cluster.access.commands.TransactionCommitSuccess;
  34 import org.opendaylight.controller.cluster.access.commands.TransactionDoCommitRequest;
  35 import org.opendaylight.controller.cluster.access.commands.TransactionPreCommitRequest;
  36 import org.opendaylight.controller.cluster.access.commands.TransactionPreCommitSuccess;
  37 import org.opendaylight.controller.cluster.access.commands.TransactionRequest;
  38 import org.opendaylight.controller.cluster.access.concepts.Request;
  39 import org.opendaylight.controller.cluster.access.concepts.RequestFailure;
  40 import org.opendaylight.controller.cluster.access.concepts.Response;
  41 import org.opendaylight.controller.cluster.access.concepts.TransactionIdentifier;
  42 import org.opendaylight.mdsal.common.api.ReadFailedException;
  43 import org.opendaylight.yangtools.concepts.Identifiable;
  44 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
  45 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
  46 import org.slf4j.Logger;
  47 import org.slf4j.LoggerFactory;
  48
  49 /**
  50  * Class translating transaction operations towards a particular backend shard.
  51  *
  52  * <p>
  53  * This class is not safe to access from multiple application threads, as is usual for transactions. Internal state
  54  * transitions coming from interactions with backend are expected to be thread-safe.
  55  *
  56  * <p>
  57  * This class interacts with the queueing mechanism in ClientActorBehavior, hence once we arrive at a decision
  58  * to use either a local or remote implementation, we are stuck with it. We can re-evaluate on the next transaction.
  59  *
  60  * @author Robert Varga
  61  */
  62 abstract class AbstractProxyTransaction implements Identifiable<TransactionIdentifier> {
  63     /**
  64      * Marker object used instead of read-type of requests, which are satisfied only once. This has a lower footprint
  65      * and allows compressing multiple requests into a single entry.
  66      */
  67     @NotThreadSafe
  68     private static final class IncrementSequence {
  69         private long delta = 1;
  70
  71         long getDelta() {
  72             return delta;
  73         }
  74
  75         void incrementDelta() {
  76             delta++;
  77         }
  78     }
  79
  80     // Generic state base class. Direct instances are used for fast paths, sub-class is used for successor transitions
  81     private static class State {
  82         private final String string;
  83
  84         State(final String string) {
  85             this.string = Preconditions.checkNotNull(string);
  86         }
  87
  88         @Override
  89         public final String toString() {
  90             return string;
  91         }
  92     }
  93
  94     // State class used when a successor has interfered. Contains coordinator latch, the successor and previous state
  95     private static final class SuccessorState extends State {
  96         private final CountDownLatch latch = new CountDownLatch(1);
  97         private AbstractProxyTransaction successor;
  98         private State prevState;
  99
 100         SuccessorState() {
 101             super("successor");
 102         }
 103
 104         // Synchronize with succession process and return the successor
 105         AbstractProxyTransaction await() {
 106             try {
 107                 latch.await();
 108             } catch (InterruptedException e) {
 109                 LOG.warn("Interrupted while waiting for latch of {}", successor);
 110                 throw Throwables.propagate(e);
 111             }
 112             return successor;
 113         }
 114
 115         void finish() {
 116             latch.countDown();
 117         }
 118
 119         State getPrevState() {
 120             return prevState;
 121         }
 122
 123         void setPrevState(final State prevState) {
 124             Verify.verify(this.prevState == null);
 125             this.prevState = Preconditions.checkNotNull(prevState);
 126         }
 127
 128         // To be called from safe contexts, where successor is known to be completed
 129         AbstractProxyTransaction getSuccessor() {
 130             return Verify.verifyNotNull(successor);
 131         }
 132
 133         void setSuccessor(final AbstractProxyTransaction successor) {
 134             Verify.verify(this.successor == null);
 135             this.successor = Preconditions.checkNotNull(successor);
 136         }
 137     }
 138
 139     private static final Logger LOG = LoggerFactory.getLogger(AbstractProxyTransaction.class);
 140     private static final AtomicIntegerFieldUpdater<AbstractProxyTransaction> SEALED_UPDATER =
 141             AtomicIntegerFieldUpdater.newUpdater(AbstractProxyTransaction.class, "sealed");
 142     private static final AtomicReferenceFieldUpdater<AbstractProxyTransaction, State> STATE_UPDATER =
 143             AtomicReferenceFieldUpdater.newUpdater(AbstractProxyTransaction.class, State.class, "state");
 144     private static final State OPEN = new State("open");
 145     private static final State SEALED = new State("sealed");
 146     private static final State FLUSHED = new State("flushed");
 147
 148     // Touched from client actor thread only
 149     private final Deque<Object> successfulRequests = new ArrayDeque<>();
 150     private final ProxyHistory parent;
 151
 152     // Accessed from user thread only, which may not access this object concurrently
 153     private long sequence;
 154
 155     /*
 156      * Atomic state-keeping is required to synchronize the process of propagating completed transaction state towards
 157      * the backend -- which may include a successor.
 158      *
 159      * Successor, unlike {@link AbstractProxyTransaction#seal()} is triggered from the client actor thread, which means
 160      * the successor placement needs to be atomic with regard to the application thread.
 161      *
 162      * In the common case, the application thread performs performs the seal operations and then "immediately" sends
 163      * the corresponding message. The uncommon case is when the seal and send operations race with a connect completion
 164      * or timeout, when a successor is injected.
 165      *
 166      * This leaves the problem of needing to completely transferring state just after all queued messages are replayed
 167      * after a successor was injected, so that it can be properly sealed if we are racing. Further complication comes
 168      * from lock ordering, where the successor injection works with a locked queue and locks proxy objects -- leading
 169      * to a potential AB-BA deadlock in case of a naive implementation.
 170      *
 171      * For tracking user-visible state we use a single volatile int, which is flipped atomically from 0 to 1 exactly
 172      * once in {@link AbstractProxyTransaction#seal()}. That keeps common operations fast, as they need to perform
 173      * only a single volatile read to assert state correctness.
 174      *
 175      * For synchronizing client actor (successor-injecting) and user (commit-driving) thread, we keep a separate state
 176      * variable. It uses pre-allocated objects for fast paths (i.e. no successor present) and a per-transition object
 177      * for slow paths (when successor is injected/present).
 178      */
 179     private volatile int sealed = 0;
 180     private volatile State state = OPEN;
 181
 182     AbstractProxyTransaction(final ProxyHistory parent) {
 183         this.parent = Preconditions.checkNotNull(parent);
 184     }
 185
 186     final ActorRef localActor() {
 187         return parent.localActor();
 188     }
 189
 190     private void incrementSequence(final long delta) {
 191         sequence += delta;
 192         LOG.debug("Transaction {} incremented sequence to {}", this, sequence);
 193     }
 194
 195     final long nextSequence() {
 196         final long ret = sequence++;
 197         LOG.debug("Transaction {} allocated sequence {}", this, ret);
 198         return ret;
 199     }
 200
 201     final void delete(final YangInstanceIdentifier path) {
 202         checkReadWrite();
 203         checkNotSealed();
 204         doDelete(path);
 205     }
 206
 207     final void merge(final YangInstanceIdentifier path, final NormalizedNode<?, ?> data) {
 208         checkReadWrite();
 209         checkNotSealed();
 210         doMerge(path, data);
 211     }
 212
 213     final void write(final YangInstanceIdentifier path, final NormalizedNode<?, ?> data) {
 214         checkReadWrite();
 215         checkNotSealed();
 216         doWrite(path, data);
 217     }
 218
 219     final CheckedFuture<Boolean, ReadFailedException> exists(final YangInstanceIdentifier path) {
 220         checkNotSealed();
 221         return doExists(path);
 222     }
 223
 224     final CheckedFuture<Optional<NormalizedNode<?, ?>>, ReadFailedException> read(final YangInstanceIdentifier path) {
 225         checkNotSealed();
 226         return doRead(path);
 227     }
 228
 229     final void sendRequest(final TransactionRequest<?> request, final Consumer<Response<?, ?>> callback) {
 230         LOG.debug("Transaction proxy {} sending request {} callback {}", this, request, callback);
 231         parent.sendRequest(request, callback);
 232     }
 233
 234     /**
 235      * Seal this transaction before it is either committed or aborted.
 236      */
 237     final void seal() {
 238         // Transition user-visible state first
 239         final boolean success = SEALED_UPDATER.compareAndSet(this, 0, 1);
 240         Preconditions.checkState(success, "Proxy %s was already sealed", getIdentifier());
 241         internalSeal();
 242     }
 243
 244     final void ensureSealed() {
 245         if (SEALED_UPDATER.compareAndSet(this, 0, 1)) {
 246             internalSeal();
 247         }
 248     }
 249
 250     private void internalSeal() {
 251         doSeal();
 252         parent.onTransactionSealed(this);
 253
 254         // Now deal with state transfer, which can occur via successor or a follow-up canCommit() or directCommit().
 255         if (!STATE_UPDATER.compareAndSet(this, OPEN, SEALED)) {
 256             // Slow path: wait for the successor to complete
 257             final AbstractProxyTransaction successor = awaitSuccessor();
 258
 259             // At this point the successor has completed transition and is possibly visible by the user thread, which is
 260             // still stuck here. The successor has not seen final part of our state, nor the fact it is sealed.
 261             // Propagate state and seal the successor.
 262             flushState(successor);
 263             successor.ensureSealed();
 264         }
 265     }
 266
 267     private void checkNotSealed() {
 268         Preconditions.checkState(sealed == 0, "Transaction %s has already been sealed", getIdentifier());
 269     }
 270
 271     private void checkSealed() {
 272         Preconditions.checkState(sealed != 0, "Transaction %s has not been sealed yet", getIdentifier());
 273     }
 274
 275     private SuccessorState getSuccessorState() {
 276         final State local = state;
 277         Verify.verify(local instanceof SuccessorState, "State %s has unexpected class", local);
 278         return (SuccessorState) local;
 279     }
 280
 281     private void checkReadWrite() {
 282         if (isSnapshotOnly()) {
 283             throw new UnsupportedOperationException("Transaction " + getIdentifier() + " is a read-only snapshot");
 284         }
 285     }
 286
 287     final void recordSuccessfulRequest(final @Nonnull TransactionRequest<?> req) {
 288         successfulRequests.add(Verify.verifyNotNull(req));
 289     }
 290
 291     final void recordFinishedRequest() {
 292         final Object last = successfulRequests.peekLast();
 293         if (last instanceof IncrementSequence) {
 294             ((IncrementSequence) last).incrementDelta();
 295         } else {
 296             successfulRequests.addLast(new IncrementSequence());
 297         }
 298     }
 299
 300     /**
 301      * Abort this transaction. This is invoked only for read-only transactions and will result in an explicit message
 302      * being sent to the backend.
 303      */
 304     final void abort() {
 305         checkNotSealed();
 306         doAbort();
 307         parent.abortTransaction(this);
 308     }
 309
 310     final void abort(final VotingFuture<Void> ret) {
 311         checkSealed();
 312
 313         sendAbort(t -> {
 314             if (t instanceof TransactionAbortSuccess) {
 315                 ret.voteYes();
 316             } else if (t instanceof RequestFailure) {
 317                 ret.voteNo(((RequestFailure<?, ?>) t).getCause());
 318             } else {
 319                 ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass()));
 320             }
 321
 322             // This is a terminal request, hence we do not need to record it
 323             LOG.debug("Transaction {} abort completed", this);
 324             parent.completeTransaction(this);
 325         });
 326     }
 327
 328     final void sendAbort(final Consumer<Response<?, ?>> callback) {
 329         sendRequest(new TransactionAbortRequest(getIdentifier(), nextSequence(), localActor()), callback);
 330     }
 331
 332     /**
 333      * Commit this transaction, possibly in a coordinated fashion.
 334      *
 335      * @param coordinated True if this transaction should be coordinated across multiple participants.
 336      * @return Future completion
 337      */
 338     final ListenableFuture<Boolean> directCommit() {
 339         checkReadWrite();
 340         checkSealed();
 341
 342         // Precludes startReconnect() from interfering with the fast path
 343         synchronized (this) {
 344             if (STATE_UPDATER.compareAndSet(this, SEALED, FLUSHED)) {
 345                 final SettableFuture<Boolean> ret = SettableFuture.create();
 346                 sendRequest(Verify.verifyNotNull(commitRequest(false)), t -> {
 347                     if (t instanceof TransactionCommitSuccess) {
 348                         ret.set(Boolean.TRUE);
 349                     } else if (t instanceof RequestFailure) {
 350                         ret.setException(((RequestFailure<?, ?>) t).getCause());
 351                     } else {
 352                         ret.setException(new IllegalStateException("Unhandled response " + t.getClass()));
 353                     }
 354
 355                     // This is a terminal request, hence we do not need to record it
 356                     LOG.debug("Transaction {} directCommit completed", this);
 357                     parent.completeTransaction(this);
 358                 });
 359
 360                 return ret;
 361             }
 362         }
 363
 364         // We have had some interference with successor injection, wait for it to complete and defer to the successor.
 365         return awaitSuccessor().directCommit();
 366     }
 367
 368     final void canCommit(final VotingFuture<?> ret) {
 369         checkReadWrite();
 370         checkSealed();
 371
 372         // Precludes startReconnect() from interfering with the fast path
 373         synchronized (this) {
 374             if (STATE_UPDATER.compareAndSet(this, SEALED, FLUSHED)) {
 375                 final TransactionRequest<?> req = Verify.verifyNotNull(commitRequest(true));
 376
 377                 sendRequest(req, t -> {
 378                     if (t instanceof TransactionCanCommitSuccess) {
 379                         ret.voteYes();
 380                     } else if (t instanceof RequestFailure) {
 381                         ret.voteNo(((RequestFailure<?, ?>) t).getCause());
 382                     } else {
 383                         ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass()));
 384                     }
 385
 386                     recordSuccessfulRequest(req);
 387                     LOG.debug("Transaction {} canCommit completed", this);
 388                 });
 389
 390                 return;
 391             }
 392         }
 393
 394         // We have had some interference with successor injection, wait for it to complete and defer to the successor.
 395         awaitSuccessor().canCommit(ret);
 396     }
 397
 398     private AbstractProxyTransaction awaitSuccessor() {
 399         return getSuccessorState().await();
 400     }
 401
 402     final void preCommit(final VotingFuture<?> ret) {
 403         checkReadWrite();
 404         checkSealed();
 405
 406         final TransactionRequest<?> req = new TransactionPreCommitRequest(getIdentifier(), nextSequence(),
 407             localActor());
 408         sendRequest(req, t -> {
 409             if (t instanceof TransactionPreCommitSuccess) {
 410                 ret.voteYes();
 411             } else if (t instanceof RequestFailure) {
 412                 ret.voteNo(((RequestFailure<?, ?>) t).getCause());
 413             } else {
 414                 ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass()));
 415             }
 416
 417             recordSuccessfulRequest(req);
 418             LOG.debug("Transaction {} preCommit completed", this);
 419         });
 420     }
 421
 422     final void doCommit(final VotingFuture<?> ret) {
 423         checkReadWrite();
 424         checkSealed();
 425
 426         sendRequest(new TransactionDoCommitRequest(getIdentifier(), nextSequence(), localActor()), t -> {
 427             if (t instanceof TransactionCommitSuccess) {
 428                 ret.voteYes();
 429             } else if (t instanceof RequestFailure) {
 430                 ret.voteNo(((RequestFailure<?, ?>) t).getCause());
 431             } else {
 432                 ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass()));
 433             }
 434
 435             LOG.debug("Transaction {} doCommit completed", this);
 436             parent.completeTransaction(this);
 437         });
 438     }
 439
 440     // Called with the connection unlocked
 441     final synchronized void startReconnect() {
 442         // At this point canCommit/directCommit are blocked, we assert a new successor state, retrieving the previous
 443         // state. This method is called with the queue still unlocked.
 444         final SuccessorState nextState = new SuccessorState();
 445         final State prevState = STATE_UPDATER.getAndSet(this, nextState);
 446
 447         LOG.debug("Start reconnect of proxy {} previous state {}", this, prevState);
 448         Verify.verify(!(prevState instanceof SuccessorState), "Proxy %s duplicate reconnect attempt after %s", this,
 449             prevState);
 450
 451         // We have asserted a slow-path state, seal(), canCommit(), directCommit() are forced to slow paths, which will
 452         // wait until we unblock nextState's latch before accessing state. Now we record prevState for later use and we
 453         // are done.
 454         nextState.setPrevState(prevState);
 455     }
 456
 457     // Called with the connection locked
 458     final void replayMessages(final AbstractProxyTransaction successor,
 459             final Iterable<ConnectionEntry> enqueuedEntries) {
 460         final SuccessorState local = getSuccessorState();
 461         local.setSuccessor(successor);
 462
 463         // Replay successful requests first
 464         for (Object obj : successfulRequests) {
 465             if (obj instanceof TransactionRequest) {
 466                 LOG.debug("Forwarding successful request {} to successor {}", obj, successor);
 467                 successor.handleForwardedRemoteRequest((TransactionRequest<?>) obj, null);
 468             } else {
 469                 Verify.verify(obj instanceof IncrementSequence);
 470                 successor.incrementSequence(((IncrementSequence) obj).getDelta());
 471             }
 472         }
 473         LOG.debug("{} replayed {} successful requests", getIdentifier(), successfulRequests.size());
 474         successfulRequests.clear();
 475
 476         // Now replay whatever is in the connection
 477         final Iterator<ConnectionEntry> it = enqueuedEntries.iterator();
 478         while (it.hasNext()) {
 479             final ConnectionEntry e = it.next();
 480             final Request<?, ?> req = e.getRequest();
 481
 482             if (getIdentifier().equals(req.getTarget())) {
 483                 Verify.verify(req instanceof TransactionRequest, "Unhandled request %s", req);
 484                 LOG.debug("Forwarding queued request{} to successor {}", req, successor);
 485                 successor.handleForwardedRemoteRequest((TransactionRequest<?>) req, e.getCallback());
 486                 it.remove();
 487             }
 488         }
 489
 490         /*
 491          * Check the state at which we have started the reconnect attempt. State transitions triggered while we were
 492          * reconnecting have been forced to slow paths, which will be unlocked once we unblock the state latch
 493          * at the end of this method.
 494          */
 495         final State prevState = local.getPrevState();
 496         if (SEALED.equals(prevState)) {
 497             LOG.debug("Proxy {} reconnected while being sealed, propagating state to successor {}", this, successor);
 498             flushState(successor);
 499             successor.ensureSealed();
 500         }
 501     }
 502
 503     // Called with the connection locked
 504     final void finishReconnect() {
 505         final SuccessorState local = getSuccessorState();
 506         LOG.debug("Finishing reconnect of proxy {}", this);
 507
 508         // All done, release the latch, unblocking seal() and canCommit() slow paths
 509         local.finish();
 510     }
 511
 512     /**
 513      * Invoked from a retired connection for requests which have been in-flight and need to be re-adjusted
 514      * and forwarded to the successor connection.
 515      *
 516      * @param request Request to be forwarded
 517      * @param callback Original callback
 518      */
 519     final void replayRequest(final TransactionRequest<?> request, final Consumer<Response<?, ?>> callback) {
 520         final AbstractProxyTransaction successor = getSuccessorState().getSuccessor();
 521
 522         if (successor instanceof LocalProxyTransaction) {
 523             forwardToLocal((LocalProxyTransaction)successor, request, callback);
 524         } else if (successor instanceof RemoteProxyTransaction) {
 525             forwardToRemote((RemoteProxyTransaction)successor, request, callback);
 526         } else {
 527             throw new IllegalStateException("Unhandled successor " + successor);
 528         }
 529     }
 530
 531     abstract boolean isSnapshotOnly();
 532
 533     abstract void doDelete(final YangInstanceIdentifier path);
 534
 535     abstract void doMerge(final YangInstanceIdentifier path, final NormalizedNode<?, ?> data);
 536
 537     abstract void doWrite(final YangInstanceIdentifier path, final NormalizedNode<?, ?> data);
 538
 539     abstract CheckedFuture<Boolean, ReadFailedException> doExists(final YangInstanceIdentifier path);
 540
 541     abstract CheckedFuture<Optional<NormalizedNode<?, ?>>, ReadFailedException> doRead(
 542             final YangInstanceIdentifier path);
 543
 544     abstract void doSeal();
 545
 546     abstract void doAbort();
 547
 548     @GuardedBy("this")
 549     abstract void flushState(AbstractProxyTransaction successor);
 550
 551     abstract TransactionRequest<?> commitRequest(boolean coordinated);
 552
 553     /**
 554      * Invoked from {@link RemoteProxyTransaction} when it replays its successful requests to its successor. There is
 555      * no equivalent of this call from {@link LocalProxyTransaction} because it does not send a request until all
 556      * operations are packaged in the message.
 557      *
 558      * <p>
 559      * Note: this method is invoked by the predecessor on the successor.
 560      *
 561      * @param request Request which needs to be forwarded
 562      * @param callback Callback to be invoked once the request completes
 563      */
 564     abstract void handleForwardedRemoteRequest(TransactionRequest<?> request,
 565             @Nullable Consumer<Response<?, ?>> callback);
 566
 567     /**
 568      * Replay a request originating in this proxy to a successor remote proxy.
 569      */
 570     abstract void forwardToRemote(RemoteProxyTransaction successor, TransactionRequest<?> request,
 571             Consumer<Response<?, ?>> callback);
 572
 573     /**
 574      * Replay a request originating in this proxy to a successor local proxy.
 575      */
 576     abstract void forwardToLocal(LocalProxyTransaction successor, TransactionRequest<?> request,
 577             Consumer<Response<?, ?>> callback);
 578 }