cb5a6cf229fae47226c0577f62f79ea2de5afcbe
[controller.git] / opendaylight / md-sal / sal-distributed-datastore / src / main / java / org / opendaylight / controller / cluster / databroker / actors / dds / AbstractProxyTransaction.java
1 /*
2  * Copyright (c) 2016 Cisco Systems, Inc. and others.  All rights reserved.
3  *
4  * This program and the accompanying materials are made available under the
5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6  * and is available at http://www.eclipse.org/legal/epl-v10.html
7  */
8 package org.opendaylight.controller.cluster.databroker.actors.dds;
9
10 import akka.actor.ActorRef;
11 import com.google.common.base.MoreObjects;
12 import com.google.common.base.Optional;
13 import com.google.common.base.Preconditions;
14 import com.google.common.base.Throwables;
15 import com.google.common.base.Verify;
16 import com.google.common.util.concurrent.CheckedFuture;
17 import com.google.common.util.concurrent.ListenableFuture;
18 import com.google.common.util.concurrent.SettableFuture;
19 import java.util.ArrayDeque;
20 import java.util.Deque;
21 import java.util.Iterator;
22 import java.util.concurrent.CountDownLatch;
23 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
24 import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
25 import java.util.function.Consumer;
26 import javax.annotation.Nonnull;
27 import javax.annotation.Nullable;
28 import javax.annotation.concurrent.GuardedBy;
29 import javax.annotation.concurrent.NotThreadSafe;
30 import org.opendaylight.controller.cluster.access.client.ConnectionEntry;
31 import org.opendaylight.controller.cluster.access.commands.TransactionAbortRequest;
32 import org.opendaylight.controller.cluster.access.commands.TransactionAbortSuccess;
33 import org.opendaylight.controller.cluster.access.commands.TransactionCanCommitSuccess;
34 import org.opendaylight.controller.cluster.access.commands.TransactionCommitSuccess;
35 import org.opendaylight.controller.cluster.access.commands.TransactionDoCommitRequest;
36 import org.opendaylight.controller.cluster.access.commands.TransactionPreCommitRequest;
37 import org.opendaylight.controller.cluster.access.commands.TransactionPreCommitSuccess;
38 import org.opendaylight.controller.cluster.access.commands.TransactionPurgeRequest;
39 import org.opendaylight.controller.cluster.access.commands.TransactionRequest;
40 import org.opendaylight.controller.cluster.access.concepts.Request;
41 import org.opendaylight.controller.cluster.access.concepts.RequestFailure;
42 import org.opendaylight.controller.cluster.access.concepts.Response;
43 import org.opendaylight.controller.cluster.access.concepts.TransactionIdentifier;
44 import org.opendaylight.mdsal.common.api.ReadFailedException;
45 import org.opendaylight.yangtools.concepts.Identifiable;
46 import org.opendaylight.yangtools.yang.data.api.YangInstanceIdentifier;
47 import org.opendaylight.yangtools.yang.data.api.schema.NormalizedNode;
48 import org.slf4j.Logger;
49 import org.slf4j.LoggerFactory;
50
51 /**
52  * Class translating transaction operations towards a particular backend shard.
53  *
54  * <p>
55  * This class is not safe to access from multiple application threads, as is usual for transactions. Internal state
56  * transitions coming from interactions with backend are expected to be thread-safe.
57  *
58  * <p>
59  * This class interacts with the queueing mechanism in ClientActorBehavior, hence once we arrive at a decision
60  * to use either a local or remote implementation, we are stuck with it. We can re-evaluate on the next transaction.
61  *
62  * @author Robert Varga
63  */
64 abstract class AbstractProxyTransaction implements Identifiable<TransactionIdentifier> {
65     /**
66      * Marker object used instead of read-type of requests, which are satisfied only once. This has a lower footprint
67      * and allows compressing multiple requests into a single entry.
68      */
69     @NotThreadSafe
70     private static final class IncrementSequence {
71         private long delta = 1;
72
73         long getDelta() {
74             return delta;
75         }
76
77         void incrementDelta() {
78             delta++;
79         }
80     }
81
82     // Generic state base class. Direct instances are used for fast paths, sub-class is used for successor transitions
83     private static class State {
84         private final String string;
85
86         State(final String string) {
87             this.string = Preconditions.checkNotNull(string);
88         }
89
90         @Override
91         public final String toString() {
92             return string;
93         }
94     }
95
96     // State class used when a successor has interfered. Contains coordinator latch, the successor and previous state
97     private static final class SuccessorState extends State {
98         private final CountDownLatch latch = new CountDownLatch(1);
99         private AbstractProxyTransaction successor;
100         private State prevState;
101
102         SuccessorState() {
103             super("successor");
104         }
105
106         // Synchronize with succession process and return the successor
107         AbstractProxyTransaction await() {
108             try {
109                 latch.await();
110             } catch (InterruptedException e) {
111                 LOG.warn("Interrupted while waiting for latch of {}", successor);
112                 throw Throwables.propagate(e);
113             }
114             return successor;
115         }
116
117         void finish() {
118             latch.countDown();
119         }
120
121         State getPrevState() {
122             return prevState;
123         }
124
125         void setPrevState(final State prevState) {
126             Verify.verify(this.prevState == null);
127             this.prevState = Preconditions.checkNotNull(prevState);
128         }
129
130         // To be called from safe contexts, where successor is known to be completed
131         AbstractProxyTransaction getSuccessor() {
132             return Verify.verifyNotNull(successor);
133         }
134
135         void setSuccessor(final AbstractProxyTransaction successor) {
136             Verify.verify(this.successor == null);
137             this.successor = Preconditions.checkNotNull(successor);
138         }
139     }
140
141     private static final Logger LOG = LoggerFactory.getLogger(AbstractProxyTransaction.class);
142     private static final AtomicIntegerFieldUpdater<AbstractProxyTransaction> SEALED_UPDATER =
143             AtomicIntegerFieldUpdater.newUpdater(AbstractProxyTransaction.class, "sealed");
144     private static final AtomicReferenceFieldUpdater<AbstractProxyTransaction, State> STATE_UPDATER =
145             AtomicReferenceFieldUpdater.newUpdater(AbstractProxyTransaction.class, State.class, "state");
146     private static final State OPEN = new State("open");
147     private static final State SEALED = new State("sealed");
148     private static final State FLUSHED = new State("flushed");
149
150     // Touched from client actor thread only
151     private final Deque<Object> successfulRequests = new ArrayDeque<>();
152     private final ProxyHistory parent;
153
154     // Accessed from user thread only, which may not access this object concurrently
155     private long sequence;
156
157     /*
158      * Atomic state-keeping is required to synchronize the process of propagating completed transaction state towards
159      * the backend -- which may include a successor.
160      *
161      * Successor, unlike {@link AbstractProxyTransaction#seal()} is triggered from the client actor thread, which means
162      * the successor placement needs to be atomic with regard to the application thread.
163      *
164      * In the common case, the application thread performs performs the seal operations and then "immediately" sends
165      * the corresponding message. The uncommon case is when the seal and send operations race with a connect completion
166      * or timeout, when a successor is injected.
167      *
168      * This leaves the problem of needing to completely transferring state just after all queued messages are replayed
169      * after a successor was injected, so that it can be properly sealed if we are racing. Further complication comes
170      * from lock ordering, where the successor injection works with a locked queue and locks proxy objects -- leading
171      * to a potential AB-BA deadlock in case of a naive implementation.
172      *
173      * For tracking user-visible state we use a single volatile int, which is flipped atomically from 0 to 1 exactly
174      * once in {@link AbstractProxyTransaction#seal()}. That keeps common operations fast, as they need to perform
175      * only a single volatile read to assert state correctness.
176      *
177      * For synchronizing client actor (successor-injecting) and user (commit-driving) thread, we keep a separate state
178      * variable. It uses pre-allocated objects for fast paths (i.e. no successor present) and a per-transition object
179      * for slow paths (when successor is injected/present).
180      */
181     private volatile int sealed = 0;
182     private volatile State state = OPEN;
183
184     AbstractProxyTransaction(final ProxyHistory parent) {
185         this.parent = Preconditions.checkNotNull(parent);
186     }
187
188     final ActorRef localActor() {
189         return parent.localActor();
190     }
191
192     private void incrementSequence(final long delta) {
193         sequence += delta;
194         LOG.debug("Transaction {} incremented sequence to {}", this, sequence);
195     }
196
197     final long nextSequence() {
198         final long ret = sequence++;
199         LOG.debug("Transaction {} allocated sequence {}", this, ret);
200         return ret;
201     }
202
203     final void delete(final YangInstanceIdentifier path) {
204         checkReadWrite();
205         checkNotSealed();
206         doDelete(path);
207     }
208
209     final void merge(final YangInstanceIdentifier path, final NormalizedNode<?, ?> data) {
210         checkReadWrite();
211         checkNotSealed();
212         doMerge(path, data);
213     }
214
215     final void write(final YangInstanceIdentifier path, final NormalizedNode<?, ?> data) {
216         checkReadWrite();
217         checkNotSealed();
218         doWrite(path, data);
219     }
220
221     final CheckedFuture<Boolean, ReadFailedException> exists(final YangInstanceIdentifier path) {
222         checkNotSealed();
223         return doExists(path);
224     }
225
226     final CheckedFuture<Optional<NormalizedNode<?, ?>>, ReadFailedException> read(final YangInstanceIdentifier path) {
227         checkNotSealed();
228         return doRead(path);
229     }
230
231     final void sendRequest(final TransactionRequest<?> request, final Consumer<Response<?, ?>> callback) {
232         LOG.debug("Transaction proxy {} sending request {} callback {}", this, request, callback);
233         parent.sendRequest(request, callback);
234     }
235
236     /**
237      * Seal this transaction before it is either committed or aborted.
238      */
239     final void seal() {
240         // Transition user-visible state first
241         final boolean success = SEALED_UPDATER.compareAndSet(this, 0, 1);
242         Preconditions.checkState(success, "Proxy %s was already sealed", getIdentifier());
243         internalSeal();
244     }
245
246     final void ensureSealed() {
247         if (SEALED_UPDATER.compareAndSet(this, 0, 1)) {
248             internalSeal();
249         }
250     }
251
252     private void internalSeal() {
253         doSeal();
254         parent.onTransactionSealed(this);
255
256         // Now deal with state transfer, which can occur via successor or a follow-up canCommit() or directCommit().
257         if (!STATE_UPDATER.compareAndSet(this, OPEN, SEALED)) {
258             // Slow path: wait for the successor to complete
259             final AbstractProxyTransaction successor = awaitSuccessor();
260
261             // At this point the successor has completed transition and is possibly visible by the user thread, which is
262             // still stuck here. The successor has not seen final part of our state, nor the fact it is sealed.
263             // Propagate state and seal the successor.
264             flushState(successor);
265             successor.ensureSealed();
266         }
267     }
268
269     private void checkNotSealed() {
270         Preconditions.checkState(sealed == 0, "Transaction %s has already been sealed", getIdentifier());
271     }
272
273     private void checkSealed() {
274         Preconditions.checkState(sealed != 0, "Transaction %s has not been sealed yet", getIdentifier());
275     }
276
277     private SuccessorState getSuccessorState() {
278         final State local = state;
279         Verify.verify(local instanceof SuccessorState, "State %s has unexpected class", local);
280         return (SuccessorState) local;
281     }
282
283     private void checkReadWrite() {
284         if (isSnapshotOnly()) {
285             throw new UnsupportedOperationException("Transaction " + getIdentifier() + " is a read-only snapshot");
286         }
287     }
288
289     final void recordSuccessfulRequest(final @Nonnull TransactionRequest<?> req) {
290         successfulRequests.add(Verify.verifyNotNull(req));
291     }
292
293     final void recordFinishedRequest() {
294         final Object last = successfulRequests.peekLast();
295         if (last instanceof IncrementSequence) {
296             ((IncrementSequence) last).incrementDelta();
297         } else {
298             successfulRequests.addLast(new IncrementSequence());
299         }
300     }
301
302     /**
303      * Abort this transaction. This is invoked only for read-only transactions and will result in an explicit message
304      * being sent to the backend.
305      */
306     final void abort() {
307         checkNotSealed();
308         doAbort();
309         parent.abortTransaction(this);
310     }
311
312     final void abort(final VotingFuture<Void> ret) {
313         checkSealed();
314
315         sendAbort(t -> {
316             if (t instanceof TransactionAbortSuccess) {
317                 ret.voteYes();
318             } else if (t instanceof RequestFailure) {
319                 ret.voteNo(((RequestFailure<?, ?>) t).getCause());
320             } else {
321                 ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass()));
322             }
323
324             // This is a terminal request, hence we do not need to record it
325             LOG.debug("Transaction {} abort completed", this);
326             purge();
327         });
328     }
329
330     final void sendAbort(final Consumer<Response<?, ?>> callback) {
331         sendRequest(new TransactionAbortRequest(getIdentifier(), nextSequence(), localActor()), callback);
332     }
333
334     /**
335      * Commit this transaction, possibly in a coordinated fashion.
336      *
337      * @param coordinated True if this transaction should be coordinated across multiple participants.
338      * @return Future completion
339      */
340     final ListenableFuture<Boolean> directCommit() {
341         checkReadWrite();
342         checkSealed();
343
344         // Precludes startReconnect() from interfering with the fast path
345         synchronized (this) {
346             if (STATE_UPDATER.compareAndSet(this, SEALED, FLUSHED)) {
347                 final SettableFuture<Boolean> ret = SettableFuture.create();
348                 sendRequest(Verify.verifyNotNull(commitRequest(false)), t -> {
349                     if (t instanceof TransactionCommitSuccess) {
350                         ret.set(Boolean.TRUE);
351                     } else if (t instanceof RequestFailure) {
352                         ret.setException(((RequestFailure<?, ?>) t).getCause());
353                     } else {
354                         ret.setException(new IllegalStateException("Unhandled response " + t.getClass()));
355                     }
356
357                     // This is a terminal request, hence we do not need to record it
358                     LOG.debug("Transaction {} directCommit completed", this);
359                     purge();
360                 });
361
362                 return ret;
363             }
364         }
365
366         // We have had some interference with successor injection, wait for it to complete and defer to the successor.
367         return awaitSuccessor().directCommit();
368     }
369
370     final void canCommit(final VotingFuture<?> ret) {
371         checkReadWrite();
372         checkSealed();
373
374         // Precludes startReconnect() from interfering with the fast path
375         synchronized (this) {
376             if (STATE_UPDATER.compareAndSet(this, SEALED, FLUSHED)) {
377                 final TransactionRequest<?> req = Verify.verifyNotNull(commitRequest(true));
378
379                 sendRequest(req, t -> {
380                     if (t instanceof TransactionCanCommitSuccess) {
381                         ret.voteYes();
382                     } else if (t instanceof RequestFailure) {
383                         ret.voteNo(((RequestFailure<?, ?>) t).getCause());
384                     } else {
385                         ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass()));
386                     }
387
388                     recordSuccessfulRequest(req);
389                     LOG.debug("Transaction {} canCommit completed", this);
390                 });
391
392                 return;
393             }
394         }
395
396         // We have had some interference with successor injection, wait for it to complete and defer to the successor.
397         awaitSuccessor().canCommit(ret);
398     }
399
400     private AbstractProxyTransaction awaitSuccessor() {
401         return getSuccessorState().await();
402     }
403
404     final void preCommit(final VotingFuture<?> ret) {
405         checkReadWrite();
406         checkSealed();
407
408         final TransactionRequest<?> req = new TransactionPreCommitRequest(getIdentifier(), nextSequence(),
409             localActor());
410         sendRequest(req, t -> {
411             if (t instanceof TransactionPreCommitSuccess) {
412                 ret.voteYes();
413             } else if (t instanceof RequestFailure) {
414                 ret.voteNo(((RequestFailure<?, ?>) t).getCause());
415             } else {
416                 ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass()));
417             }
418
419             onPreCommitComplete(req);
420         });
421     }
422
423     private void onPreCommitComplete(final TransactionRequest<?> req) {
424         /*
425          * The backend has agreed that the transaction has entered PRE_COMMIT phase, meaning it will be committed
426          * to storage after the timeout completes.
427          *
428          * All state has been replicated to the backend, hence we do not need to keep it around. Retain only
429          * the precommit request, so we know which request to use for resync.
430          */
431         LOG.debug("Transaction {} preCommit completed, clearing successfulRequests", this);
432         successfulRequests.clear();
433
434         // TODO: this works, but can contain some useless state (like batched operations). Create an empty
435         //       equivalent of this request and store that.
436         recordSuccessfulRequest(req);
437     }
438
439     final void doCommit(final VotingFuture<?> ret) {
440         checkReadWrite();
441         checkSealed();
442
443         sendRequest(new TransactionDoCommitRequest(getIdentifier(), nextSequence(), localActor()), t -> {
444             if (t instanceof TransactionCommitSuccess) {
445                 ret.voteYes();
446             } else if (t instanceof RequestFailure) {
447                 ret.voteNo(((RequestFailure<?, ?>) t).getCause());
448             } else {
449                 ret.voteNo(new IllegalStateException("Unhandled response " + t.getClass()));
450             }
451
452             LOG.debug("Transaction {} doCommit completed", this);
453             purge();
454         });
455     }
456
457     private void purge() {
458         successfulRequests.clear();
459
460         final TransactionRequest<?> req = new TransactionPurgeRequest(getIdentifier(), nextSequence(), localActor());
461         sendRequest(req, t -> {
462             LOG.debug("Transaction {} purge completed", this);
463             parent.completeTransaction(this);
464         });
465     }
466
467     // Called with the connection unlocked
468     final synchronized void startReconnect() {
469         // At this point canCommit/directCommit are blocked, we assert a new successor state, retrieving the previous
470         // state. This method is called with the queue still unlocked.
471         final SuccessorState nextState = new SuccessorState();
472         final State prevState = STATE_UPDATER.getAndSet(this, nextState);
473
474         LOG.debug("Start reconnect of proxy {} previous state {}", this, prevState);
475         Verify.verify(!(prevState instanceof SuccessorState), "Proxy %s duplicate reconnect attempt after %s", this,
476             prevState);
477
478         // We have asserted a slow-path state, seal(), canCommit(), directCommit() are forced to slow paths, which will
479         // wait until we unblock nextState's latch before accessing state. Now we record prevState for later use and we
480         // are done.
481         nextState.setPrevState(prevState);
482     }
483
484     // Called with the connection locked
485     final void replayMessages(final AbstractProxyTransaction successor,
486             final Iterable<ConnectionEntry> enqueuedEntries) {
487         final SuccessorState local = getSuccessorState();
488         local.setSuccessor(successor);
489
490         // Replay successful requests first
491         for (Object obj : successfulRequests) {
492             if (obj instanceof TransactionRequest) {
493                 LOG.debug("Forwarding successful request {} to successor {}", obj, successor);
494                 successor.handleForwardedRemoteRequest((TransactionRequest<?>) obj, null);
495             } else {
496                 Verify.verify(obj instanceof IncrementSequence);
497                 successor.incrementSequence(((IncrementSequence) obj).getDelta());
498             }
499         }
500         LOG.debug("{} replayed {} successful requests", getIdentifier(), successfulRequests.size());
501         successfulRequests.clear();
502
503         // Now replay whatever is in the connection
504         final Iterator<ConnectionEntry> it = enqueuedEntries.iterator();
505         while (it.hasNext()) {
506             final ConnectionEntry e = it.next();
507             final Request<?, ?> req = e.getRequest();
508
509             if (getIdentifier().equals(req.getTarget())) {
510                 Verify.verify(req instanceof TransactionRequest, "Unhandled request %s", req);
511                 LOG.debug("Forwarding queued request{} to successor {}", req, successor);
512                 successor.handleForwardedRemoteRequest((TransactionRequest<?>) req, e.getCallback());
513                 it.remove();
514             }
515         }
516
517         /*
518          * Check the state at which we have started the reconnect attempt. State transitions triggered while we were
519          * reconnecting have been forced to slow paths, which will be unlocked once we unblock the state latch
520          * at the end of this method.
521          */
522         final State prevState = local.getPrevState();
523         if (SEALED.equals(prevState)) {
524             LOG.debug("Proxy {} reconnected while being sealed, propagating state to successor {}", this, successor);
525             flushState(successor);
526             successor.ensureSealed();
527         }
528     }
529
530     // Called with the connection locked
531     final void finishReconnect() {
532         final SuccessorState local = getSuccessorState();
533         LOG.debug("Finishing reconnect of proxy {}", this);
534
535         // All done, release the latch, unblocking seal() and canCommit() slow paths
536         local.finish();
537     }
538
539     /**
540      * Invoked from a retired connection for requests which have been in-flight and need to be re-adjusted
541      * and forwarded to the successor connection.
542      *
543      * @param request Request to be forwarded
544      * @param callback Original callback
545      */
546     final void replayRequest(final TransactionRequest<?> request, final Consumer<Response<?, ?>> callback) {
547         final AbstractProxyTransaction successor = getSuccessorState().getSuccessor();
548
549         if (successor instanceof LocalProxyTransaction) {
550             forwardToLocal((LocalProxyTransaction)successor, request, callback);
551         } else if (successor instanceof RemoteProxyTransaction) {
552             forwardToRemote((RemoteProxyTransaction)successor, request, callback);
553         } else {
554             throw new IllegalStateException("Unhandled successor " + successor);
555         }
556     }
557
558     abstract boolean isSnapshotOnly();
559
560     abstract void doDelete(YangInstanceIdentifier path);
561
562     abstract void doMerge(YangInstanceIdentifier path, NormalizedNode<?, ?> data);
563
564     abstract void doWrite(YangInstanceIdentifier path, NormalizedNode<?, ?> data);
565
566     abstract CheckedFuture<Boolean, ReadFailedException> doExists(YangInstanceIdentifier path);
567
568     abstract CheckedFuture<Optional<NormalizedNode<?, ?>>, ReadFailedException> doRead(YangInstanceIdentifier path);
569
570     abstract void doSeal();
571
572     abstract void doAbort();
573
574     @GuardedBy("this")
575     abstract void flushState(AbstractProxyTransaction successor);
576
577     abstract TransactionRequest<?> commitRequest(boolean coordinated);
578
579     /**
580      * Invoked from {@link RemoteProxyTransaction} when it replays its successful requests to its successor. There is
581      * no equivalent of this call from {@link LocalProxyTransaction} because it does not send a request until all
582      * operations are packaged in the message.
583      *
584      * <p>
585      * Note: this method is invoked by the predecessor on the successor.
586      *
587      * @param request Request which needs to be forwarded
588      * @param callback Callback to be invoked once the request completes
589      */
590     abstract void handleForwardedRemoteRequest(TransactionRequest<?> request,
591             @Nullable Consumer<Response<?, ?>> callback);
592
593     /**
594      * Replay a request originating in this proxy to a successor remote proxy.
595      */
596     abstract void forwardToRemote(RemoteProxyTransaction successor, TransactionRequest<?> request,
597             Consumer<Response<?, ?>> callback);
598
599     /**
600      * Replay a request originating in this proxy to a successor local proxy.
601      */
602     abstract void forwardToLocal(LocalProxyTransaction successor, TransactionRequest<?> request,
603             Consumer<Response<?, ?>> callback);
604
605     @Override
606     public final String toString() {
607         return MoreObjects.toStringHelper(this).add("identifier", getIdentifier()).add("state", state).toString();
608     }
609 }

©2013 OpenDaylight, A Linux Foundation Collaborative Project. All Rights Reserved.
OpenDaylight is a registered trademark of The OpenDaylight Project, Inc.
Linux Foundation and OpenDaylight are registered trademarks of the Linux Foundation.
Linux is a registered trademark of Linus Torvalds.