singleton-service/mdsal-singleton-dom-impl/src/main/java/org/opendaylight/mdsal/singleton/dom/impl/ClusterSingletonServiceGroupImpl.java

   1 /*
   2  * Copyright (c) 2016 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.mdsal.singleton.dom.impl;
   9
  10 import static com.google.common.base.Preconditions.checkArgument;
  11 import static com.google.common.base.Preconditions.checkState;
  12 import static com.google.common.base.Verify.verify;
  13 import static com.google.common.base.Verify.verifyNotNull;
  14 import static java.util.Objects.requireNonNull;
  15
  16 import com.google.common.annotations.VisibleForTesting;
  17 import com.google.common.base.MoreObjects;
  18 import com.google.common.collect.ImmutableList;
  19 import com.google.common.collect.ImmutableSet;
  20 import com.google.common.util.concurrent.FutureCallback;
  21 import com.google.common.util.concurrent.Futures;
  22 import com.google.common.util.concurrent.ListenableFuture;
  23 import com.google.common.util.concurrent.MoreExecutors;
  24 import com.google.common.util.concurrent.SettableFuture;
  25 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  26 import java.util.Collection;
  27 import java.util.HashMap;
  28 import java.util.Iterator;
  29 import java.util.Map;
  30 import java.util.Map.Entry;
  31 import java.util.Set;
  32 import java.util.concurrent.ConcurrentHashMap;
  33 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
  34 import java.util.concurrent.atomic.AtomicReference;
  35 import org.checkerframework.checker.lock.qual.GuardedBy;
  36 import org.checkerframework.checker.lock.qual.Holding;
  37 import org.eclipse.jdt.annotation.NonNull;
  38 import org.opendaylight.mdsal.eos.common.api.CandidateAlreadyRegisteredException;
  39 import org.opendaylight.mdsal.eos.common.api.EntityOwnershipChangeState;
  40 import org.opendaylight.mdsal.eos.common.api.GenericEntity;
  41 import org.opendaylight.mdsal.eos.common.api.GenericEntityOwnershipCandidateRegistration;
  42 import org.opendaylight.mdsal.eos.common.api.GenericEntityOwnershipChange;
  43 import org.opendaylight.mdsal.eos.common.api.GenericEntityOwnershipListener;
  44 import org.opendaylight.mdsal.eos.common.api.GenericEntityOwnershipService;
  45 import org.opendaylight.mdsal.singleton.common.api.ClusterSingletonService;
  46 import org.opendaylight.mdsal.singleton.common.api.ClusterSingletonServiceRegistration;
  47 import org.opendaylight.yangtools.concepts.HierarchicalIdentifier;
  48 import org.slf4j.Logger;
  49 import org.slf4j.LoggerFactory;
  50
  51 /**
  52  * Implementation of {@link ClusterSingletonServiceGroup} on top of the Entity Ownership Service. Since EOS is atomic
  53  * in its operation and singleton services incur startup and most notably cleanup, we need to do something smart here.
  54  *
  55  * <p>
  56  * The implementation takes advantage of the fact that EOS provides stable ownership, i.e. owners are not moved as
  57  * a result on new candidates appearing. We use two entities:
  58  * - service entity, to which all nodes register
  59  * - cleanup entity, which only the service entity owner registers to
  60  *
  61  * <p>
  62  * Once the cleanup entity ownership is acquired, services are started. As long as the cleanup entity is registered,
  63  * it should remain the owner. In case a new service owner emerges, the old owner will start the cleanup process,
  64  * eventually releasing the cleanup entity. The new owner registers for the cleanup entity -- but will not see it
  65  * granted until the old owner finishes the cleanup.
  66  *
  67  * @param <P> the instance identifier path type
  68  * @param <E> the GenericEntity type
  69  * @param <C> the GenericEntityOwnershipChange type
  70  * @param <G> the GenericEntityOwnershipListener type
  71  * @param <S> the GenericEntityOwnershipService type
  72  */
  73 final class ClusterSingletonServiceGroupImpl<P extends HierarchicalIdentifier<P>, E extends GenericEntity<P>,
  74         C extends GenericEntityOwnershipChange<P, E>,  G extends GenericEntityOwnershipListener<P, C>,
  75         S extends GenericEntityOwnershipService<P, E, G>> extends ClusterSingletonServiceGroup<P, E, C> {
  76
  77     private enum EntityState {
  78         /**
  79          * This entity was never registered.
  80          */
  81         UNREGISTERED,
  82         /**
  83          * Registration exists, but we are waiting for it to resolve.
  84          */
  85         REGISTERED,
  86         /**
  87          * Registration indicated we are the owner.
  88          */
  89         OWNED,
  90         /**
  91          * Registration indicated we are the owner, but global state is uncertain -- meaning there can be owners in
  92          * another partition, for example.
  93          */
  94         OWNED_JEOPARDY,
  95         /**
  96          * Registration indicated we are not the owner. In this state we do not care about global state, therefore we
  97          * do not need an UNOWNED_JEOPARDY state.
  98          */
  99         UNOWNED,
 100     }
 101
 102     enum ServiceState {
 103         /**
 104          * Local service is up and running.
 105          */
 106         // FIXME: we should support async startup, which will require a STARTING state.
 107         STARTED,
 108         /**
 109          * Local service is being stopped.
 110          */
 111         STOPPING,
 112     }
 113
 114     private static final Logger LOG = LoggerFactory.getLogger(ClusterSingletonServiceGroupImpl.class);
 115
 116     private final S entityOwnershipService;
 117     private final String identifier;
 118
 119     /* Entity instances */
 120     private final E serviceEntity;
 121     private final E cleanupEntity;
 122
 123     private final Set<ClusterSingletonServiceRegistration> members = ConcurrentHashMap.newKeySet();
 124     // Guarded by lock
 125     private final Map<ClusterSingletonServiceRegistration, ServiceInfo> services = new HashMap<>();
 126
 127     // Marker for when any state changed
 128     @SuppressWarnings("rawtypes")
 129     private static final AtomicIntegerFieldUpdater<ClusterSingletonServiceGroupImpl> DIRTY_UPDATER =
 130             AtomicIntegerFieldUpdater.newUpdater(ClusterSingletonServiceGroupImpl.class, "dirty");
 131     private volatile int dirty;
 132
 133     // Simplified lock: non-reentrant, support tryLock() only
 134     @SuppressWarnings("rawtypes")
 135     private static final AtomicIntegerFieldUpdater<ClusterSingletonServiceGroupImpl> LOCK_UPDATER =
 136             AtomicIntegerFieldUpdater.newUpdater(ClusterSingletonServiceGroupImpl.class, "lock");
 137     @SuppressWarnings("unused")
 138     private volatile int lock;
 139
 140     /*
 141      * State tracking is quite involved, as we are tracking up to four asynchronous sources of events:
 142      * - user calling close()
 143      * - service entity ownership
 144      * - cleanup entity ownership
 145      * - service shutdown future
 146      *
 147      * Absolutely correct solution would be a set of behaviors, which govern each state, remembering where we want to
 148      * get to and what we are doing. That would result in ~15 classes which would quickly render this code unreadable
 149      * due to boilerplate overhead.
 150      *
 151      * We therefore take a different approach, tracking state directly in this class and evaluate state transitions
 152      * based on recorded bits -- without explicit representation of state machine state.
 153      */
 154     /**
 155      * Group close future. In can only go from null to non-null reference. Whenever it is non-null, it indicates that
 156      * the user has closed the group and we are converging to termination.
 157      */
 158     // We are using volatile get-and-set to support non-blocking close(). It may be more efficient to inline it here,
 159     // as we perform a volatile read after unlocking -- that volatile read may easier on L1 cache.
 160     // XXX: above needs a microbenchmark contention ever becomes a problem.
 161     private final AtomicReference<SettableFuture<Void>> closeFuture = new AtomicReference<>();
 162
 163     /**
 164      * Service (base) entity registration. This entity selects an owner candidate across nodes. Candidates proceed to
 165      * acquire {@link #cleanupEntity}.
 166      */
 167     @GuardedBy("this")
 168     private GenericEntityOwnershipCandidateRegistration<P, E> serviceEntityReg = null;
 169     /**
 170      * Service (base) entity last reported state.
 171      */
 172     @GuardedBy("this")
 173     private EntityState serviceEntityState = EntityState.UNREGISTERED;
 174
 175     /**
 176      * Cleanup (owner) entity registration. This entity guards access to service state and coordinates shutdown cleanup
 177      * and startup.
 178      */
 179     @GuardedBy("this")
 180     private GenericEntityOwnershipCandidateRegistration<P, E> cleanupEntityReg;
 181     /**
 182      * Cleanup (owner) entity last reported state.
 183      */
 184     @GuardedBy("this")
 185     private EntityState cleanupEntityState = EntityState.UNREGISTERED;
 186
 187     private volatile boolean initialized;
 188
 189     /**
 190      * Class constructor. Note: last argument is reused as-is.
 191      *
 192      * @param identifier non-empty string as identifier
 193      * @param mainEntity as Entity instance
 194      * @param closeEntity as Entity instance
 195      * @param entityOwnershipService GenericEntityOwnershipService instance
 196      * @param parent parent service
 197      * @param services Services list
 198      */
 199     ClusterSingletonServiceGroupImpl(final String identifier, final S entityOwnershipService, final E mainEntity,
 200             final E closeEntity, final Collection<ClusterSingletonServiceRegistration> services) {
 201         checkArgument(!identifier.isEmpty(), "Identifier may not be empty");
 202         this.identifier = identifier;
 203         this.entityOwnershipService = requireNonNull(entityOwnershipService);
 204         this.serviceEntity = requireNonNull(mainEntity);
 205         this.cleanupEntity = requireNonNull(closeEntity);
 206         members.addAll(services);
 207
 208         LOG.debug("Instantiated new service group for {}", identifier);
 209     }
 210
 211     @VisibleForTesting
 212     ClusterSingletonServiceGroupImpl(final String identifier, final E mainEntity,
 213             final E closeEntity, final S entityOwnershipService) {
 214         this(identifier, entityOwnershipService, mainEntity, closeEntity, ImmutableList.of());
 215     }
 216
 217     @Override
 218     public String getIdentifier() {
 219         return identifier;
 220     }
 221
 222     @Override
 223     ListenableFuture<?> closeClusterSingletonGroup() {
 224         final ListenableFuture<?> ret = destroyGroup();
 225         members.clear();
 226         markDirty();
 227
 228         if (tryLock()) {
 229             reconcileState();
 230         } else {
 231             LOG.debug("Service group {} postponing sync on close", identifier);
 232         }
 233
 234         return ret;
 235     }
 236
 237     private boolean isClosed() {
 238         return closeFuture.get() != null;
 239     }
 240
 241     @Override
 242     void initialize() throws CandidateAlreadyRegisteredException {
 243         verify(tryLock());
 244         try {
 245             checkState(!initialized, "Singleton group %s was already initilized", identifier);
 246             LOG.debug("Initializing service group {} with services {}", identifier, members);
 247             synchronized (this) {
 248                 serviceEntityState = EntityState.REGISTERED;
 249                 serviceEntityReg = entityOwnershipService.registerCandidate(serviceEntity);
 250                 initialized = true;
 251             }
 252         } finally {
 253             unlock();
 254         }
 255     }
 256
 257     private void checkNotClosed() {
 258         checkState(!isClosed(), "Service group %s has already been closed", identifier);
 259     }
 260
 261     @Override
 262     void registerService(final ClusterSingletonServiceRegistration reg) {
 263         final ClusterSingletonService service = verifyRegistration(reg);
 264         checkNotClosed();
 265
 266         checkState(initialized, "Service group %s is not initialized yet", identifier);
 267
 268         // First put the service
 269         LOG.debug("Adding service {} to service group {}", service, identifier);
 270         verify(members.add(reg));
 271         markDirty();
 272
 273         if (!tryLock()) {
 274             LOG.debug("Service group {} delayed register of {}", identifier, reg);
 275             return;
 276         }
 277
 278         reconcileState();
 279     }
 280
 281     @Override
 282     ListenableFuture<?> unregisterService(final ClusterSingletonServiceRegistration reg) {
 283         verifyRegistration(reg);
 284         checkNotClosed();
 285
 286         verify(members.remove(reg));
 287         markDirty();
 288         if (members.isEmpty()) {
 289             // We need to let AbstractClusterSingletonServiceProviderImpl know this group is to be shutdown
 290             // before we start applying state, because while we do not re-enter, the user is free to do whatever,
 291             // notably including registering a service with the same ID from the service shutdown hook. That
 292             // registration request needs to hit the successor of this group.
 293             return destroyGroup();
 294         }
 295
 296         if (tryLock()) {
 297             reconcileState();
 298         } else {
 299             LOG.debug("Service group {} delayed unregister of {}", identifier, reg);
 300         }
 301         return null;
 302     }
 303
 304     private ClusterSingletonService verifyRegistration(final ClusterSingletonServiceRegistration reg) {
 305         final ClusterSingletonService service = reg.getInstance();
 306         verify(identifier.equals(service.getIdentifier().getName()));
 307         return service;
 308     }
 309
 310     private synchronized @NonNull ListenableFuture<?> destroyGroup() {
 311         final SettableFuture<Void> future = SettableFuture.create();
 312         if (!closeFuture.compareAndSet(null, future)) {
 313             return verifyNotNull(closeFuture.get());
 314         }
 315
 316         if (serviceEntityReg != null) {
 317             // We are still holding the service registration, close it now...
 318             LOG.debug("Service group {} unregistering service entity {}", identifier, serviceEntity);
 319             serviceEntityReg.close();
 320             serviceEntityReg = null;
 321         }
 322
 323         markDirty();
 324         return future;
 325     }
 326
 327     @Override
 328     void ownershipChanged(final C ownershipChange) {
 329         LOG.debug("Ownership change {} for ClusterSingletonServiceGroup {}", ownershipChange, identifier);
 330
 331         synchronized (this) {
 332             lockedOwnershipChanged(ownershipChange);
 333         }
 334
 335         if (isDirty()) {
 336             if (!tryLock()) {
 337                 LOG.debug("Service group {} postponing ownership change sync", identifier);
 338                 return;
 339             }
 340
 341             reconcileState();
 342         }
 343     }
 344
 345     /**
 346      * Handle an ownership change with the lock held. Callers are expected to handle termination conditions, this method
 347      * and anything it calls must not call {@link #lockedClose(SettableFuture)}.
 348      *
 349      * @param ownershipChange reported change
 350      */
 351     @Holding("this")
 352     private void lockedOwnershipChanged(final C ownershipChange) {
 353         final E entity = ownershipChange.getEntity();
 354         if (serviceEntity.equals(entity)) {
 355             serviceOwnershipChanged(ownershipChange.getState(), ownershipChange.inJeopardy());
 356             markDirty();
 357         } else if (cleanupEntity.equals(entity)) {
 358             cleanupCandidateOwnershipChanged(ownershipChange.getState(), ownershipChange.inJeopardy());
 359             markDirty();
 360         } else {
 361             LOG.warn("Group {} received unrecognized change {}", identifier, ownershipChange);
 362         }
 363     }
 364
 365     @Holding("this")
 366     private void cleanupCandidateOwnershipChanged(final EntityOwnershipChangeState state, final boolean jeopardy) {
 367         if (jeopardy) {
 368             switch (state) {
 369                 case LOCAL_OWNERSHIP_GRANTED:
 370                 case LOCAL_OWNERSHIP_RETAINED_WITH_NO_CHANGE:
 371                     LOG.warn("Service group {} cleanup entity owned without certainty", identifier);
 372                     cleanupEntityState = EntityState.OWNED_JEOPARDY;
 373                     break;
 374                 case LOCAL_OWNERSHIP_LOST_NEW_OWNER:
 375                 case LOCAL_OWNERSHIP_LOST_NO_OWNER:
 376                 case REMOTE_OWNERSHIP_CHANGED:
 377                 case REMOTE_OWNERSHIP_LOST_NO_OWNER:
 378                     LOG.info("Service group {} cleanup entity ownership uncertain", identifier);
 379                     cleanupEntityState = EntityState.UNOWNED;
 380                     break;
 381                 default:
 382                     throw new IllegalStateException("Unhandled cleanup entity jeopardy change " + state);
 383             }
 384
 385             return;
 386         }
 387
 388         if (cleanupEntityState == EntityState.OWNED_JEOPARDY) {
 389             // Pair info message with previous jeopardy
 390             LOG.info("Service group {} cleanup entity ownership ascertained", identifier);
 391         }
 392
 393         switch (state) {
 394             case LOCAL_OWNERSHIP_GRANTED:
 395             case LOCAL_OWNERSHIP_RETAINED_WITH_NO_CHANGE:
 396                 cleanupEntityState = EntityState.OWNED;
 397                 break;
 398             case LOCAL_OWNERSHIP_LOST_NEW_OWNER:
 399             case LOCAL_OWNERSHIP_LOST_NO_OWNER:
 400             case REMOTE_OWNERSHIP_LOST_NO_OWNER:
 401             case REMOTE_OWNERSHIP_CHANGED:
 402                 cleanupEntityState = EntityState.UNOWNED;
 403                 break;
 404             default:
 405                 LOG.warn("Service group {} ignoring unhandled cleanup entity change {}", identifier, state);
 406                 break;
 407         }
 408     }
 409
 410     @Holding("this")
 411     private void serviceOwnershipChanged(final EntityOwnershipChangeState state, final boolean jeopardy) {
 412         if (jeopardy) {
 413             LOG.info("Service group {} service entity ownership uncertain", identifier);
 414             switch (state) {
 415                 case LOCAL_OWNERSHIP_GRANTED:
 416                 case LOCAL_OWNERSHIP_RETAINED_WITH_NO_CHANGE:
 417                     serviceEntityState = EntityState.OWNED_JEOPARDY;
 418                     break;
 419                 case LOCAL_OWNERSHIP_LOST_NEW_OWNER:
 420                 case LOCAL_OWNERSHIP_LOST_NO_OWNER:
 421                 case REMOTE_OWNERSHIP_CHANGED:
 422                 case REMOTE_OWNERSHIP_LOST_NO_OWNER:
 423                     serviceEntityState = EntityState.UNOWNED;
 424                     break;
 425                 default:
 426                     throw new IllegalStateException("Unhandled cleanup entity jeopardy change " + state);
 427             }
 428             return;
 429         }
 430
 431         if (serviceEntityState == EntityState.OWNED_JEOPARDY) {
 432             // Pair info message with previous jeopardy
 433             LOG.info("Service group {} service entity ownership ascertained", identifier);
 434         }
 435
 436         switch (state) {
 437             case LOCAL_OWNERSHIP_GRANTED:
 438             case LOCAL_OWNERSHIP_RETAINED_WITH_NO_CHANGE:
 439                 LOG.debug("Service group {} acquired service entity ownership", identifier);
 440                 serviceEntityState = EntityState.OWNED;
 441                 break;
 442             case LOCAL_OWNERSHIP_LOST_NEW_OWNER:
 443             case LOCAL_OWNERSHIP_LOST_NO_OWNER:
 444             case REMOTE_OWNERSHIP_CHANGED:
 445             case REMOTE_OWNERSHIP_LOST_NO_OWNER:
 446                 LOG.debug("Service group {} lost service entity ownership", identifier);
 447                 serviceEntityState = EntityState.UNOWNED;
 448                 break;
 449             default:
 450                 LOG.warn("Service group {} ignoring unhandled cleanup entity change {}", identifier, state);
 451         }
 452     }
 453
 454     // has to be called with lock asserted, which will be released prior to returning
 455     private void reconcileState() {
 456         // Always check if there is any state change to be applied.
 457         while (true) {
 458             try {
 459                 if (conditionalClean()) {
 460                     tryReconcileState();
 461                 }
 462             } finally {
 463                 // We may have ran a round of reconciliation, but the either one of may have happened asynchronously:
 464                 // - registration
 465                 // - unregistration
 466                 // - service future completed
 467                 // - entity state changed
 468                 //
 469                 // We are dropping the lock, but we need to recheck dirty and try to apply state again if it is found to
 470                 // be dirty again. This closes the following race condition:
 471                 //
 472                 // A: runs these checks holding the lock
 473                 // B: modifies them, fails to acquire lock
 474                 // A: releases lock -> noone takes care of reconciliation
 475
 476                 unlock();
 477             }
 478
 479             if (isDirty()) {
 480                 if (tryLock()) {
 481                     LOG.debug("Service group {} re-running reconciliation", identifier);
 482                     continue;
 483                 }
 484
 485                 LOG.debug("Service group {} will be reconciled by someone else", identifier);
 486             } else {
 487                 LOG.debug("Service group {} is completely reconciled", identifier);
 488             }
 489
 490             break;
 491         }
 492     }
 493
 494     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
 495             justification = "https://github.com/spotbugs/spotbugs/issues/811")
 496     private void serviceTransitionCompleted() {
 497         markDirty();
 498         if (tryLock()) {
 499             reconcileState();
 500         }
 501     }
 502
 503     // Has to be called with lock asserted
 504     private void tryReconcileState() {
 505         // First take a safe snapshot of current state on which we will base our decisions.
 506         final Set<ClusterSingletonServiceRegistration> localMembers;
 507         final boolean haveCleanup;
 508         final boolean haveService;
 509         synchronized (this) {
 510             if (serviceEntityReg != null) {
 511                 switch (serviceEntityState) {
 512                     case OWNED:
 513                     case OWNED_JEOPARDY:
 514                         haveService = true;
 515                         break;
 516                     case REGISTERED:
 517                     case UNOWNED:
 518                     case UNREGISTERED:
 519                         haveService = false;
 520                         break;
 521                     default:
 522                         throw new IllegalStateException("Unhandled service entity state " + serviceEntityState);
 523                 }
 524             } else {
 525                 haveService = false;
 526             }
 527
 528             if (haveService && cleanupEntityReg == null) {
 529                 // We have the service entity but have not registered for cleanup entity. Do that now and retry.
 530                 LOG.debug("Service group {} registering cleanup entity", identifier);
 531                 try {
 532                     cleanupEntityState = EntityState.REGISTERED;
 533                     cleanupEntityReg = entityOwnershipService.registerCandidate(cleanupEntity);
 534                 } catch (CandidateAlreadyRegisteredException e) {
 535                     LOG.error("Service group {} failed to take ownership, aborting", identifier, e);
 536                     if (serviceEntityReg != null) {
 537                         serviceEntityReg.close();
 538                         serviceEntityReg = null;
 539                     }
 540                 }
 541                 markDirty();
 542                 return;
 543             }
 544
 545             if (cleanupEntityReg != null) {
 546                 switch (cleanupEntityState) {
 547                     case OWNED:
 548                         haveCleanup = true;
 549                         break;
 550                     case OWNED_JEOPARDY:
 551                     case REGISTERED:
 552                     case UNOWNED:
 553                     case UNREGISTERED:
 554                         haveCleanup = false;
 555                         break;
 556                     default:
 557                         throw new IllegalStateException("Unhandled service entity state " + serviceEntityState);
 558                 }
 559             } else {
 560                 haveCleanup = false;
 561             }
 562
 563             localMembers = ImmutableSet.copyOf(members);
 564         }
 565
 566         if (haveService && haveCleanup) {
 567             ensureServicesStarting(localMembers);
 568             return;
 569         }
 570
 571         ensureServicesStopping();
 572
 573         if (!haveService && services.isEmpty()) {
 574             LOG.debug("Service group {} has no running services", identifier);
 575             final boolean canFinishClose;
 576             synchronized (this) {
 577                 if (cleanupEntityReg != null) {
 578                     LOG.debug("Service group {} releasing cleanup entity", identifier);
 579                     cleanupEntityReg.close();
 580                     cleanupEntityReg = null;
 581                 }
 582
 583                 switch (cleanupEntityState) {
 584                     case OWNED:
 585                     case OWNED_JEOPARDY:
 586                     case REGISTERED:
 587                         // When we are registered we need to wait for registration to resolve, otherwise
 588                         // the notification could be routed to the next incarnation of this group -- which could be
 589                         // confused by the fact it is not registered, but receives, for example, OWNED notification.
 590                         canFinishClose = false;
 591                         break;
 592                     case UNOWNED:
 593                     case UNREGISTERED:
 594                         canFinishClose = true;
 595                         break;
 596                     default:
 597                         throw new IllegalStateException("Unhandled cleanup entity state " + cleanupEntityState);
 598                 }
 599             }
 600
 601             if (canFinishClose) {
 602                 final SettableFuture<Void> localFuture = closeFuture.get();
 603                 if (localFuture != null && !localFuture.isDone()) {
 604                     LOG.debug("Service group {} completing termination", identifier);
 605                     localFuture.set(null);
 606                 }
 607             }
 608         }
 609     }
 610
 611     // Has to be called with lock asserted
 612     @SuppressWarnings("illegalCatch")
 613     private void ensureServicesStarting(final Set<ClusterSingletonServiceRegistration> localConfig) {
 614         LOG.debug("Service group {} starting services", identifier);
 615
 616         // This may look counter-intuitive, but the localConfig may be missing some services that are started -- for
 617         // example when this method is executed as part of unregisterService() call. In that case we need to ensure
 618         // services in the list are stopping
 619         final Iterator<Entry<ClusterSingletonServiceRegistration, ServiceInfo>> it = services.entrySet().iterator();
 620         while (it.hasNext()) {
 621             final Entry<ClusterSingletonServiceRegistration, ServiceInfo> entry = it.next();
 622             final ClusterSingletonServiceRegistration reg = entry.getKey();
 623             if (!localConfig.contains(reg)) {
 624                 final ServiceInfo newInfo = ensureStopping(reg, entry.getValue());
 625                 if (newInfo != null) {
 626                     entry.setValue(newInfo);
 627                 } else {
 628                     it.remove();
 629                 }
 630             }
 631         }
 632
 633         // Now make sure member services are being juggled around
 634         for (ClusterSingletonServiceRegistration reg : localConfig) {
 635             if (!services.containsKey(reg)) {
 636                 final ClusterSingletonService service = reg.getInstance();
 637                 LOG.debug("Starting service {}", service);
 638
 639                 try {
 640                     service.instantiateServiceInstance();
 641                 } catch (Exception e) {
 642                     LOG.warn("Service group {} service {} failed to start, attempting to continue", identifier, service,
 643                         e);
 644                     continue;
 645                 }
 646
 647                 services.put(reg, ServiceInfo.started());
 648             }
 649         }
 650     }
 651
 652     // Has to be called with lock asserted
 653     private void ensureServicesStopping() {
 654         final Iterator<Entry<ClusterSingletonServiceRegistration, ServiceInfo>> it = services.entrySet().iterator();
 655         while (it.hasNext()) {
 656             final Entry<ClusterSingletonServiceRegistration, ServiceInfo> entry = it.next();
 657             final ServiceInfo newInfo = ensureStopping(entry.getKey(), entry.getValue());
 658             if (newInfo != null) {
 659                 entry.setValue(newInfo);
 660             } else {
 661                 it.remove();
 662             }
 663         }
 664     }
 665
 666     @SuppressWarnings("illegalCatch")
 667     private ServiceInfo ensureStopping(final ClusterSingletonServiceRegistration reg, final ServiceInfo info) {
 668         switch (info.getState()) {
 669             case STARTED:
 670                 final ClusterSingletonService service = reg.getInstance();
 671
 672                 LOG.debug("Service group {} stopping service {}", identifier, service);
 673                 final @NonNull ListenableFuture<?> future;
 674                 try {
 675                     future = verifyNotNull(service.closeServiceInstance());
 676                 } catch (Exception e) {
 677                     LOG.warn("Service group {} service {} failed to stop, attempting to continue", identifier, service,
 678                         e);
 679                     return null;
 680                 }
 681
 682                 Futures.addCallback(future, new FutureCallback<Object>() {
 683                     @Override
 684                     public void onSuccess(final Object result) {
 685                         LOG.debug("Service group {} service {} stopped successfully", identifier, service);
 686                         serviceTransitionCompleted();
 687                     }
 688
 689                     @Override
 690                     public void onFailure(final Throwable cause) {
 691                         LOG.debug("Service group {} service {} stopped with error", identifier, service, cause);
 692                         serviceTransitionCompleted();
 693                     }
 694                 }, MoreExecutors.directExecutor());
 695                 return info.toState(ServiceState.STOPPING, future);
 696             case STOPPING:
 697                 if (info.getFuture().isDone()) {
 698                     LOG.debug("Service group {} removed stopped service {}", identifier, reg.getInstance());
 699                     return null;
 700                 }
 701                 return info;
 702             default:
 703                 throw new IllegalStateException("Unhandled state " + info.getState());
 704         }
 705     }
 706
 707     private void markDirty() {
 708         dirty = 1;
 709     }
 710
 711     private boolean isDirty() {
 712         return dirty != 0;
 713     }
 714
 715     private boolean conditionalClean() {
 716         return DIRTY_UPDATER.compareAndSet(this, 1, 0);
 717     }
 718
 719     private boolean tryLock() {
 720         return LOCK_UPDATER.compareAndSet(this, 0, 1);
 721     }
 722
 723     private boolean unlock() {
 724         verify(LOCK_UPDATER.compareAndSet(this, 1, 0));
 725         return true;
 726     }
 727
 728     @Override
 729     public String toString() {
 730         return MoreObjects.toStringHelper(this).add("identifier", identifier).toString();
 731     }
 732 }