singleton-service/mdsal-singleton-dom-impl/src/main/java/org/opendaylight/mdsal/singleton/dom/impl/ClusterSingletonServiceGroupImpl.java

   1 /*
   2  * Copyright (c) 2016 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.mdsal.singleton.dom.impl;
   9
  10 import static com.google.common.base.Preconditions.checkArgument;
  11 import static com.google.common.base.Preconditions.checkState;
  12 import static com.google.common.base.Verify.verify;
  13 import static com.google.common.base.Verify.verifyNotNull;
  14 import static java.util.Objects.requireNonNull;
  15
  16 import com.google.common.annotations.VisibleForTesting;
  17 import com.google.common.base.MoreObjects;
  18 import com.google.common.collect.ImmutableList;
  19 import com.google.common.collect.ImmutableSet;
  20 import com.google.common.util.concurrent.FutureCallback;
  21 import com.google.common.util.concurrent.Futures;
  22 import com.google.common.util.concurrent.ListenableFuture;
  23 import com.google.common.util.concurrent.MoreExecutors;
  24 import com.google.common.util.concurrent.SettableFuture;
  25 import java.util.Collection;
  26 import java.util.HashMap;
  27 import java.util.Iterator;
  28 import java.util.Map;
  29 import java.util.Map.Entry;
  30 import java.util.Set;
  31 import java.util.concurrent.ConcurrentHashMap;
  32 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
  33 import java.util.concurrent.atomic.AtomicReference;
  34 import org.checkerframework.checker.lock.qual.GuardedBy;
  35 import org.checkerframework.checker.lock.qual.Holding;
  36 import org.eclipse.jdt.annotation.NonNull;
  37 import org.opendaylight.mdsal.eos.common.api.CandidateAlreadyRegisteredException;
  38 import org.opendaylight.mdsal.eos.common.api.EntityOwnershipChangeState;
  39 import org.opendaylight.mdsal.eos.common.api.GenericEntity;
  40 import org.opendaylight.mdsal.eos.common.api.GenericEntityOwnershipChange;
  41 import org.opendaylight.mdsal.eos.common.api.GenericEntityOwnershipListener;
  42 import org.opendaylight.mdsal.eos.common.api.GenericEntityOwnershipService;
  43 import org.opendaylight.mdsal.singleton.common.api.ClusterSingletonService;
  44 import org.opendaylight.mdsal.singleton.common.api.ClusterSingletonServiceRegistration;
  45 import org.opendaylight.yangtools.concepts.HierarchicalIdentifier;
  46 import org.opendaylight.yangtools.concepts.Registration;
  47 import org.slf4j.Logger;
  48 import org.slf4j.LoggerFactory;
  49
  50 /**
  51  * Implementation of {@link ClusterSingletonServiceGroup} on top of the Entity Ownership Service. Since EOS is atomic
  52  * in its operation and singleton services incur startup and most notably cleanup, we need to do something smart here.
  53  *
  54  * <p>
  55  * The implementation takes advantage of the fact that EOS provides stable ownership, i.e. owners are not moved as
  56  * a result on new candidates appearing. We use two entities:
  57  * - service entity, to which all nodes register
  58  * - cleanup entity, which only the service entity owner registers to
  59  *
  60  * <p>
  61  * Once the cleanup entity ownership is acquired, services are started. As long as the cleanup entity is registered,
  62  * it should remain the owner. In case a new service owner emerges, the old owner will start the cleanup process,
  63  * eventually releasing the cleanup entity. The new owner registers for the cleanup entity -- but will not see it
  64  * granted until the old owner finishes the cleanup.
  65  *
  66  * @param <P> the instance identifier path type
  67  * @param <E> the GenericEntity type
  68  * @param <C> the GenericEntityOwnershipChange type
  69  * @param <G> the GenericEntityOwnershipListener type
  70  * @param <S> the GenericEntityOwnershipService type
  71  */
  72 final class ClusterSingletonServiceGroupImpl<P extends HierarchicalIdentifier<P>, E extends GenericEntity<P>,
  73         C extends GenericEntityOwnershipChange<P, E>,  G extends GenericEntityOwnershipListener<P, C>,
  74         S extends GenericEntityOwnershipService<P, E, G>> extends ClusterSingletonServiceGroup<P, E, C> {
  75
  76     private enum EntityState {
  77         /**
  78          * This entity was never registered.
  79          */
  80         UNREGISTERED,
  81         /**
  82          * Registration exists, but we are waiting for it to resolve.
  83          */
  84         REGISTERED,
  85         /**
  86          * Registration indicated we are the owner.
  87          */
  88         OWNED,
  89         /**
  90          * Registration indicated we are the owner, but global state is uncertain -- meaning there can be owners in
  91          * another partition, for example.
  92          */
  93         OWNED_JEOPARDY,
  94         /**
  95          * Registration indicated we are not the owner. In this state we do not care about global state, therefore we
  96          * do not need an UNOWNED_JEOPARDY state.
  97          */
  98         UNOWNED,
  99     }
 100
 101     enum ServiceState {
 102         /**
 103          * Local service is up and running.
 104          */
 105         // FIXME: we should support async startup, which will require a STARTING state.
 106         STARTED,
 107         /**
 108          * Local service is being stopped.
 109          */
 110         STOPPING,
 111     }
 112
 113     private static final Logger LOG = LoggerFactory.getLogger(ClusterSingletonServiceGroupImpl.class);
 114
 115     private final S entityOwnershipService;
 116     private final String identifier;
 117
 118     /* Entity instances */
 119     private final E serviceEntity;
 120     private final E cleanupEntity;
 121
 122     private final Set<ClusterSingletonServiceRegistration> members = ConcurrentHashMap.newKeySet();
 123     // Guarded by lock
 124     private final Map<ClusterSingletonServiceRegistration, ServiceInfo> services = new HashMap<>();
 125
 126     // Marker for when any state changed
 127     @SuppressWarnings("rawtypes")
 128     private static final AtomicIntegerFieldUpdater<ClusterSingletonServiceGroupImpl> DIRTY_UPDATER =
 129             AtomicIntegerFieldUpdater.newUpdater(ClusterSingletonServiceGroupImpl.class, "dirty");
 130     private volatile int dirty;
 131
 132     // Simplified lock: non-reentrant, support tryLock() only
 133     @SuppressWarnings("rawtypes")
 134     private static final AtomicIntegerFieldUpdater<ClusterSingletonServiceGroupImpl> LOCK_UPDATER =
 135             AtomicIntegerFieldUpdater.newUpdater(ClusterSingletonServiceGroupImpl.class, "lock");
 136     @SuppressWarnings("unused")
 137     private volatile int lock;
 138
 139     /*
 140      * State tracking is quite involved, as we are tracking up to four asynchronous sources of events:
 141      * - user calling close()
 142      * - service entity ownership
 143      * - cleanup entity ownership
 144      * - service shutdown future
 145      *
 146      * Absolutely correct solution would be a set of behaviors, which govern each state, remembering where we want to
 147      * get to and what we are doing. That would result in ~15 classes which would quickly render this code unreadable
 148      * due to boilerplate overhead.
 149      *
 150      * We therefore take a different approach, tracking state directly in this class and evaluate state transitions
 151      * based on recorded bits -- without explicit representation of state machine state.
 152      */
 153     /**
 154      * Group close future. In can only go from null to non-null reference. Whenever it is non-null, it indicates that
 155      * the user has closed the group and we are converging to termination.
 156      */
 157     // We are using volatile get-and-set to support non-blocking close(). It may be more efficient to inline it here,
 158     // as we perform a volatile read after unlocking -- that volatile read may easier on L1 cache.
 159     // XXX: above needs a microbenchmark contention ever becomes a problem.
 160     private final AtomicReference<SettableFuture<Void>> closeFuture = new AtomicReference<>();
 161
 162     /**
 163      * Service (base) entity registration. This entity selects an owner candidate across nodes. Candidates proceed to
 164      * acquire {@link #cleanupEntity}.
 165      */
 166     @GuardedBy("this")
 167     private Registration serviceEntityReg = null;
 168     /**
 169      * Service (base) entity last reported state.
 170      */
 171     @GuardedBy("this")
 172     private EntityState serviceEntityState = EntityState.UNREGISTERED;
 173
 174     /**
 175      * Cleanup (owner) entity registration. This entity guards access to service state and coordinates shutdown cleanup
 176      * and startup.
 177      */
 178     @GuardedBy("this")
 179     private Registration cleanupEntityReg;
 180     /**
 181      * Cleanup (owner) entity last reported state.
 182      */
 183     @GuardedBy("this")
 184     private EntityState cleanupEntityState = EntityState.UNREGISTERED;
 185
 186     private volatile boolean initialized;
 187
 188     /**
 189      * Class constructor. Note: last argument is reused as-is.
 190      *
 191      * @param identifier non-empty string as identifier
 192      * @param mainEntity as Entity instance
 193      * @param closeEntity as Entity instance
 194      * @param entityOwnershipService GenericEntityOwnershipService instance
 195      * @param parent parent service
 196      * @param services Services list
 197      */
 198     ClusterSingletonServiceGroupImpl(final String identifier, final S entityOwnershipService, final E mainEntity,
 199             final E closeEntity, final Collection<ClusterSingletonServiceRegistration> services) {
 200         checkArgument(!identifier.isEmpty(), "Identifier may not be empty");
 201         this.identifier = identifier;
 202         this.entityOwnershipService = requireNonNull(entityOwnershipService);
 203         this.serviceEntity = requireNonNull(mainEntity);
 204         this.cleanupEntity = requireNonNull(closeEntity);
 205         members.addAll(services);
 206
 207         LOG.debug("Instantiated new service group for {}", identifier);
 208     }
 209
 210     @VisibleForTesting
 211     ClusterSingletonServiceGroupImpl(final String identifier, final E mainEntity,
 212             final E closeEntity, final S entityOwnershipService) {
 213         this(identifier, entityOwnershipService, mainEntity, closeEntity, ImmutableList.of());
 214     }
 215
 216     @Override
 217     public String getIdentifier() {
 218         return identifier;
 219     }
 220
 221     @Override
 222     ListenableFuture<?> closeClusterSingletonGroup() {
 223         final ListenableFuture<?> ret = destroyGroup();
 224         members.clear();
 225         markDirty();
 226
 227         if (tryLock()) {
 228             reconcileState();
 229         } else {
 230             LOG.debug("Service group {} postponing sync on close", identifier);
 231         }
 232
 233         return ret;
 234     }
 235
 236     private boolean isClosed() {
 237         return closeFuture.get() != null;
 238     }
 239
 240     @Override
 241     void initialize() throws CandidateAlreadyRegisteredException {
 242         verify(tryLock());
 243         try {
 244             checkState(!initialized, "Singleton group %s was already initilized", identifier);
 245             LOG.debug("Initializing service group {} with services {}", identifier, members);
 246             synchronized (this) {
 247                 serviceEntityState = EntityState.REGISTERED;
 248                 serviceEntityReg = entityOwnershipService.registerCandidate(serviceEntity);
 249                 initialized = true;
 250             }
 251         } finally {
 252             unlock();
 253         }
 254     }
 255
 256     private void checkNotClosed() {
 257         checkState(!isClosed(), "Service group %s has already been closed", identifier);
 258     }
 259
 260     @Override
 261     void registerService(final ClusterSingletonServiceRegistration reg) {
 262         final ClusterSingletonService service = verifyRegistration(reg);
 263         checkNotClosed();
 264
 265         checkState(initialized, "Service group %s is not initialized yet", identifier);
 266
 267         // First put the service
 268         LOG.debug("Adding service {} to service group {}", service, identifier);
 269         verify(members.add(reg));
 270         markDirty();
 271
 272         if (!tryLock()) {
 273             LOG.debug("Service group {} delayed register of {}", identifier, reg);
 274             return;
 275         }
 276
 277         reconcileState();
 278     }
 279
 280     @Override
 281     ListenableFuture<?> unregisterService(final ClusterSingletonServiceRegistration reg) {
 282         verifyRegistration(reg);
 283         checkNotClosed();
 284
 285         verify(members.remove(reg));
 286         markDirty();
 287         if (members.isEmpty()) {
 288             // We need to let AbstractClusterSingletonServiceProviderImpl know this group is to be shutdown
 289             // before we start applying state, because while we do not re-enter, the user is free to do whatever,
 290             // notably including registering a service with the same ID from the service shutdown hook. That
 291             // registration request needs to hit the successor of this group.
 292             return destroyGroup();
 293         }
 294
 295         if (tryLock()) {
 296             reconcileState();
 297         } else {
 298             LOG.debug("Service group {} delayed unregister of {}", identifier, reg);
 299         }
 300         return null;
 301     }
 302
 303     private ClusterSingletonService verifyRegistration(final ClusterSingletonServiceRegistration reg) {
 304         final ClusterSingletonService service = reg.getInstance();
 305         verify(identifier.equals(service.getIdentifier().getName()));
 306         return service;
 307     }
 308
 309     private synchronized @NonNull ListenableFuture<?> destroyGroup() {
 310         final SettableFuture<Void> future = SettableFuture.create();
 311         if (!closeFuture.compareAndSet(null, future)) {
 312             return verifyNotNull(closeFuture.get());
 313         }
 314
 315         if (serviceEntityReg != null) {
 316             // We are still holding the service registration, close it now...
 317             LOG.debug("Service group {} unregistering service entity {}", identifier, serviceEntity);
 318             serviceEntityReg.close();
 319             serviceEntityReg = null;
 320         }
 321
 322         markDirty();
 323         return future;
 324     }
 325
 326     @Override
 327     void ownershipChanged(final C ownershipChange) {
 328         LOG.debug("Ownership change {} for ClusterSingletonServiceGroup {}", ownershipChange, identifier);
 329
 330         synchronized (this) {
 331             lockedOwnershipChanged(ownershipChange);
 332         }
 333
 334         if (isDirty()) {
 335             if (!tryLock()) {
 336                 LOG.debug("Service group {} postponing ownership change sync", identifier);
 337                 return;
 338             }
 339
 340             reconcileState();
 341         }
 342     }
 343
 344     /**
 345      * Handle an ownership change with the lock held. Callers are expected to handle termination conditions, this method
 346      * and anything it calls must not call {@link #lockedClose(SettableFuture)}.
 347      *
 348      * @param ownershipChange reported change
 349      */
 350     @Holding("this")
 351     private void lockedOwnershipChanged(final C ownershipChange) {
 352         final E entity = ownershipChange.getEntity();
 353         if (serviceEntity.equals(entity)) {
 354             serviceOwnershipChanged(ownershipChange.getState(), ownershipChange.inJeopardy());
 355             markDirty();
 356         } else if (cleanupEntity.equals(entity)) {
 357             cleanupCandidateOwnershipChanged(ownershipChange.getState(), ownershipChange.inJeopardy());
 358             markDirty();
 359         } else {
 360             LOG.warn("Group {} received unrecognized change {}", identifier, ownershipChange);
 361         }
 362     }
 363
 364     @Holding("this")
 365     private void cleanupCandidateOwnershipChanged(final EntityOwnershipChangeState state, final boolean jeopardy) {
 366         if (jeopardy) {
 367             cleanupEntityState = switch (state) {
 368                 case LOCAL_OWNERSHIP_GRANTED, LOCAL_OWNERSHIP_RETAINED_WITH_NO_CHANGE -> {
 369                     LOG.warn("Service group {} cleanup entity owned without certainty", identifier);
 370                     yield EntityState.OWNED_JEOPARDY;
 371                 }
 372                 case LOCAL_OWNERSHIP_LOST_NEW_OWNER, LOCAL_OWNERSHIP_LOST_NO_OWNER, REMOTE_OWNERSHIP_CHANGED,
 373                      REMOTE_OWNERSHIP_LOST_NO_OWNER -> {
 374                     LOG.info("Service group {} cleanup entity ownership uncertain", identifier);
 375                     yield EntityState.UNOWNED;
 376                 }
 377             };
 378             return;
 379         }
 380
 381         if (cleanupEntityState == EntityState.OWNED_JEOPARDY) {
 382             // Pair info message with previous jeopardy
 383             LOG.info("Service group {} cleanup entity ownership ascertained", identifier);
 384         }
 385
 386         cleanupEntityState = switch (state) {
 387             case LOCAL_OWNERSHIP_GRANTED, LOCAL_OWNERSHIP_RETAINED_WITH_NO_CHANGE -> EntityState.OWNED;
 388             case LOCAL_OWNERSHIP_LOST_NEW_OWNER, LOCAL_OWNERSHIP_LOST_NO_OWNER, REMOTE_OWNERSHIP_LOST_NO_OWNER,
 389                  REMOTE_OWNERSHIP_CHANGED -> EntityState.UNOWNED;
 390         };
 391     }
 392
 393     @Holding("this")
 394     private void serviceOwnershipChanged(final EntityOwnershipChangeState state, final boolean jeopardy) {
 395         if (jeopardy) {
 396             LOG.info("Service group {} service entity ownership uncertain", identifier);
 397             serviceEntityState = switch (state) {
 398                 case LOCAL_OWNERSHIP_GRANTED, LOCAL_OWNERSHIP_RETAINED_WITH_NO_CHANGE -> EntityState.OWNED_JEOPARDY;
 399                 case LOCAL_OWNERSHIP_LOST_NEW_OWNER, LOCAL_OWNERSHIP_LOST_NO_OWNER, REMOTE_OWNERSHIP_CHANGED,
 400                      REMOTE_OWNERSHIP_LOST_NO_OWNER -> EntityState.UNOWNED;
 401             };
 402             return;
 403         }
 404
 405         if (serviceEntityState == EntityState.OWNED_JEOPARDY) {
 406             // Pair info message with previous jeopardy
 407             LOG.info("Service group {} service entity ownership ascertained", identifier);
 408         }
 409
 410         switch (state) {
 411             case LOCAL_OWNERSHIP_GRANTED:
 412             case LOCAL_OWNERSHIP_RETAINED_WITH_NO_CHANGE:
 413                 LOG.debug("Service group {} acquired service entity ownership", identifier);
 414                 serviceEntityState = EntityState.OWNED;
 415                 break;
 416             case LOCAL_OWNERSHIP_LOST_NEW_OWNER:
 417             case LOCAL_OWNERSHIP_LOST_NO_OWNER:
 418             case REMOTE_OWNERSHIP_CHANGED:
 419             case REMOTE_OWNERSHIP_LOST_NO_OWNER:
 420                 LOG.debug("Service group {} lost service entity ownership", identifier);
 421                 serviceEntityState = EntityState.UNOWNED;
 422                 break;
 423             default:
 424                 LOG.warn("Service group {} ignoring unhandled cleanup entity change {}", identifier, state);
 425         }
 426     }
 427
 428     // has to be called with lock asserted, which will be released prior to returning
 429     private void reconcileState() {
 430         // Always check if there is any state change to be applied.
 431         while (true) {
 432             try {
 433                 if (conditionalClean()) {
 434                     tryReconcileState();
 435                 }
 436             } finally {
 437                 // We may have ran a round of reconciliation, but the either one of may have happened asynchronously:
 438                 // - registration
 439                 // - unregistration
 440                 // - service future completed
 441                 // - entity state changed
 442                 //
 443                 // We are dropping the lock, but we need to recheck dirty and try to apply state again if it is found to
 444                 // be dirty again. This closes the following race condition:
 445                 //
 446                 // A: runs these checks holding the lock
 447                 // B: modifies them, fails to acquire lock
 448                 // A: releases lock -> noone takes care of reconciliation
 449
 450                 unlock();
 451             }
 452
 453             if (isDirty()) {
 454                 if (tryLock()) {
 455                     LOG.debug("Service group {} re-running reconciliation", identifier);
 456                     continue;
 457                 }
 458
 459                 LOG.debug("Service group {} will be reconciled by someone else", identifier);
 460             } else {
 461                 LOG.debug("Service group {} is completely reconciled", identifier);
 462             }
 463
 464             break;
 465         }
 466     }
 467
 468     private void serviceTransitionCompleted() {
 469         markDirty();
 470         if (tryLock()) {
 471             reconcileState();
 472         }
 473     }
 474
 475     // Has to be called with lock asserted
 476     private void tryReconcileState() {
 477         // First take a safe snapshot of current state on which we will base our decisions.
 478         final Set<ClusterSingletonServiceRegistration> localMembers;
 479         final boolean haveCleanup;
 480         final boolean haveService;
 481         synchronized (this) {
 482             if (serviceEntityReg != null) {
 483                 haveService = switch (serviceEntityState) {
 484                     case OWNED, OWNED_JEOPARDY -> true;
 485                     case REGISTERED, UNOWNED, UNREGISTERED -> false;
 486                 };
 487             } else {
 488                 haveService = false;
 489             }
 490
 491             if (haveService && cleanupEntityReg == null) {
 492                 // We have the service entity but have not registered for cleanup entity. Do that now and retry.
 493                 LOG.debug("Service group {} registering cleanup entity", identifier);
 494                 try {
 495                     cleanupEntityState = EntityState.REGISTERED;
 496                     cleanupEntityReg = entityOwnershipService.registerCandidate(cleanupEntity);
 497                 } catch (CandidateAlreadyRegisteredException e) {
 498                     LOG.error("Service group {} failed to take ownership, aborting", identifier, e);
 499                     if (serviceEntityReg != null) {
 500                         serviceEntityReg.close();
 501                         serviceEntityReg = null;
 502                     }
 503                 }
 504                 markDirty();
 505                 return;
 506             }
 507
 508             if (cleanupEntityReg != null) {
 509                 haveCleanup = switch (cleanupEntityState) {
 510                     case OWNED -> true;
 511                     case OWNED_JEOPARDY, REGISTERED, UNOWNED, UNREGISTERED -> false;
 512                 };
 513             } else {
 514                 haveCleanup = false;
 515             }
 516
 517             localMembers = ImmutableSet.copyOf(members);
 518         }
 519
 520         if (haveService && haveCleanup) {
 521             ensureServicesStarting(localMembers);
 522             return;
 523         }
 524
 525         ensureServicesStopping();
 526
 527         if (!haveService && services.isEmpty()) {
 528             LOG.debug("Service group {} has no running services", identifier);
 529             final boolean canFinishClose;
 530             synchronized (this) {
 531                 if (cleanupEntityReg != null) {
 532                     LOG.debug("Service group {} releasing cleanup entity", identifier);
 533                     cleanupEntityReg.close();
 534                     cleanupEntityReg = null;
 535                 }
 536
 537                 canFinishClose = switch (cleanupEntityState) {
 538                     case OWNED, OWNED_JEOPARDY, REGISTERED -> false;
 539                     case UNOWNED, UNREGISTERED -> true;
 540                 };
 541             }
 542
 543             if (canFinishClose) {
 544                 final SettableFuture<Void> localFuture = closeFuture.get();
 545                 if (localFuture != null && !localFuture.isDone()) {
 546                     LOG.debug("Service group {} completing termination", identifier);
 547                     localFuture.set(null);
 548                 }
 549             }
 550         }
 551     }
 552
 553     // Has to be called with lock asserted
 554     @SuppressWarnings("illegalCatch")
 555     private void ensureServicesStarting(final Set<ClusterSingletonServiceRegistration> localConfig) {
 556         LOG.debug("Service group {} starting services", identifier);
 557
 558         // This may look counter-intuitive, but the localConfig may be missing some services that are started -- for
 559         // example when this method is executed as part of unregisterService() call. In that case we need to ensure
 560         // services in the list are stopping
 561         final Iterator<Entry<ClusterSingletonServiceRegistration, ServiceInfo>> it = services.entrySet().iterator();
 562         while (it.hasNext()) {
 563             final Entry<ClusterSingletonServiceRegistration, ServiceInfo> entry = it.next();
 564             final ClusterSingletonServiceRegistration reg = entry.getKey();
 565             if (!localConfig.contains(reg)) {
 566                 final ServiceInfo newInfo = ensureStopping(reg, entry.getValue());
 567                 if (newInfo != null) {
 568                     entry.setValue(newInfo);
 569                 } else {
 570                     it.remove();
 571                 }
 572             }
 573         }
 574
 575         // Now make sure member services are being juggled around
 576         for (ClusterSingletonServiceRegistration reg : localConfig) {
 577             if (!services.containsKey(reg)) {
 578                 final ClusterSingletonService service = reg.getInstance();
 579                 LOG.debug("Starting service {}", service);
 580
 581                 try {
 582                     service.instantiateServiceInstance();
 583                 } catch (Exception e) {
 584                     LOG.warn("Service group {} service {} failed to start, attempting to continue", identifier, service,
 585                         e);
 586                     continue;
 587                 }
 588
 589                 services.put(reg, ServiceInfo.started());
 590             }
 591         }
 592     }
 593
 594     // Has to be called with lock asserted
 595     private void ensureServicesStopping() {
 596         final Iterator<Entry<ClusterSingletonServiceRegistration, ServiceInfo>> it = services.entrySet().iterator();
 597         while (it.hasNext()) {
 598             final Entry<ClusterSingletonServiceRegistration, ServiceInfo> entry = it.next();
 599             final ServiceInfo newInfo = ensureStopping(entry.getKey(), entry.getValue());
 600             if (newInfo != null) {
 601                 entry.setValue(newInfo);
 602             } else {
 603                 it.remove();
 604             }
 605         }
 606     }
 607
 608     @SuppressWarnings("illegalCatch")
 609     private ServiceInfo ensureStopping(final ClusterSingletonServiceRegistration reg, final ServiceInfo info) {
 610         switch (info.getState()) {
 611             case STARTED:
 612                 final ClusterSingletonService service = reg.getInstance();
 613
 614                 LOG.debug("Service group {} stopping service {}", identifier, service);
 615                 final @NonNull ListenableFuture<?> future;
 616                 try {
 617                     future = verifyNotNull(service.closeServiceInstance());
 618                 } catch (Exception e) {
 619                     LOG.warn("Service group {} service {} failed to stop, attempting to continue", identifier, service,
 620                         e);
 621                     return null;
 622                 }
 623
 624                 Futures.addCallback(future, new FutureCallback<Object>() {
 625                     @Override
 626                     public void onSuccess(final Object result) {
 627                         LOG.debug("Service group {} service {} stopped successfully", identifier, service);
 628                         serviceTransitionCompleted();
 629                     }
 630
 631                     @Override
 632                     public void onFailure(final Throwable cause) {
 633                         LOG.debug("Service group {} service {} stopped with error", identifier, service, cause);
 634                         serviceTransitionCompleted();
 635                     }
 636                 }, MoreExecutors.directExecutor());
 637                 return info.toState(ServiceState.STOPPING, future);
 638             case STOPPING:
 639                 if (info.getFuture().isDone()) {
 640                     LOG.debug("Service group {} removed stopped service {}", identifier, reg.getInstance());
 641                     return null;
 642                 }
 643                 return info;
 644             default:
 645                 throw new IllegalStateException("Unhandled state " + info.getState());
 646         }
 647     }
 648
 649     private void markDirty() {
 650         dirty = 1;
 651     }
 652
 653     private boolean isDirty() {
 654         return dirty != 0;
 655     }
 656
 657     private boolean conditionalClean() {
 658         return DIRTY_UPDATER.compareAndSet(this, 1, 0);
 659     }
 660
 661     private boolean tryLock() {
 662         return LOCK_UPDATER.compareAndSet(this, 0, 1);
 663     }
 664
 665     private boolean unlock() {
 666         verify(LOCK_UPDATER.compareAndSet(this, 1, 0));
 667         return true;
 668     }
 669
 670     @Override
 671     public String toString() {
 672         return MoreObjects.toStringHelper(this).add("identifier", identifier).toString();
 673     }
 674 }