singleton-service/mdsal-singleton-dom-impl/src/main/java/org/opendaylight/mdsal/singleton/dom/impl/ClusterSingletonServiceGroupImpl.java

   1 /*
   2  * Copyright (c) 2016 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.mdsal.singleton.dom.impl;
   9
  10 import static com.google.common.base.Preconditions.checkArgument;
  11 import static com.google.common.base.Preconditions.checkState;
  12 import static com.google.common.base.Verify.verify;
  13 import static com.google.common.base.Verify.verifyNotNull;
  14 import static java.util.Objects.requireNonNull;
  15
  16 import com.google.common.annotations.VisibleForTesting;
  17 import com.google.common.base.MoreObjects;
  18 import com.google.common.collect.ImmutableList;
  19 import com.google.common.collect.ImmutableSet;
  20 import com.google.common.util.concurrent.FutureCallback;
  21 import com.google.common.util.concurrent.Futures;
  22 import com.google.common.util.concurrent.ListenableFuture;
  23 import com.google.common.util.concurrent.MoreExecutors;
  24 import com.google.common.util.concurrent.SettableFuture;
  25 import java.util.Collection;
  26 import java.util.HashMap;
  27 import java.util.Iterator;
  28 import java.util.Map;
  29 import java.util.Map.Entry;
  30 import java.util.Set;
  31 import java.util.concurrent.ConcurrentHashMap;
  32 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
  33 import java.util.concurrent.atomic.AtomicReference;
  34 import javax.annotation.concurrent.GuardedBy;
  35 import javax.annotation.concurrent.ThreadSafe;
  36 import org.eclipse.jdt.annotation.NonNull;
  37 import org.opendaylight.mdsal.eos.common.api.CandidateAlreadyRegisteredException;
  38 import org.opendaylight.mdsal.eos.common.api.EntityOwnershipChangeState;
  39 import org.opendaylight.mdsal.eos.common.api.GenericEntity;
  40 import org.opendaylight.mdsal.eos.common.api.GenericEntityOwnershipCandidateRegistration;
  41 import org.opendaylight.mdsal.eos.common.api.GenericEntityOwnershipChange;
  42 import org.opendaylight.mdsal.eos.common.api.GenericEntityOwnershipListener;
  43 import org.opendaylight.mdsal.eos.common.api.GenericEntityOwnershipService;
  44 import org.opendaylight.mdsal.singleton.common.api.ClusterSingletonService;
  45 import org.opendaylight.mdsal.singleton.common.api.ClusterSingletonServiceRegistration;
  46 import org.opendaylight.yangtools.concepts.Path;
  47 import org.slf4j.Logger;
  48 import org.slf4j.LoggerFactory;
  49
  50 /**
  51  * Implementation of {@link ClusterSingletonServiceGroup} on top of the Entity Ownership Service. Since EOS is atomic
  52  * in its operation and singleton services incur startup and most notably cleanup, we need to do something smart here.
  53  *
  54  * <p>
  55  * The implementation takes advantage of the fact that EOS provides stable ownership, i.e. owners are not moved as
  56  * a result on new candidates appearing. We use two entities:
  57  * - service entity, to which all nodes register
  58  * - cleanup entity, which only the service entity owner registers to
  59  *
  60  * <p>
  61  * Once the cleanup entity ownership is acquired, services are started. As long as the cleanup entity is registered,
  62  * it should remain the owner. In case a new service owner emerges, the old owner will start the cleanup process,
  63  * eventually releasing the cleanup entity. The new owner registers for the cleanup entity -- but will not see it
  64  * granted until the old owner finishes the cleanup.
  65  *
  66  * @param <P> the instance identifier path type
  67  * @param <E> the GenericEntity type
  68  * @param <C> the GenericEntityOwnershipChange type
  69  * @param <G> the GenericEntityOwnershipListener type
  70  * @param <S> the GenericEntityOwnershipService type
  71  */
  72 @ThreadSafe
  73 final class ClusterSingletonServiceGroupImpl<P extends Path<P>, E extends GenericEntity<P>,
  74         C extends GenericEntityOwnershipChange<P, E>,  G extends GenericEntityOwnershipListener<P, C>,
  75         S extends GenericEntityOwnershipService<P, E, G>> extends ClusterSingletonServiceGroup<P, E, C> {
  76
  77     private enum EntityState {
  78         /**
  79          * This entity was never registered.
  80          */
  81         UNREGISTERED,
  82         /**
  83          * Registration exists, but we are waiting for it to resolve.
  84          */
  85         REGISTERED,
  86         /**
  87          * Registration indicated we are the owner.
  88          */
  89         OWNED,
  90         /**
  91          * Registration indicated we are the owner, but global state is uncertain -- meaning there can be owners in
  92          * another partition, for example.
  93          */
  94         OWNED_JEOPARDY,
  95         /**
  96          * Registration indicated we are not the owner. In this state we do not care about global state, therefore we
  97          * do not need an UNOWNED_JEOPARDY state.
  98          */
  99         UNOWNED,
 100     }
 101
 102     enum ServiceState {
 103         /**
 104          * Local service is up and running.
 105          */
 106         // FIXME: we should support async startup, which will require a STARTING state.
 107         STARTED,
 108         /**
 109          * Local service is being stopped.
 110          */
 111         STOPPING,
 112     }
 113
 114     private static final Logger LOG = LoggerFactory.getLogger(ClusterSingletonServiceGroupImpl.class);
 115
 116     private final S entityOwnershipService;
 117     private final String identifier;
 118
 119     /* Entity instances */
 120     private final E serviceEntity;
 121     private final E cleanupEntity;
 122
 123     private final Set<ClusterSingletonServiceRegistration> members = ConcurrentHashMap.newKeySet();
 124     // Guarded by lock
 125     private final Map<ClusterSingletonServiceRegistration, ServiceInfo> services = new HashMap<>();
 126
 127     // Marker for when any state changed
 128     @SuppressWarnings("rawtypes")
 129     private static final AtomicIntegerFieldUpdater<ClusterSingletonServiceGroupImpl> DIRTY_UPDATER =
 130             AtomicIntegerFieldUpdater.newUpdater(ClusterSingletonServiceGroupImpl.class, "dirty");
 131     private volatile int dirty;
 132
 133     // Simplified lock: non-reentrant, support tryLock() only
 134     @SuppressWarnings("rawtypes")
 135     private static final AtomicIntegerFieldUpdater<ClusterSingletonServiceGroupImpl> LOCK_UPDATER =
 136             AtomicIntegerFieldUpdater.newUpdater(ClusterSingletonServiceGroupImpl.class, "lock");
 137     @SuppressWarnings("unused")
 138     private volatile int lock;
 139
 140     /*
 141      * State tracking is quite involved, as we are tracking up to four asynchronous sources of events:
 142      * - user calling close()
 143      * - service entity ownership
 144      * - cleanup entity ownership
 145      * - service shutdown future
 146      *
 147      * Absolutely correct solution would be a set of behaviors, which govern each state, remembering where we want to
 148      * get to and what we are doing. That would result in ~15 classes which would quickly render this code unreadable
 149      * due to boilerplate overhead.
 150      *
 151      * We therefore take a different approach, tracking state directly in this class and evaluate state transitions
 152      * based on recorded bits -- without explicit representation of state machine state.
 153      */
 154     /**
 155      * Group close future. In can only go from null to non-null reference. Whenever it is non-null, it indicates that
 156      * the user has closed the group and we are converging to termination.
 157      */
 158     // We are using volatile get-and-set to support non-blocking close(). It may be more efficient to inline it here,
 159     // as we perform a volatile read after unlocking -- that volatile read may easier on L1 cache.
 160     // XXX: above needs a microbenchmark contention ever becomes a problem.
 161     private final AtomicReference<SettableFuture<Void>> closeFuture = new AtomicReference<>();
 162
 163     /**
 164      * Service (base) entity registration. This entity selects an owner candidate across nodes. Candidates proceed to
 165      * acquire {@link #cleanupEntity}.
 166      */
 167     @GuardedBy("this")
 168     private GenericEntityOwnershipCandidateRegistration<P, E> serviceEntityReg = null;
 169     /**
 170      * Service (base) entity last reported state.
 171      */
 172     @GuardedBy("this")
 173     private EntityState serviceEntityState = EntityState.UNREGISTERED;
 174
 175     /**
 176      * Cleanup (owner) entity registration. This entity guards access to service state and coordinates shutdown cleanup
 177      * and startup.
 178      */
 179     @GuardedBy("this")
 180     private GenericEntityOwnershipCandidateRegistration<P, E> cleanupEntityReg;
 181     /**
 182      * Cleanup (owner) entity last reported state.
 183      */
 184     @GuardedBy("this")
 185     private EntityState cleanupEntityState = EntityState.UNREGISTERED;
 186
 187     private volatile boolean initialized;
 188
 189     /**
 190      * Class constructor. Note: last argument is reused as-is.
 191      *
 192      * @param identifier non-empty string as identifier
 193      * @param mainEntity as Entity instance
 194      * @param closeEntity as Entity instance
 195      * @param entityOwnershipService GenericEntityOwnershipService instance
 196      * @param parent parent service
 197      * @param services Services list
 198      */
 199     ClusterSingletonServiceGroupImpl(final String identifier, final S entityOwnershipService, final E mainEntity,
 200             final E closeEntity, final Collection<ClusterSingletonServiceRegistration> services) {
 201         checkArgument(!identifier.isEmpty(), "Identifier may not be empty");
 202         this.identifier = identifier;
 203         this.entityOwnershipService = requireNonNull(entityOwnershipService);
 204         this.serviceEntity = requireNonNull(mainEntity);
 205         this.cleanupEntity = requireNonNull(closeEntity);
 206         members.addAll(services);
 207
 208         LOG.debug("Instantiated new service group for {}", identifier);
 209     }
 210
 211     @VisibleForTesting
 212     ClusterSingletonServiceGroupImpl(final String identifier, final E mainEntity,
 213             final E closeEntity, final S entityOwnershipService) {
 214         this(identifier, entityOwnershipService, mainEntity, closeEntity, ImmutableList.of());
 215     }
 216
 217     @Override
 218     public String getIdentifier() {
 219         return identifier;
 220     }
 221
 222     @Override
 223     ListenableFuture<?> closeClusterSingletonGroup() {
 224         final ListenableFuture<?> ret = destroyGroup();
 225         members.clear();
 226         markDirty();
 227
 228         if (tryLock()) {
 229             reconcileState();
 230         } else {
 231             LOG.debug("Service group {} postponing sync on close", identifier);
 232         }
 233
 234         return ret;
 235     }
 236
 237     private boolean isClosed() {
 238         return closeFuture.get() != null;
 239     }
 240
 241     @Override
 242     void initialize() throws CandidateAlreadyRegisteredException {
 243         verify(tryLock());
 244         try {
 245             checkState(!initialized, "Singleton group %s was already initilized", identifier);
 246             LOG.debug("Initializing service group {} with services {}", identifier, members);
 247             synchronized (this) {
 248                 serviceEntityState = EntityState.REGISTERED;
 249                 serviceEntityReg = entityOwnershipService.registerCandidate(serviceEntity);
 250                 initialized = true;
 251             }
 252         } finally {
 253             unlock();
 254         }
 255     }
 256
 257     private void checkNotClosed() {
 258         checkState(!isClosed(), "Service group %s has already been closed", identifier);
 259     }
 260
 261     @Override
 262     void registerService(final ClusterSingletonServiceRegistration reg) {
 263         final ClusterSingletonService service = reg.getInstance();
 264         verify(identifier.equals(service.getIdentifier().getValue()));
 265         checkNotClosed();
 266
 267         checkState(initialized, "Service group %s is not initialized yet", identifier);
 268
 269         // First put the service
 270         LOG.debug("Adding service {} to service group {}", service, identifier);
 271         verify(members.add(reg));
 272         markDirty();
 273
 274         if (!tryLock()) {
 275             LOG.debug("Service group {} delayed register of {}", identifier, reg);
 276             return;
 277         }
 278
 279         reconcileState();
 280     }
 281
 282     @Override
 283     ListenableFuture<?> unregisterService(final ClusterSingletonServiceRegistration reg) {
 284         final ClusterSingletonService service = reg.getInstance();
 285         verify(identifier.equals(service.getIdentifier().getValue()));
 286         checkNotClosed();
 287
 288         verify(members.remove(reg));
 289         markDirty();
 290         if (members.isEmpty()) {
 291             // We need to let AbstractClusterSingletonServiceProviderImpl know this group is to be shutdown
 292             // before we start applying state, because while we do not re-enter, the user is free to do whatever,
 293             // notably including registering a service with the same ID from the service shutdown hook. That
 294             // registration request needs to hit the successor of this group.
 295             return destroyGroup();
 296         }
 297
 298         if (tryLock()) {
 299             reconcileState();
 300         } else {
 301             LOG.debug("Service group {} delayed unregister of {}", identifier, reg);
 302         }
 303         return null;
 304     }
 305
 306     private synchronized @NonNull ListenableFuture<?> destroyGroup() {
 307         final SettableFuture<Void> future = SettableFuture.create();
 308         if (!closeFuture.compareAndSet(null, future)) {
 309             return verifyNotNull(closeFuture.get());
 310         }
 311
 312         if (serviceEntityReg != null) {
 313             // We are still holding the service registration, close it now...
 314             LOG.debug("Service group {} unregistering service entity {}", identifier, serviceEntity);
 315             serviceEntityReg.close();
 316             serviceEntityReg = null;
 317         }
 318
 319         markDirty();
 320         return future;
 321     }
 322
 323     @Override
 324     void ownershipChanged(final C ownershipChange) {
 325         LOG.debug("Ownership change {} for ClusterSingletonServiceGroup {}", ownershipChange, identifier);
 326
 327         synchronized (this) {
 328             lockedOwnershipChanged(ownershipChange);
 329         }
 330
 331         if (isDirty()) {
 332             if (!tryLock()) {
 333                 LOG.debug("Service group {} postponing ownership change sync", identifier);
 334                 return;
 335             }
 336
 337             reconcileState();
 338         }
 339     }
 340
 341     /**
 342      * Handle an ownership change with the lock held. Callers are expected to handle termination conditions, this method
 343      * and anything it calls must not call {@link #lockedClose(SettableFuture)}.
 344      *
 345      * @param ownershipChange reported change
 346      */
 347     @GuardedBy("this")
 348     private void lockedOwnershipChanged(final C ownershipChange) {
 349         final E entity = ownershipChange.getEntity();
 350         if (serviceEntity.equals(entity)) {
 351             serviceOwnershipChanged(ownershipChange.getState(), ownershipChange.inJeopardy());
 352             markDirty();
 353         } else if (cleanupEntity.equals(entity)) {
 354             cleanupCandidateOwnershipChanged(ownershipChange.getState(), ownershipChange.inJeopardy());
 355             markDirty();
 356         } else {
 357             LOG.warn("Group {} received unrecognized change {}", identifier, ownershipChange);
 358         }
 359     }
 360
 361     @GuardedBy("this")
 362     private void cleanupCandidateOwnershipChanged(final EntityOwnershipChangeState state, final boolean jeopardy) {
 363         if (jeopardy) {
 364             switch (state) {
 365                 case LOCAL_OWNERSHIP_GRANTED:
 366                 case LOCAL_OWNERSHIP_RETAINED_WITH_NO_CHANGE:
 367                     LOG.warn("Service group {} cleanup entity owned without certainty", identifier);
 368                     cleanupEntityState = EntityState.OWNED_JEOPARDY;
 369                     break;
 370                 case LOCAL_OWNERSHIP_LOST_NEW_OWNER:
 371                 case LOCAL_OWNERSHIP_LOST_NO_OWNER:
 372                 case REMOTE_OWNERSHIP_CHANGED:
 373                 case REMOTE_OWNERSHIP_LOST_NO_OWNER:
 374                     LOG.info("Service group {} cleanup entity ownership uncertain", identifier);
 375                     cleanupEntityState = EntityState.UNOWNED;
 376                     break;
 377                 default:
 378                     throw new IllegalStateException("Unhandled cleanup entity jeopardy change " + state);
 379             }
 380
 381             return;
 382         }
 383
 384         if (cleanupEntityState == EntityState.OWNED_JEOPARDY) {
 385             // Pair info message with previous jeopardy
 386             LOG.info("Service group {} cleanup entity ownership ascertained", identifier);
 387         }
 388
 389         switch (state) {
 390             case LOCAL_OWNERSHIP_GRANTED:
 391             case LOCAL_OWNERSHIP_RETAINED_WITH_NO_CHANGE:
 392                 cleanupEntityState = EntityState.OWNED;
 393                 break;
 394             case LOCAL_OWNERSHIP_LOST_NEW_OWNER:
 395             case LOCAL_OWNERSHIP_LOST_NO_OWNER:
 396             case REMOTE_OWNERSHIP_LOST_NO_OWNER:
 397             case REMOTE_OWNERSHIP_CHANGED:
 398                 cleanupEntityState = EntityState.UNOWNED;
 399                 break;
 400             default:
 401                 LOG.warn("Service group {} ignoring unhandled cleanup entity change {}", identifier, state);
 402                 break;
 403         }
 404     }
 405
 406     @GuardedBy("this")
 407     private void serviceOwnershipChanged(final EntityOwnershipChangeState state, final boolean jeopardy) {
 408         if (jeopardy) {
 409             LOG.info("Service group {} service entity ownership uncertain", identifier);
 410             switch (state) {
 411                 case LOCAL_OWNERSHIP_GRANTED:
 412                 case LOCAL_OWNERSHIP_RETAINED_WITH_NO_CHANGE:
 413                     serviceEntityState = EntityState.OWNED_JEOPARDY;
 414                     break;
 415                 case LOCAL_OWNERSHIP_LOST_NEW_OWNER:
 416                 case LOCAL_OWNERSHIP_LOST_NO_OWNER:
 417                 case REMOTE_OWNERSHIP_CHANGED:
 418                 case REMOTE_OWNERSHIP_LOST_NO_OWNER:
 419                     serviceEntityState = EntityState.UNOWNED;
 420                     break;
 421                 default:
 422                     throw new IllegalStateException("Unhandled cleanup entity jeopardy change " + state);
 423             }
 424             return;
 425         }
 426
 427         if (serviceEntityState == EntityState.OWNED_JEOPARDY) {
 428             // Pair info message with previous jeopardy
 429             LOG.info("Service group {} service entity ownership ascertained", identifier);
 430         }
 431
 432         switch (state) {
 433             case LOCAL_OWNERSHIP_GRANTED:
 434             case LOCAL_OWNERSHIP_RETAINED_WITH_NO_CHANGE:
 435                 LOG.debug("Service group {} acquired service entity ownership", identifier);
 436                 serviceEntityState = EntityState.OWNED;
 437                 break;
 438             case LOCAL_OWNERSHIP_LOST_NEW_OWNER:
 439             case LOCAL_OWNERSHIP_LOST_NO_OWNER:
 440             case REMOTE_OWNERSHIP_CHANGED:
 441             case REMOTE_OWNERSHIP_LOST_NO_OWNER:
 442                 LOG.debug("Service group {} lost service entity ownership", identifier);
 443                 serviceEntityState = EntityState.UNOWNED;
 444                 break;
 445             default:
 446                 LOG.warn("Service group {} ignoring unhandled cleanup entity change {}", identifier, state);
 447         }
 448     }
 449
 450     // has to be called with lock asserted, which will be released prior to returning
 451     private void reconcileState() {
 452         // Always check if there is any state change to be applied.
 453         while (true) {
 454             try {
 455                 if (conditionalClean()) {
 456                     tryReconcileState();
 457                 }
 458             } finally {
 459                 // We may have ran a round of reconciliation, but the either one of may have happened asynchronously:
 460                 // - registration
 461                 // - unregistration
 462                 // - service future completed
 463                 // - entity state changed
 464                 //
 465                 // We are dropping the lock, but we need to recheck dirty and try to apply state again if it is found to
 466                 // be dirty again. This closes the following race condition:
 467                 //
 468                 // A: runs these checks holding the lock
 469                 // B: modifies them, fails to acquire lock
 470                 // A: releases lock -> noone takes care of reconciliation
 471
 472                 unlock();
 473             }
 474
 475             if (isDirty()) {
 476                 if (tryLock()) {
 477                     LOG.debug("Service group {} re-running reconciliation", identifier);
 478                     continue;
 479                 }
 480
 481                 LOG.debug("Service group {} will be reconciled by someone else", identifier);
 482             } else {
 483                 LOG.debug("Service group {} is completely reconciled", identifier);
 484             }
 485
 486             break;
 487         }
 488     }
 489
 490     private void serviceTransitionCompleted() {
 491         markDirty();
 492         if (tryLock()) {
 493             reconcileState();
 494         }
 495     }
 496
 497     // Has to be called with lock asserted
 498     private void tryReconcileState() {
 499         // First take a safe snapshot of current state on which we will base our decisions.
 500         final Set<ClusterSingletonServiceRegistration> localMembers;
 501         final boolean haveCleanup;
 502         final boolean haveService;
 503         synchronized (this) {
 504             if (serviceEntityReg != null) {
 505                 switch (serviceEntityState) {
 506                     case OWNED:
 507                     case OWNED_JEOPARDY:
 508                         haveService = true;
 509                         break;
 510                     case REGISTERED:
 511                     case UNOWNED:
 512                     case UNREGISTERED:
 513                         haveService = false;
 514                         break;
 515                     default:
 516                         throw new IllegalStateException("Unhandled service entity state " + serviceEntityState);
 517                 }
 518             } else {
 519                 haveService = false;
 520             }
 521
 522             if (haveService && cleanupEntityReg == null) {
 523                 // We have the service entity but have not registered for cleanup entity. Do that now and retry.
 524                 LOG.debug("Service group {} registering cleanup entity", identifier);
 525                 try {
 526                     cleanupEntityState = EntityState.REGISTERED;
 527                     cleanupEntityReg = entityOwnershipService.registerCandidate(cleanupEntity);
 528                 } catch (CandidateAlreadyRegisteredException e) {
 529                     LOG.error("Service group {} failed to take ownership, aborting", identifier, e);
 530                     if (serviceEntityReg != null) {
 531                         serviceEntityReg.close();
 532                         serviceEntityReg = null;
 533                     }
 534                 }
 535                 markDirty();
 536                 return;
 537             }
 538
 539             if (cleanupEntityReg != null) {
 540                 switch (cleanupEntityState) {
 541                     case OWNED:
 542                         haveCleanup = true;
 543                         break;
 544                     case OWNED_JEOPARDY:
 545                     case REGISTERED:
 546                     case UNOWNED:
 547                     case UNREGISTERED:
 548                         haveCleanup = false;
 549                         break;
 550                     default:
 551                         throw new IllegalStateException("Unhandled service entity state " + serviceEntityState);
 552                 }
 553             } else {
 554                 haveCleanup = false;
 555             }
 556
 557             localMembers = ImmutableSet.copyOf(members);
 558         }
 559
 560         if (haveService && haveCleanup) {
 561             ensureServicesStarting(localMembers);
 562             return;
 563         }
 564
 565         ensureServicesStopping();
 566
 567         if (!haveService && services.isEmpty()) {
 568             LOG.debug("Service group {} has no running services", identifier);
 569             final boolean canFinishClose;
 570             synchronized (this) {
 571                 if (cleanupEntityReg != null) {
 572                     LOG.debug("Service group {} releasing cleanup entity", identifier);
 573                     cleanupEntityReg.close();
 574                     cleanupEntityReg = null;
 575                 }
 576
 577                 switch (cleanupEntityState) {
 578                     case OWNED:
 579                     case OWNED_JEOPARDY:
 580                     case REGISTERED:
 581                         // When we are registered we need to wait for registration to resolve, otherwise
 582                         // the notification could be routed to the next incarnation of this group -- which could be
 583                         // confused by the fact it is not registered, but receives, for example, OWNED notification.
 584                         canFinishClose = false;
 585                         break;
 586                     case UNOWNED:
 587                     case UNREGISTERED:
 588                         canFinishClose = true;
 589                         break;
 590                     default:
 591                         throw new IllegalStateException("Unhandled cleanup entity state " + cleanupEntityState);
 592                 }
 593             }
 594
 595             if (canFinishClose) {
 596                 final SettableFuture<Void> localFuture = closeFuture.get();
 597                 if (localFuture != null && !localFuture.isDone()) {
 598                     LOG.debug("Service group {} completing termination", identifier);
 599                     localFuture.set(null);
 600                 }
 601             }
 602         }
 603     }
 604
 605     // Has to be called with lock asserted
 606     @SuppressWarnings("illegalCatch")
 607     private void ensureServicesStarting(final Set<ClusterSingletonServiceRegistration> localConfig) {
 608         LOG.debug("Service group {} starting services", identifier);
 609
 610         // This may look counter-intuitive, but the localConfig may be missing some services that are started -- for
 611         // example when this method is executed as part of unregisterService() call. In that case we need to ensure
 612         // services in the list are stopping
 613         final Iterator<Entry<ClusterSingletonServiceRegistration, ServiceInfo>> it = services.entrySet().iterator();
 614         while (it.hasNext()) {
 615             final Entry<ClusterSingletonServiceRegistration, ServiceInfo> entry = it.next();
 616             final ClusterSingletonServiceRegistration reg = entry.getKey();
 617             if (!localConfig.contains(reg)) {
 618                 final ServiceInfo newInfo = ensureStopping(reg, entry.getValue());
 619                 if (newInfo != null) {
 620                     entry.setValue(newInfo);
 621                 } else {
 622                     it.remove();
 623                 }
 624             }
 625         }
 626
 627         // Now make sure member services are being juggled around
 628         for (ClusterSingletonServiceRegistration reg : localConfig) {
 629             if (!services.containsKey(reg)) {
 630                 final ClusterSingletonService service = reg.getInstance();
 631                 LOG.debug("Starting service {}", service);
 632
 633                 try {
 634                     service.instantiateServiceInstance();
 635                 } catch (Exception e) {
 636                     LOG.warn("Service group {} service {} failed to start, attempting to continue", identifier, service,
 637                         e);
 638                     continue;
 639                 }
 640
 641                 services.put(reg, ServiceInfo.started());
 642             }
 643         }
 644     }
 645
 646     // Has to be called with lock asserted
 647     private void ensureServicesStopping() {
 648         final Iterator<Entry<ClusterSingletonServiceRegistration, ServiceInfo>> it = services.entrySet().iterator();
 649         while (it.hasNext()) {
 650             final Entry<ClusterSingletonServiceRegistration, ServiceInfo> entry = it.next();
 651             final ServiceInfo newInfo = ensureStopping(entry.getKey(), entry.getValue());
 652             if (newInfo != null) {
 653                 entry.setValue(newInfo);
 654             } else {
 655                 it.remove();
 656             }
 657         }
 658     }
 659
 660     @SuppressWarnings("illegalCatch")
 661     private ServiceInfo ensureStopping(final ClusterSingletonServiceRegistration reg, final ServiceInfo info) {
 662         switch (info.getState()) {
 663             case STARTED:
 664                 final ClusterSingletonService service = reg.getInstance();
 665
 666                 LOG.debug("Service group {} stopping service {}", identifier, service);
 667                 final @NonNull ListenableFuture<?> future;
 668                 try {
 669                     future = verifyNotNull(service.closeServiceInstance());
 670                 } catch (Exception e) {
 671                     LOG.warn("Service group {} service {} failed to stop, attempting to continue", identifier, service,
 672                         e);
 673                     return null;
 674                 }
 675
 676                 Futures.addCallback(future, new FutureCallback<Object>() {
 677                     @Override
 678                     public void onSuccess(final Object result) {
 679                         LOG.debug("Service group {} service {} stopped successfully", identifier, service);
 680                         serviceTransitionCompleted();
 681                     }
 682
 683                     @Override
 684                     public void onFailure(final Throwable cause) {
 685                         LOG.debug("Service group {} service {} stopped with error", identifier, service, cause);
 686                         serviceTransitionCompleted();
 687                     }
 688                 }, MoreExecutors.directExecutor());
 689                 return info.toState(ServiceState.STOPPING, future);
 690             case STOPPING:
 691                 if (info.getFuture().isDone()) {
 692                     LOG.debug("Service group {} removed stopped service {}", identifier, reg.getInstance());
 693                     return null;
 694                 }
 695                 return info;
 696             default:
 697                 throw new IllegalStateException("Unhandled state " + info.getState());
 698         }
 699     }
 700
 701     private void markDirty() {
 702         dirty = 1;
 703     }
 704
 705     private boolean isDirty() {
 706         return dirty != 0;
 707     }
 708
 709     private boolean conditionalClean() {
 710         return DIRTY_UPDATER.compareAndSet(this, 1, 0);
 711     }
 712
 713     private boolean tryLock() {
 714         return LOCK_UPDATER.compareAndSet(this, 0, 1);
 715     }
 716
 717     private boolean unlock() {
 718         verify(LOCK_UPDATER.compareAndSet(this, 1, 0));
 719         return true;
 720     }
 721
 722     @Override
 723     public String toString() {
 724         return MoreObjects.toStringHelper(this).add("identifier", identifier).toString();
 725     }
 726 }