singleton-service/mdsal-singleton-dom-impl/src/main/java/org/opendaylight/mdsal/singleton/dom/impl/ClusterSingletonServiceGroupImpl.java

   1 /*
   2  * Copyright (c) 2016 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.mdsal.singleton.dom.impl;
   9
  10 import static com.google.common.base.Preconditions.checkArgument;
  11 import static com.google.common.base.Preconditions.checkState;
  12 import static com.google.common.base.Verify.verify;
  13 import static com.google.common.base.Verify.verifyNotNull;
  14 import static java.util.Objects.requireNonNull;
  15
  16 import com.google.common.annotations.VisibleForTesting;
  17 import com.google.common.base.MoreObjects;
  18 import com.google.common.collect.ImmutableList;
  19 import com.google.common.collect.ImmutableSet;
  20 import com.google.common.util.concurrent.FutureCallback;
  21 import com.google.common.util.concurrent.Futures;
  22 import com.google.common.util.concurrent.ListenableFuture;
  23 import com.google.common.util.concurrent.MoreExecutors;
  24 import com.google.common.util.concurrent.SettableFuture;
  25 import java.util.Collection;
  26 import java.util.HashMap;
  27 import java.util.Iterator;
  28 import java.util.Map;
  29 import java.util.Map.Entry;
  30 import java.util.Set;
  31 import java.util.concurrent.ConcurrentHashMap;
  32 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
  33 import java.util.concurrent.atomic.AtomicReference;
  34 import org.checkerframework.checker.lock.qual.GuardedBy;
  35 import org.checkerframework.checker.lock.qual.Holding;
  36 import org.eclipse.jdt.annotation.NonNull;
  37 import org.opendaylight.mdsal.eos.common.api.CandidateAlreadyRegisteredException;
  38 import org.opendaylight.mdsal.eos.common.api.EntityOwnershipChangeState;
  39 import org.opendaylight.mdsal.eos.common.api.GenericEntity;
  40 import org.opendaylight.mdsal.eos.common.api.GenericEntityOwnershipCandidateRegistration;
  41 import org.opendaylight.mdsal.eos.common.api.GenericEntityOwnershipChange;
  42 import org.opendaylight.mdsal.eos.common.api.GenericEntityOwnershipListener;
  43 import org.opendaylight.mdsal.eos.common.api.GenericEntityOwnershipService;
  44 import org.opendaylight.mdsal.singleton.common.api.ClusterSingletonService;
  45 import org.opendaylight.mdsal.singleton.common.api.ClusterSingletonServiceRegistration;
  46 import org.opendaylight.yangtools.concepts.HierarchicalIdentifier;
  47 import org.slf4j.Logger;
  48 import org.slf4j.LoggerFactory;
  49
  50 /**
  51  * Implementation of {@link ClusterSingletonServiceGroup} on top of the Entity Ownership Service. Since EOS is atomic
  52  * in its operation and singleton services incur startup and most notably cleanup, we need to do something smart here.
  53  *
  54  * <p>
  55  * The implementation takes advantage of the fact that EOS provides stable ownership, i.e. owners are not moved as
  56  * a result on new candidates appearing. We use two entities:
  57  * - service entity, to which all nodes register
  58  * - cleanup entity, which only the service entity owner registers to
  59  *
  60  * <p>
  61  * Once the cleanup entity ownership is acquired, services are started. As long as the cleanup entity is registered,
  62  * it should remain the owner. In case a new service owner emerges, the old owner will start the cleanup process,
  63  * eventually releasing the cleanup entity. The new owner registers for the cleanup entity -- but will not see it
  64  * granted until the old owner finishes the cleanup.
  65  *
  66  * @param <P> the instance identifier path type
  67  * @param <E> the GenericEntity type
  68  * @param <C> the GenericEntityOwnershipChange type
  69  * @param <G> the GenericEntityOwnershipListener type
  70  * @param <S> the GenericEntityOwnershipService type
  71  */
  72 final class ClusterSingletonServiceGroupImpl<P extends HierarchicalIdentifier<P>, E extends GenericEntity<P>,
  73         C extends GenericEntityOwnershipChange<P, E>,  G extends GenericEntityOwnershipListener<P, C>,
  74         S extends GenericEntityOwnershipService<P, E, G>> extends ClusterSingletonServiceGroup<P, E, C> {
  75
  76     private enum EntityState {
  77         /**
  78          * This entity was never registered.
  79          */
  80         UNREGISTERED,
  81         /**
  82          * Registration exists, but we are waiting for it to resolve.
  83          */
  84         REGISTERED,
  85         /**
  86          * Registration indicated we are the owner.
  87          */
  88         OWNED,
  89         /**
  90          * Registration indicated we are the owner, but global state is uncertain -- meaning there can be owners in
  91          * another partition, for example.
  92          */
  93         OWNED_JEOPARDY,
  94         /**
  95          * Registration indicated we are not the owner. In this state we do not care about global state, therefore we
  96          * do not need an UNOWNED_JEOPARDY state.
  97          */
  98         UNOWNED,
  99     }
 100
 101     enum ServiceState {
 102         /**
 103          * Local service is up and running.
 104          */
 105         // FIXME: we should support async startup, which will require a STARTING state.
 106         STARTED,
 107         /**
 108          * Local service is being stopped.
 109          */
 110         STOPPING,
 111     }
 112
 113     private static final Logger LOG = LoggerFactory.getLogger(ClusterSingletonServiceGroupImpl.class);
 114
 115     private final S entityOwnershipService;
 116     private final String identifier;
 117
 118     /* Entity instances */
 119     private final E serviceEntity;
 120     private final E cleanupEntity;
 121
 122     private final Set<ClusterSingletonServiceRegistration> members = ConcurrentHashMap.newKeySet();
 123     // Guarded by lock
 124     private final Map<ClusterSingletonServiceRegistration, ServiceInfo> services = new HashMap<>();
 125
 126     // Marker for when any state changed
 127     @SuppressWarnings("rawtypes")
 128     private static final AtomicIntegerFieldUpdater<ClusterSingletonServiceGroupImpl> DIRTY_UPDATER =
 129             AtomicIntegerFieldUpdater.newUpdater(ClusterSingletonServiceGroupImpl.class, "dirty");
 130     private volatile int dirty;
 131
 132     // Simplified lock: non-reentrant, support tryLock() only
 133     @SuppressWarnings("rawtypes")
 134     private static final AtomicIntegerFieldUpdater<ClusterSingletonServiceGroupImpl> LOCK_UPDATER =
 135             AtomicIntegerFieldUpdater.newUpdater(ClusterSingletonServiceGroupImpl.class, "lock");
 136     @SuppressWarnings("unused")
 137     private volatile int lock;
 138
 139     /*
 140      * State tracking is quite involved, as we are tracking up to four asynchronous sources of events:
 141      * - user calling close()
 142      * - service entity ownership
 143      * - cleanup entity ownership
 144      * - service shutdown future
 145      *
 146      * Absolutely correct solution would be a set of behaviors, which govern each state, remembering where we want to
 147      * get to and what we are doing. That would result in ~15 classes which would quickly render this code unreadable
 148      * due to boilerplate overhead.
 149      *
 150      * We therefore take a different approach, tracking state directly in this class and evaluate state transitions
 151      * based on recorded bits -- without explicit representation of state machine state.
 152      */
 153     /**
 154      * Group close future. In can only go from null to non-null reference. Whenever it is non-null, it indicates that
 155      * the user has closed the group and we are converging to termination.
 156      */
 157     // We are using volatile get-and-set to support non-blocking close(). It may be more efficient to inline it here,
 158     // as we perform a volatile read after unlocking -- that volatile read may easier on L1 cache.
 159     // XXX: above needs a microbenchmark contention ever becomes a problem.
 160     private final AtomicReference<SettableFuture<Void>> closeFuture = new AtomicReference<>();
 161
 162     /**
 163      * Service (base) entity registration. This entity selects an owner candidate across nodes. Candidates proceed to
 164      * acquire {@link #cleanupEntity}.
 165      */
 166     @GuardedBy("this")
 167     private GenericEntityOwnershipCandidateRegistration<P, E> serviceEntityReg = null;
 168     /**
 169      * Service (base) entity last reported state.
 170      */
 171     @GuardedBy("this")
 172     private EntityState serviceEntityState = EntityState.UNREGISTERED;
 173
 174     /**
 175      * Cleanup (owner) entity registration. This entity guards access to service state and coordinates shutdown cleanup
 176      * and startup.
 177      */
 178     @GuardedBy("this")
 179     private GenericEntityOwnershipCandidateRegistration<P, E> cleanupEntityReg;
 180     /**
 181      * Cleanup (owner) entity last reported state.
 182      */
 183     @GuardedBy("this")
 184     private EntityState cleanupEntityState = EntityState.UNREGISTERED;
 185
 186     private volatile boolean initialized;
 187
 188     /**
 189      * Class constructor. Note: last argument is reused as-is.
 190      *
 191      * @param identifier non-empty string as identifier
 192      * @param mainEntity as Entity instance
 193      * @param closeEntity as Entity instance
 194      * @param entityOwnershipService GenericEntityOwnershipService instance
 195      * @param parent parent service
 196      * @param services Services list
 197      */
 198     ClusterSingletonServiceGroupImpl(final String identifier, final S entityOwnershipService, final E mainEntity,
 199             final E closeEntity, final Collection<ClusterSingletonServiceRegistration> services) {
 200         checkArgument(!identifier.isEmpty(), "Identifier may not be empty");
 201         this.identifier = identifier;
 202         this.entityOwnershipService = requireNonNull(entityOwnershipService);
 203         this.serviceEntity = requireNonNull(mainEntity);
 204         this.cleanupEntity = requireNonNull(closeEntity);
 205         members.addAll(services);
 206
 207         LOG.debug("Instantiated new service group for {}", identifier);
 208     }
 209
 210     @VisibleForTesting
 211     ClusterSingletonServiceGroupImpl(final String identifier, final E mainEntity,
 212             final E closeEntity, final S entityOwnershipService) {
 213         this(identifier, entityOwnershipService, mainEntity, closeEntity, ImmutableList.of());
 214     }
 215
 216     @Override
 217     public String getIdentifier() {
 218         return identifier;
 219     }
 220
 221     @Override
 222     ListenableFuture<?> closeClusterSingletonGroup() {
 223         final ListenableFuture<?> ret = destroyGroup();
 224         members.clear();
 225         markDirty();
 226
 227         if (tryLock()) {
 228             reconcileState();
 229         } else {
 230             LOG.debug("Service group {} postponing sync on close", identifier);
 231         }
 232
 233         return ret;
 234     }
 235
 236     private boolean isClosed() {
 237         return closeFuture.get() != null;
 238     }
 239
 240     @Override
 241     void initialize() throws CandidateAlreadyRegisteredException {
 242         verify(tryLock());
 243         try {
 244             checkState(!initialized, "Singleton group %s was already initilized", identifier);
 245             LOG.debug("Initializing service group {} with services {}", identifier, members);
 246             synchronized (this) {
 247                 serviceEntityState = EntityState.REGISTERED;
 248                 serviceEntityReg = entityOwnershipService.registerCandidate(serviceEntity);
 249                 initialized = true;
 250             }
 251         } finally {
 252             unlock();
 253         }
 254     }
 255
 256     private void checkNotClosed() {
 257         checkState(!isClosed(), "Service group %s has already been closed", identifier);
 258     }
 259
 260     @Override
 261     void registerService(final ClusterSingletonServiceRegistration reg) {
 262         final ClusterSingletonService service = verifyRegistration(reg);
 263         checkNotClosed();
 264
 265         checkState(initialized, "Service group %s is not initialized yet", identifier);
 266
 267         // First put the service
 268         LOG.debug("Adding service {} to service group {}", service, identifier);
 269         verify(members.add(reg));
 270         markDirty();
 271
 272         if (!tryLock()) {
 273             LOG.debug("Service group {} delayed register of {}", identifier, reg);
 274             return;
 275         }
 276
 277         reconcileState();
 278     }
 279
 280     @Override
 281     ListenableFuture<?> unregisterService(final ClusterSingletonServiceRegistration reg) {
 282         verifyRegistration(reg);
 283         checkNotClosed();
 284
 285         verify(members.remove(reg));
 286         markDirty();
 287         if (members.isEmpty()) {
 288             // We need to let AbstractClusterSingletonServiceProviderImpl know this group is to be shutdown
 289             // before we start applying state, because while we do not re-enter, the user is free to do whatever,
 290             // notably including registering a service with the same ID from the service shutdown hook. That
 291             // registration request needs to hit the successor of this group.
 292             return destroyGroup();
 293         }
 294
 295         if (tryLock()) {
 296             reconcileState();
 297         } else {
 298             LOG.debug("Service group {} delayed unregister of {}", identifier, reg);
 299         }
 300         return null;
 301     }
 302
 303     private ClusterSingletonService verifyRegistration(final ClusterSingletonServiceRegistration reg) {
 304         final ClusterSingletonService service = reg.getInstance();
 305         verify(identifier.equals(service.getIdentifier().getName()));
 306         return service;
 307     }
 308
 309     private synchronized @NonNull ListenableFuture<?> destroyGroup() {
 310         final SettableFuture<Void> future = SettableFuture.create();
 311         if (!closeFuture.compareAndSet(null, future)) {
 312             return verifyNotNull(closeFuture.get());
 313         }
 314
 315         if (serviceEntityReg != null) {
 316             // We are still holding the service registration, close it now...
 317             LOG.debug("Service group {} unregistering service entity {}", identifier, serviceEntity);
 318             serviceEntityReg.close();
 319             serviceEntityReg = null;
 320         }
 321
 322         markDirty();
 323         return future;
 324     }
 325
 326     @Override
 327     void ownershipChanged(final C ownershipChange) {
 328         LOG.debug("Ownership change {} for ClusterSingletonServiceGroup {}", ownershipChange, identifier);
 329
 330         synchronized (this) {
 331             lockedOwnershipChanged(ownershipChange);
 332         }
 333
 334         if (isDirty()) {
 335             if (!tryLock()) {
 336                 LOG.debug("Service group {} postponing ownership change sync", identifier);
 337                 return;
 338             }
 339
 340             reconcileState();
 341         }
 342     }
 343
 344     /**
 345      * Handle an ownership change with the lock held. Callers are expected to handle termination conditions, this method
 346      * and anything it calls must not call {@link #lockedClose(SettableFuture)}.
 347      *
 348      * @param ownershipChange reported change
 349      */
 350     @Holding("this")
 351     private void lockedOwnershipChanged(final C ownershipChange) {
 352         final E entity = ownershipChange.getEntity();
 353         if (serviceEntity.equals(entity)) {
 354             serviceOwnershipChanged(ownershipChange.getState(), ownershipChange.inJeopardy());
 355             markDirty();
 356         } else if (cleanupEntity.equals(entity)) {
 357             cleanupCandidateOwnershipChanged(ownershipChange.getState(), ownershipChange.inJeopardy());
 358             markDirty();
 359         } else {
 360             LOG.warn("Group {} received unrecognized change {}", identifier, ownershipChange);
 361         }
 362     }
 363
 364     @Holding("this")
 365     private void cleanupCandidateOwnershipChanged(final EntityOwnershipChangeState state, final boolean jeopardy) {
 366         if (jeopardy) {
 367             switch (state) {
 368                 case LOCAL_OWNERSHIP_GRANTED:
 369                 case LOCAL_OWNERSHIP_RETAINED_WITH_NO_CHANGE:
 370                     LOG.warn("Service group {} cleanup entity owned without certainty", identifier);
 371                     cleanupEntityState = EntityState.OWNED_JEOPARDY;
 372                     break;
 373                 case LOCAL_OWNERSHIP_LOST_NEW_OWNER:
 374                 case LOCAL_OWNERSHIP_LOST_NO_OWNER:
 375                 case REMOTE_OWNERSHIP_CHANGED:
 376                 case REMOTE_OWNERSHIP_LOST_NO_OWNER:
 377                     LOG.info("Service group {} cleanup entity ownership uncertain", identifier);
 378                     cleanupEntityState = EntityState.UNOWNED;
 379                     break;
 380                 default:
 381                     throw new IllegalStateException("Unhandled cleanup entity jeopardy change " + state);
 382             }
 383
 384             return;
 385         }
 386
 387         if (cleanupEntityState == EntityState.OWNED_JEOPARDY) {
 388             // Pair info message with previous jeopardy
 389             LOG.info("Service group {} cleanup entity ownership ascertained", identifier);
 390         }
 391
 392         switch (state) {
 393             case LOCAL_OWNERSHIP_GRANTED:
 394             case LOCAL_OWNERSHIP_RETAINED_WITH_NO_CHANGE:
 395                 cleanupEntityState = EntityState.OWNED;
 396                 break;
 397             case LOCAL_OWNERSHIP_LOST_NEW_OWNER:
 398             case LOCAL_OWNERSHIP_LOST_NO_OWNER:
 399             case REMOTE_OWNERSHIP_LOST_NO_OWNER:
 400             case REMOTE_OWNERSHIP_CHANGED:
 401                 cleanupEntityState = EntityState.UNOWNED;
 402                 break;
 403             default:
 404                 LOG.warn("Service group {} ignoring unhandled cleanup entity change {}", identifier, state);
 405                 break;
 406         }
 407     }
 408
 409     @Holding("this")
 410     private void serviceOwnershipChanged(final EntityOwnershipChangeState state, final boolean jeopardy) {
 411         if (jeopardy) {
 412             LOG.info("Service group {} service entity ownership uncertain", identifier);
 413             switch (state) {
 414                 case LOCAL_OWNERSHIP_GRANTED:
 415                 case LOCAL_OWNERSHIP_RETAINED_WITH_NO_CHANGE:
 416                     serviceEntityState = EntityState.OWNED_JEOPARDY;
 417                     break;
 418                 case LOCAL_OWNERSHIP_LOST_NEW_OWNER:
 419                 case LOCAL_OWNERSHIP_LOST_NO_OWNER:
 420                 case REMOTE_OWNERSHIP_CHANGED:
 421                 case REMOTE_OWNERSHIP_LOST_NO_OWNER:
 422                     serviceEntityState = EntityState.UNOWNED;
 423                     break;
 424                 default:
 425                     throw new IllegalStateException("Unhandled cleanup entity jeopardy change " + state);
 426             }
 427             return;
 428         }
 429
 430         if (serviceEntityState == EntityState.OWNED_JEOPARDY) {
 431             // Pair info message with previous jeopardy
 432             LOG.info("Service group {} service entity ownership ascertained", identifier);
 433         }
 434
 435         switch (state) {
 436             case LOCAL_OWNERSHIP_GRANTED:
 437             case LOCAL_OWNERSHIP_RETAINED_WITH_NO_CHANGE:
 438                 LOG.debug("Service group {} acquired service entity ownership", identifier);
 439                 serviceEntityState = EntityState.OWNED;
 440                 break;
 441             case LOCAL_OWNERSHIP_LOST_NEW_OWNER:
 442             case LOCAL_OWNERSHIP_LOST_NO_OWNER:
 443             case REMOTE_OWNERSHIP_CHANGED:
 444             case REMOTE_OWNERSHIP_LOST_NO_OWNER:
 445                 LOG.debug("Service group {} lost service entity ownership", identifier);
 446                 serviceEntityState = EntityState.UNOWNED;
 447                 break;
 448             default:
 449                 LOG.warn("Service group {} ignoring unhandled cleanup entity change {}", identifier, state);
 450         }
 451     }
 452
 453     // has to be called with lock asserted, which will be released prior to returning
 454     private void reconcileState() {
 455         // Always check if there is any state change to be applied.
 456         while (true) {
 457             try {
 458                 if (conditionalClean()) {
 459                     tryReconcileState();
 460                 }
 461             } finally {
 462                 // We may have ran a round of reconciliation, but the either one of may have happened asynchronously:
 463                 // - registration
 464                 // - unregistration
 465                 // - service future completed
 466                 // - entity state changed
 467                 //
 468                 // We are dropping the lock, but we need to recheck dirty and try to apply state again if it is found to
 469                 // be dirty again. This closes the following race condition:
 470                 //
 471                 // A: runs these checks holding the lock
 472                 // B: modifies them, fails to acquire lock
 473                 // A: releases lock -> noone takes care of reconciliation
 474
 475                 unlock();
 476             }
 477
 478             if (isDirty()) {
 479                 if (tryLock()) {
 480                     LOG.debug("Service group {} re-running reconciliation", identifier);
 481                     continue;
 482                 }
 483
 484                 LOG.debug("Service group {} will be reconciled by someone else", identifier);
 485             } else {
 486                 LOG.debug("Service group {} is completely reconciled", identifier);
 487             }
 488
 489             break;
 490         }
 491     }
 492
 493     private void serviceTransitionCompleted() {
 494         markDirty();
 495         if (tryLock()) {
 496             reconcileState();
 497         }
 498     }
 499
 500     // Has to be called with lock asserted
 501     private void tryReconcileState() {
 502         // First take a safe snapshot of current state on which we will base our decisions.
 503         final Set<ClusterSingletonServiceRegistration> localMembers;
 504         final boolean haveCleanup;
 505         final boolean haveService;
 506         synchronized (this) {
 507             if (serviceEntityReg != null) {
 508                 switch (serviceEntityState) {
 509                     case OWNED:
 510                     case OWNED_JEOPARDY:
 511                         haveService = true;
 512                         break;
 513                     case REGISTERED:
 514                     case UNOWNED:
 515                     case UNREGISTERED:
 516                         haveService = false;
 517                         break;
 518                     default:
 519                         throw new IllegalStateException("Unhandled service entity state " + serviceEntityState);
 520                 }
 521             } else {
 522                 haveService = false;
 523             }
 524
 525             if (haveService && cleanupEntityReg == null) {
 526                 // We have the service entity but have not registered for cleanup entity. Do that now and retry.
 527                 LOG.debug("Service group {} registering cleanup entity", identifier);
 528                 try {
 529                     cleanupEntityState = EntityState.REGISTERED;
 530                     cleanupEntityReg = entityOwnershipService.registerCandidate(cleanupEntity);
 531                 } catch (CandidateAlreadyRegisteredException e) {
 532                     LOG.error("Service group {} failed to take ownership, aborting", identifier, e);
 533                     if (serviceEntityReg != null) {
 534                         serviceEntityReg.close();
 535                         serviceEntityReg = null;
 536                     }
 537                 }
 538                 markDirty();
 539                 return;
 540             }
 541
 542             if (cleanupEntityReg != null) {
 543                 switch (cleanupEntityState) {
 544                     case OWNED:
 545                         haveCleanup = true;
 546                         break;
 547                     case OWNED_JEOPARDY:
 548                     case REGISTERED:
 549                     case UNOWNED:
 550                     case UNREGISTERED:
 551                         haveCleanup = false;
 552                         break;
 553                     default:
 554                         throw new IllegalStateException("Unhandled service entity state " + serviceEntityState);
 555                 }
 556             } else {
 557                 haveCleanup = false;
 558             }
 559
 560             localMembers = ImmutableSet.copyOf(members);
 561         }
 562
 563         if (haveService && haveCleanup) {
 564             ensureServicesStarting(localMembers);
 565             return;
 566         }
 567
 568         ensureServicesStopping();
 569
 570         if (!haveService && services.isEmpty()) {
 571             LOG.debug("Service group {} has no running services", identifier);
 572             final boolean canFinishClose;
 573             synchronized (this) {
 574                 if (cleanupEntityReg != null) {
 575                     LOG.debug("Service group {} releasing cleanup entity", identifier);
 576                     cleanupEntityReg.close();
 577                     cleanupEntityReg = null;
 578                 }
 579
 580                 switch (cleanupEntityState) {
 581                     case OWNED:
 582                     case OWNED_JEOPARDY:
 583                     case REGISTERED:
 584                         // When we are registered we need to wait for registration to resolve, otherwise
 585                         // the notification could be routed to the next incarnation of this group -- which could be
 586                         // confused by the fact it is not registered, but receives, for example, OWNED notification.
 587                         canFinishClose = false;
 588                         break;
 589                     case UNOWNED:
 590                     case UNREGISTERED:
 591                         canFinishClose = true;
 592                         break;
 593                     default:
 594                         throw new IllegalStateException("Unhandled cleanup entity state " + cleanupEntityState);
 595                 }
 596             }
 597
 598             if (canFinishClose) {
 599                 final SettableFuture<Void> localFuture = closeFuture.get();
 600                 if (localFuture != null && !localFuture.isDone()) {
 601                     LOG.debug("Service group {} completing termination", identifier);
 602                     localFuture.set(null);
 603                 }
 604             }
 605         }
 606     }
 607
 608     // Has to be called with lock asserted
 609     @SuppressWarnings("illegalCatch")
 610     private void ensureServicesStarting(final Set<ClusterSingletonServiceRegistration> localConfig) {
 611         LOG.debug("Service group {} starting services", identifier);
 612
 613         // This may look counter-intuitive, but the localConfig may be missing some services that are started -- for
 614         // example when this method is executed as part of unregisterService() call. In that case we need to ensure
 615         // services in the list are stopping
 616         final Iterator<Entry<ClusterSingletonServiceRegistration, ServiceInfo>> it = services.entrySet().iterator();
 617         while (it.hasNext()) {
 618             final Entry<ClusterSingletonServiceRegistration, ServiceInfo> entry = it.next();
 619             final ClusterSingletonServiceRegistration reg = entry.getKey();
 620             if (!localConfig.contains(reg)) {
 621                 final ServiceInfo newInfo = ensureStopping(reg, entry.getValue());
 622                 if (newInfo != null) {
 623                     entry.setValue(newInfo);
 624                 } else {
 625                     it.remove();
 626                 }
 627             }
 628         }
 629
 630         // Now make sure member services are being juggled around
 631         for (ClusterSingletonServiceRegistration reg : localConfig) {
 632             if (!services.containsKey(reg)) {
 633                 final ClusterSingletonService service = reg.getInstance();
 634                 LOG.debug("Starting service {}", service);
 635
 636                 try {
 637                     service.instantiateServiceInstance();
 638                 } catch (Exception e) {
 639                     LOG.warn("Service group {} service {} failed to start, attempting to continue", identifier, service,
 640                         e);
 641                     continue;
 642                 }
 643
 644                 services.put(reg, ServiceInfo.started());
 645             }
 646         }
 647     }
 648
 649     // Has to be called with lock asserted
 650     private void ensureServicesStopping() {
 651         final Iterator<Entry<ClusterSingletonServiceRegistration, ServiceInfo>> it = services.entrySet().iterator();
 652         while (it.hasNext()) {
 653             final Entry<ClusterSingletonServiceRegistration, ServiceInfo> entry = it.next();
 654             final ServiceInfo newInfo = ensureStopping(entry.getKey(), entry.getValue());
 655             if (newInfo != null) {
 656                 entry.setValue(newInfo);
 657             } else {
 658                 it.remove();
 659             }
 660         }
 661     }
 662
 663     @SuppressWarnings("illegalCatch")
 664     private ServiceInfo ensureStopping(final ClusterSingletonServiceRegistration reg, final ServiceInfo info) {
 665         switch (info.getState()) {
 666             case STARTED:
 667                 final ClusterSingletonService service = reg.getInstance();
 668
 669                 LOG.debug("Service group {} stopping service {}", identifier, service);
 670                 final @NonNull ListenableFuture<?> future;
 671                 try {
 672                     future = verifyNotNull(service.closeServiceInstance());
 673                 } catch (Exception e) {
 674                     LOG.warn("Service group {} service {} failed to stop, attempting to continue", identifier, service,
 675                         e);
 676                     return null;
 677                 }
 678
 679                 Futures.addCallback(future, new FutureCallback<Object>() {
 680                     @Override
 681                     public void onSuccess(final Object result) {
 682                         LOG.debug("Service group {} service {} stopped successfully", identifier, service);
 683                         serviceTransitionCompleted();
 684                     }
 685
 686                     @Override
 687                     public void onFailure(final Throwable cause) {
 688                         LOG.debug("Service group {} service {} stopped with error", identifier, service, cause);
 689                         serviceTransitionCompleted();
 690                     }
 691                 }, MoreExecutors.directExecutor());
 692                 return info.toState(ServiceState.STOPPING, future);
 693             case STOPPING:
 694                 if (info.getFuture().isDone()) {
 695                     LOG.debug("Service group {} removed stopped service {}", identifier, reg.getInstance());
 696                     return null;
 697                 }
 698                 return info;
 699             default:
 700                 throw new IllegalStateException("Unhandled state " + info.getState());
 701         }
 702     }
 703
 704     private void markDirty() {
 705         dirty = 1;
 706     }
 707
 708     private boolean isDirty() {
 709         return dirty != 0;
 710     }
 711
 712     private boolean conditionalClean() {
 713         return DIRTY_UPDATER.compareAndSet(this, 1, 0);
 714     }
 715
 716     private boolean tryLock() {
 717         return LOCK_UPDATER.compareAndSet(this, 0, 1);
 718     }
 719
 720     private boolean unlock() {
 721         verify(LOCK_UPDATER.compareAndSet(this, 1, 0));
 722         return true;
 723     }
 724
 725     @Override
 726     public String toString() {
 727         return MoreObjects.toStringHelper(this).add("identifier", identifier).toString();
 728     }
 729 }