singleton-service/mdsal-singleton-dom-impl/src/main/java/org/opendaylight/mdsal/singleton/dom/impl/ClusterSingletonServiceGroupImpl.java

   1 /*
   2  * Copyright (c) 2016 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.mdsal.singleton.dom.impl;
   9
  10 import static com.google.common.base.Preconditions.checkArgument;
  11 import static com.google.common.base.Preconditions.checkState;
  12 import static com.google.common.base.Verify.verify;
  13 import static com.google.common.base.Verify.verifyNotNull;
  14 import static java.util.Objects.requireNonNull;
  15
  16 import com.google.common.annotations.VisibleForTesting;
  17 import com.google.common.base.MoreObjects;
  18 import com.google.common.collect.ImmutableList;
  19 import com.google.common.collect.ImmutableSet;
  20 import com.google.common.util.concurrent.FutureCallback;
  21 import com.google.common.util.concurrent.Futures;
  22 import com.google.common.util.concurrent.ListenableFuture;
  23 import com.google.common.util.concurrent.MoreExecutors;
  24 import com.google.common.util.concurrent.SettableFuture;
  25 import java.util.Collection;
  26 import java.util.HashMap;
  27 import java.util.Iterator;
  28 import java.util.Map;
  29 import java.util.Map.Entry;
  30 import java.util.Set;
  31 import java.util.concurrent.ConcurrentHashMap;
  32 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
  33 import java.util.concurrent.atomic.AtomicReference;
  34 import javax.annotation.CheckReturnValue;
  35 import javax.annotation.concurrent.GuardedBy;
  36 import javax.annotation.concurrent.ThreadSafe;
  37 import org.eclipse.jdt.annotation.NonNull;
  38 import org.opendaylight.mdsal.eos.common.api.CandidateAlreadyRegisteredException;
  39 import org.opendaylight.mdsal.eos.common.api.EntityOwnershipChangeState;
  40 import org.opendaylight.mdsal.eos.common.api.GenericEntity;
  41 import org.opendaylight.mdsal.eos.common.api.GenericEntityOwnershipCandidateRegistration;
  42 import org.opendaylight.mdsal.eos.common.api.GenericEntityOwnershipChange;
  43 import org.opendaylight.mdsal.eos.common.api.GenericEntityOwnershipListener;
  44 import org.opendaylight.mdsal.eos.common.api.GenericEntityOwnershipService;
  45 import org.opendaylight.mdsal.singleton.common.api.ClusterSingletonService;
  46 import org.opendaylight.mdsal.singleton.common.api.ClusterSingletonServiceRegistration;
  47 import org.opendaylight.yangtools.concepts.Path;
  48 import org.slf4j.Logger;
  49 import org.slf4j.LoggerFactory;
  50
  51 /**
  52  * Implementation of {@link ClusterSingletonServiceGroup} on top of the Entitiy Ownership Service. Since EOS is atomic
  53  * in its operation and singleton services incur startup and most notably cleanup, we need to do something smart here.
  54  *
  55  * <p>
  56  * The implementation takes advantage of the fact that EOS provides stable ownership, i.e. owners are not moved as
  57  * a result on new candidates appearing. We use two entities:
  58  * - service entity, to which all nodes register
  59  * - cleanup entity, which only the service entity owner registers to
  60  *
  61  * <p>
  62  * Once the cleanup entity ownership is acquired, services are started. As long as the cleanup entity is registered,
  63  * it should remain the owner. In case a new service owner emerges, the old owner will start the cleanup process,
  64  * eventually releasing the cleanup entity. The new owner registers for the cleanup entity -- but will not see it
  65  * granted until the old owner finishes the cleanup.
  66  *
  67  * @param <P> the instance identifier path type
  68  * @param <E> the GenericEntity type
  69  * @param <C> the GenericEntityOwnershipChange type
  70  * @param <G> the GenericEntityOwnershipListener type
  71  * @param <S> the GenericEntityOwnershipService type
  72  */
  73 @ThreadSafe
  74 final class ClusterSingletonServiceGroupImpl<P extends Path<P>, E extends GenericEntity<P>,
  75         C extends GenericEntityOwnershipChange<P, E>,  G extends GenericEntityOwnershipListener<P, C>,
  76         S extends GenericEntityOwnershipService<P, E, G>> extends ClusterSingletonServiceGroup<P, E, C> {
  77
  78     private enum EntityState {
  79         /**
  80          * This entity was never registered.
  81          */
  82         UNREGISTERED,
  83         /**
  84          * Registration exists, but we are waiting for it to resolve.
  85          */
  86         REGISTERED,
  87         /**
  88          * Registration indicated we are the owner.
  89          */
  90         OWNED,
  91         /**
  92          * Registration indicated we are the owner, but global state is uncertain -- meaning there can be owners in
  93          * another partition, for example.
  94          */
  95         OWNED_JEOPARDY,
  96         /**
  97          * Registration indicated we are not the owner. In this state we do not care about global state, therefore we
  98          * do not need an UNOWNED_JEOPARDY state.
  99          */
 100         UNOWNED,
 101     }
 102
 103     enum ServiceState {
 104         /**
 105          * Local service is up and running.
 106          */
 107         // FIXME: we should support async startup, which will require a STARTING state.
 108         STARTED,
 109         /**
 110          * Local service is being stopped.
 111          */
 112         STOPPING,
 113     }
 114
 115     private static final Logger LOG = LoggerFactory.getLogger(ClusterSingletonServiceGroupImpl.class);
 116
 117     private final S entityOwnershipService;
 118     private final String identifier;
 119
 120     /* Entity instances */
 121     private final E serviceEntity;
 122     private final E cleanupEntity;
 123
 124     private final Set<ClusterSingletonServiceRegistration> members = ConcurrentHashMap.newKeySet();
 125     // Guarded by lock
 126     private final Map<ClusterSingletonServiceRegistration, ServiceInfo> services = new HashMap<>();
 127
 128     // Marker for when any state changed
 129     @SuppressWarnings("rawtypes")
 130     private static final AtomicIntegerFieldUpdater<ClusterSingletonServiceGroupImpl> DIRTY_UPDATER =
 131             AtomicIntegerFieldUpdater.newUpdater(ClusterSingletonServiceGroupImpl.class, "dirty");
 132     private volatile int dirty;
 133
 134     // Simplified lock: non-reentrant, support tryLock() only
 135     @SuppressWarnings("rawtypes")
 136     private static final AtomicIntegerFieldUpdater<ClusterSingletonServiceGroupImpl> LOCK_UPDATER =
 137             AtomicIntegerFieldUpdater.newUpdater(ClusterSingletonServiceGroupImpl.class, "lock");
 138     @SuppressWarnings("unused")
 139     private volatile int lock;
 140
 141     /*
 142      * State tracking is quite involved, as we are tracking up to four asynchronous sources of events:
 143      * - user calling close()
 144      * - service entity ownership
 145      * - cleanup entity ownership
 146      * - service shutdown future
 147      *
 148      * Absolutely correct solution would be a set of behaviors, which govern each state, remembering where we want to
 149      * get to and what we are doing. That would result in ~15 classes which would quickly render this code unreadable
 150      * due to boilerplate overhead.
 151      *
 152      * We therefore take a different approach, tracking state directly in this class and evaluate state transitions
 153      * based on recorded bits -- without explicit representation of state machine state.
 154      */
 155     /**
 156      * Group close future. In can only go from null to non-null reference. Whenever it is non-null, it indicates that
 157      * the user has closed the group and we are converging to termination.
 158      */
 159     // We are using volatile get-and-set to support non-blocking close(). It may be more efficient to inline it here,
 160     // as we perform a volatile read after unlocking -- that volatile read may easier on L1 cache.
 161     // XXX: above needs a microbenchmark contention ever becomes a problem.
 162     private final AtomicReference<SettableFuture<Void>> closeFuture = new AtomicReference<>();
 163
 164     /**
 165      * Service (base) entity registration. This entity selects an owner candidate across nodes. Candidates proceed to
 166      * acquire {@link #cleanupEntity}.
 167      */
 168     @GuardedBy("this")
 169     private GenericEntityOwnershipCandidateRegistration<P, E> serviceEntityReg = null;
 170     /**
 171      * Service (base) entity last reported state.
 172      */
 173     @GuardedBy("this")
 174     private EntityState serviceEntityState = EntityState.UNREGISTERED;
 175
 176     /**
 177      * Cleanup (owner) entity registration. This entity guards access to service state and coordinates shutdown cleanup
 178      * and startup.
 179      */
 180     @GuardedBy("this")
 181     private GenericEntityOwnershipCandidateRegistration<P, E> cleanupEntityReg;
 182     /**
 183      * Cleanup (owner) entity last reported state.
 184      */
 185     @GuardedBy("this")
 186     private EntityState cleanupEntityState = EntityState.UNREGISTERED;
 187
 188     private volatile boolean initialized;
 189
 190     /**
 191      * Class constructor. Note: last argument is reused as-is.
 192      *
 193      * @param identifier non-empty string as identifier
 194      * @param mainEntity as Entity instance
 195      * @param closeEntity as Entity instance
 196      * @param entityOwnershipService GenericEntityOwnershipService instance
 197      * @param parent parent service
 198      * @param services Services list
 199      */
 200     ClusterSingletonServiceGroupImpl(final String identifier, final S entityOwnershipService, final E mainEntity,
 201             final E closeEntity, final Collection<ClusterSingletonServiceRegistration> services) {
 202         checkArgument(!identifier.isEmpty(), "Identifier may not be empty");
 203         this.identifier = identifier;
 204         this.entityOwnershipService = requireNonNull(entityOwnershipService);
 205         this.serviceEntity = requireNonNull(mainEntity);
 206         this.cleanupEntity = requireNonNull(closeEntity);
 207         members.addAll(services);
 208
 209         LOG.debug("Instantiated new service group for {}", identifier);
 210     }
 211
 212     @VisibleForTesting
 213     ClusterSingletonServiceGroupImpl(final String identifier, final E mainEntity,
 214             final E closeEntity, final S entityOwnershipService) {
 215         this(identifier, entityOwnershipService, mainEntity, closeEntity, ImmutableList.of());
 216     }
 217
 218     @Override
 219     public String getIdentifier() {
 220         return identifier;
 221     }
 222
 223     @Override
 224     ListenableFuture<?> closeClusterSingletonGroup() {
 225         final ListenableFuture<?> ret = destroyGroup();
 226         members.clear();
 227         markDirty();
 228
 229         if (tryLock()) {
 230             reconcileState();
 231         } else {
 232             LOG.debug("Service group {} postponing sync on close", identifier);
 233         }
 234
 235         return ret;
 236     }
 237
 238     private boolean isClosed() {
 239         return closeFuture.get() != null;
 240     }
 241
 242     @Override
 243     void initialize() throws CandidateAlreadyRegisteredException {
 244         verify(tryLock());
 245         try {
 246             checkState(!initialized, "Singleton group %s was already initilized", identifier);
 247             LOG.debug("Initializing service group {} with services {}", identifier, members);
 248             synchronized (this) {
 249                 serviceEntityState = EntityState.REGISTERED;
 250                 serviceEntityReg = entityOwnershipService.registerCandidate(serviceEntity);
 251                 initialized = true;
 252             }
 253         } finally {
 254             unlock();
 255         }
 256     }
 257
 258     private void checkNotClosed() {
 259         checkState(!isClosed(), "Service group %s has already been closed", identifier);
 260     }
 261
 262     @Override
 263     void registerService(final ClusterSingletonServiceRegistration reg) {
 264         final ClusterSingletonService service = reg.getInstance();
 265         verify(identifier.equals(service.getIdentifier().getValue()));
 266         checkNotClosed();
 267
 268         checkState(initialized, "Service group %s is not initialized yet", identifier);
 269
 270         // First put the service
 271         LOG.debug("Adding service {} to service group {}", service, identifier);
 272         verify(members.add(reg));
 273         markDirty();
 274
 275         if (!tryLock()) {
 276             LOG.debug("Service group {} delayed register of {}", identifier, reg);
 277             return;
 278         }
 279
 280         reconcileState();
 281     }
 282
 283     @CheckReturnValue
 284     @Override
 285     ListenableFuture<?> unregisterService(final ClusterSingletonServiceRegistration reg) {
 286         final ClusterSingletonService service = reg.getInstance();
 287         verify(identifier.equals(service.getIdentifier().getValue()));
 288         checkNotClosed();
 289
 290         verify(members.remove(reg));
 291         markDirty();
 292         if (members.isEmpty()) {
 293             // We need to let AbstractClusterSingletonServiceProviderImpl know this group is to be shutdown
 294             // before we start applying state, because while we do not re-enter, the user is free to do whatever,
 295             // notably including registering a service with the same ID from the service shutdown hook. That
 296             // registration request needs to hit the successor of this group.
 297             return destroyGroup();
 298         }
 299
 300         if (tryLock()) {
 301             reconcileState();
 302         } else {
 303             LOG.debug("Service group {} delayed unregister of {}", identifier, reg);
 304         }
 305         return null;
 306     }
 307
 308     private synchronized @NonNull ListenableFuture<?> destroyGroup() {
 309         final SettableFuture<Void> future = SettableFuture.create();
 310         if (!closeFuture.compareAndSet(null, future)) {
 311             return verifyNotNull(closeFuture.get());
 312         }
 313
 314         if (serviceEntityReg != null) {
 315             // We are still holding the service registration, close it now...
 316             LOG.debug("Service group {} unregistering service entity {}", identifier, serviceEntity);
 317             serviceEntityReg.close();
 318             serviceEntityReg = null;
 319         }
 320
 321         markDirty();
 322         return future;
 323     }
 324
 325     @Override
 326     void ownershipChanged(final C ownershipChange) {
 327         LOG.debug("Ownership change {} for ClusterSingletonServiceGroup {}", ownershipChange, identifier);
 328
 329         synchronized (this) {
 330             lockedOwnershipChanged(ownershipChange);
 331         }
 332
 333         if (isDirty()) {
 334             if (!tryLock()) {
 335                 LOG.debug("Service group {} postponing ownership change sync");
 336                 return;
 337             }
 338
 339             reconcileState();
 340         }
 341     }
 342
 343     /**
 344      * Handle an ownership change with the lock held. Callers are expected to handle termination conditions, this method
 345      * and anything it calls must not call {@link #lockedClose(SettableFuture)}.
 346      *
 347      * @param ownershipChange reported change
 348      */
 349     @GuardedBy("this")
 350     private void lockedOwnershipChanged(final C ownershipChange) {
 351         final E entity = ownershipChange.getEntity();
 352         if (serviceEntity.equals(entity)) {
 353             serviceOwnershipChanged(ownershipChange.getState(), ownershipChange.inJeopardy());
 354             markDirty();
 355         } else if (cleanupEntity.equals(entity)) {
 356             cleanupCandidateOwnershipChanged(ownershipChange.getState(), ownershipChange.inJeopardy());
 357             markDirty();
 358         } else {
 359             LOG.warn("Group {} received unrecognized change {}", identifier, ownershipChange);
 360         }
 361     }
 362
 363     @GuardedBy("this")
 364     private void cleanupCandidateOwnershipChanged(final EntityOwnershipChangeState state, final boolean jeopardy) {
 365         if (jeopardy) {
 366             switch (state) {
 367                 case LOCAL_OWNERSHIP_GRANTED:
 368                 case LOCAL_OWNERSHIP_RETAINED_WITH_NO_CHANGE:
 369                     LOG.warn("Service group {} cleanup entity owned without certainty", identifier);
 370                     cleanupEntityState = EntityState.OWNED_JEOPARDY;
 371                     break;
 372                 case LOCAL_OWNERSHIP_LOST_NEW_OWNER:
 373                 case LOCAL_OWNERSHIP_LOST_NO_OWNER:
 374                 case REMOTE_OWNERSHIP_CHANGED:
 375                 case REMOTE_OWNERSHIP_LOST_NO_OWNER:
 376                     LOG.info("Service group {} cleanup entity ownership uncertain", identifier);
 377                     cleanupEntityState = EntityState.UNOWNED;
 378                     break;
 379                 default:
 380                     throw new IllegalStateException("Unhandled cleanup entity jeopardy change " + state);
 381             }
 382
 383             return;
 384         }
 385
 386         if (cleanupEntityState == EntityState.OWNED_JEOPARDY) {
 387             // Pair info message with previous jeopardy
 388             LOG.info("Service group {} cleanup entity ownership ascertained", identifier);
 389         }
 390
 391         switch (state) {
 392             case LOCAL_OWNERSHIP_GRANTED:
 393             case LOCAL_OWNERSHIP_RETAINED_WITH_NO_CHANGE:
 394                 cleanupEntityState = EntityState.OWNED;
 395                 break;
 396             case LOCAL_OWNERSHIP_LOST_NEW_OWNER:
 397             case LOCAL_OWNERSHIP_LOST_NO_OWNER:
 398             case REMOTE_OWNERSHIP_LOST_NO_OWNER:
 399             case REMOTE_OWNERSHIP_CHANGED:
 400                 cleanupEntityState = EntityState.UNOWNED;
 401                 break;
 402             default:
 403                 LOG.warn("Service group {} ignoring unhandled cleanup entity change {}", identifier, state);
 404                 break;
 405         }
 406     }
 407
 408     @GuardedBy("this")
 409     private void serviceOwnershipChanged(final EntityOwnershipChangeState state, final boolean jeopardy) {
 410         if (jeopardy) {
 411             LOG.info("Service group {} service entity ownership uncertain", identifier);
 412             switch (state) {
 413                 case LOCAL_OWNERSHIP_GRANTED:
 414                 case LOCAL_OWNERSHIP_RETAINED_WITH_NO_CHANGE:
 415                     serviceEntityState = EntityState.OWNED_JEOPARDY;
 416                     break;
 417                 case LOCAL_OWNERSHIP_LOST_NEW_OWNER:
 418                 case LOCAL_OWNERSHIP_LOST_NO_OWNER:
 419                 case REMOTE_OWNERSHIP_CHANGED:
 420                 case REMOTE_OWNERSHIP_LOST_NO_OWNER:
 421                     serviceEntityState = EntityState.UNOWNED;
 422                     break;
 423                 default:
 424                     throw new IllegalStateException("Unhandled cleanup entity jeopardy change " + state);
 425             }
 426             return;
 427         }
 428
 429         if (serviceEntityState == EntityState.OWNED_JEOPARDY) {
 430             // Pair info message with previous jeopardy
 431             LOG.info("Service group {} service entity ownership ascertained", identifier);
 432         }
 433
 434         switch (state) {
 435             case LOCAL_OWNERSHIP_GRANTED:
 436             case LOCAL_OWNERSHIP_RETAINED_WITH_NO_CHANGE:
 437                 LOG.debug("Service group {} acquired service entity ownership", identifier);
 438                 serviceEntityState = EntityState.OWNED;
 439                 break;
 440             case LOCAL_OWNERSHIP_LOST_NEW_OWNER:
 441             case LOCAL_OWNERSHIP_LOST_NO_OWNER:
 442             case REMOTE_OWNERSHIP_CHANGED:
 443             case REMOTE_OWNERSHIP_LOST_NO_OWNER:
 444                 LOG.debug("Service group {} lost service entity ownership", identifier);
 445                 serviceEntityState = EntityState.UNOWNED;
 446                 break;
 447             default:
 448                 LOG.warn("Service group {} ignoring unhandled cleanup entity change {}", identifier, state);
 449         }
 450     }
 451
 452     // has to be called with lock asserted, which will be released prior to returning
 453     private void reconcileState() {
 454         // Always check if there is any state change to be applied.
 455         while (true) {
 456             try {
 457                 if (conditionalClean()) {
 458                     tryReconcileState();
 459                 }
 460             } finally {
 461                 // We may have ran a round of reconciliation, but the either one of may have happened asynchronously:
 462                 // - registration
 463                 // - unregistration
 464                 // - service future completed
 465                 // - entity state changed
 466                 //
 467                 // We are dropping the lock, but we need to recheck dirty and try to apply state again if it is found to
 468                 // be dirty again. This closes the following race condition:
 469                 //
 470                 // A: runs these checks holding the lock
 471                 // B: modifies them, fails to acquire lock
 472                 // A: releases lock -> noone takes care of reconciliation
 473
 474                 unlock();
 475             }
 476
 477             if (isDirty()) {
 478                 if (tryLock()) {
 479                     LOG.debug("Service group {} re-running reconciliation", identifier);
 480                     continue;
 481                 }
 482
 483                 LOG.debug("Service group {} will be reconciled by someone else", identifier);
 484             } else {
 485                 LOG.debug("Service group {} is completely reconciled", identifier);
 486             }
 487
 488             break;
 489         }
 490     }
 491
 492     private void serviceTransitionCompleted() {
 493         markDirty();
 494         if (tryLock()) {
 495             reconcileState();
 496         }
 497     }
 498
 499     // Has to be called with lock asserted
 500     private void tryReconcileState() {
 501         // First take a safe snapshot of current state on which we will base our decisions.
 502         final Set<ClusterSingletonServiceRegistration> localMembers;
 503         final boolean haveCleanup;
 504         final boolean haveService;
 505         synchronized (this) {
 506             if (serviceEntityReg != null) {
 507                 switch (serviceEntityState) {
 508                     case OWNED:
 509                     case OWNED_JEOPARDY:
 510                         haveService = true;
 511                         break;
 512                     case REGISTERED:
 513                     case UNOWNED:
 514                     case UNREGISTERED:
 515                         haveService = false;
 516                         break;
 517                     default:
 518                         throw new IllegalStateException("Unhandled service entity state " + serviceEntityState);
 519                 }
 520             } else {
 521                 haveService = false;
 522             }
 523
 524             if (haveService && cleanupEntityReg == null) {
 525                 // We have the service entity but have not registered for cleanup entity. Do that now and retry.
 526                 LOG.debug("Service group {} registering cleanup entity", identifier);
 527                 try {
 528                     cleanupEntityState = EntityState.REGISTERED;
 529                     cleanupEntityReg = entityOwnershipService.registerCandidate(cleanupEntity);
 530                 } catch (CandidateAlreadyRegisteredException e) {
 531                     LOG.error("Service group {} failed to take ownership, aborting", identifier, e);
 532                     if (serviceEntityReg != null) {
 533                         serviceEntityReg.close();
 534                         serviceEntityReg = null;
 535                     }
 536                 }
 537                 markDirty();
 538                 return;
 539             }
 540
 541             if (cleanupEntityReg != null) {
 542                 switch (cleanupEntityState) {
 543                     case OWNED:
 544                         haveCleanup = true;
 545                         break;
 546                     case OWNED_JEOPARDY:
 547                     case REGISTERED:
 548                     case UNOWNED:
 549                     case UNREGISTERED:
 550                         haveCleanup = false;
 551                         break;
 552                     default:
 553                         throw new IllegalStateException("Unhandled service entity state " + serviceEntityState);
 554                 }
 555             } else {
 556                 haveCleanup = false;
 557             }
 558
 559             localMembers = ImmutableSet.copyOf(members);
 560         }
 561
 562         if (haveService && haveCleanup) {
 563             ensureServicesStarting(localMembers);
 564             return;
 565         }
 566
 567         ensureServicesStopping();
 568
 569         if (!haveService && services.isEmpty()) {
 570             LOG.debug("Service group {} has no running services", identifier);
 571             final boolean canFinishClose;
 572             synchronized (this) {
 573                 if (cleanupEntityReg != null) {
 574                     LOG.debug("Service group {} releasing cleanup entity", identifier);
 575                     cleanupEntityReg.close();
 576                     cleanupEntityReg = null;
 577                 }
 578
 579                 switch (cleanupEntityState) {
 580                     case OWNED:
 581                     case OWNED_JEOPARDY:
 582                     case REGISTERED:
 583                         // When we are registered we need to wait for registration to resolve, otherwise
 584                         // the notification could be routed to the next incarnation of this group -- which could be
 585                         // confused by the fact it is not registered, but receives, for example, OWNED notification.
 586                         canFinishClose = false;
 587                         break;
 588                     case UNOWNED:
 589                     case UNREGISTERED:
 590                         canFinishClose = true;
 591                         break;
 592                     default:
 593                         throw new IllegalStateException("Unhandled cleanup entity state " + cleanupEntityState);
 594                 }
 595             }
 596
 597             if (canFinishClose) {
 598                 final SettableFuture<Void> localFuture = closeFuture.get();
 599                 if (localFuture != null && !localFuture.isDone()) {
 600                     LOG.debug("Service group {} completing termination", identifier);
 601                     localFuture.set(null);
 602                 }
 603             }
 604         }
 605     }
 606
 607     // Has to be called with lock asserted
 608     @SuppressWarnings("illegalCatch")
 609     private void ensureServicesStarting(final Set<ClusterSingletonServiceRegistration> localConfig) {
 610         LOG.debug("Service group {} starting services", identifier);
 611
 612         // This may look counter-intuitive, but the localConfig may be missing some services that are started -- for
 613         // example when this method is executed as part of unregisterService() call. In that case we need to ensure
 614         // services in the list are stopping
 615         final Iterator<Entry<ClusterSingletonServiceRegistration, ServiceInfo>> it = services.entrySet().iterator();
 616         while (it.hasNext()) {
 617             final Entry<ClusterSingletonServiceRegistration, ServiceInfo> entry = it.next();
 618             final ClusterSingletonServiceRegistration reg = entry.getKey();
 619             if (!localConfig.contains(reg)) {
 620                 final ServiceInfo newInfo = ensureStopping(reg, entry.getValue());
 621                 if (newInfo != null) {
 622                     entry.setValue(newInfo);
 623                 } else {
 624                     it.remove();
 625                 }
 626             }
 627         }
 628
 629         // Now make sure member services are being juggled around
 630         for (ClusterSingletonServiceRegistration reg : localConfig) {
 631             if (!services.containsKey(reg)) {
 632                 final ClusterSingletonService service = reg.getInstance();
 633                 LOG.debug("Starting service {}", service);
 634
 635                 try {
 636                     service.instantiateServiceInstance();
 637                 } catch (Exception e) {
 638                     LOG.warn("Service group {} service {} failed to start, attempting to continue", identifier, service,
 639                         e);
 640                     continue;
 641                 }
 642
 643                 services.put(reg, ServiceInfo.started());
 644             }
 645         }
 646     }
 647
 648     // Has to be called with lock asserted
 649     private void ensureServicesStopping() {
 650         final Iterator<Entry<ClusterSingletonServiceRegistration, ServiceInfo>> it = services.entrySet().iterator();
 651         while (it.hasNext()) {
 652             final Entry<ClusterSingletonServiceRegistration, ServiceInfo> entry = it.next();
 653             final ServiceInfo newInfo = ensureStopping(entry.getKey(), entry.getValue());
 654             if (newInfo != null) {
 655                 entry.setValue(newInfo);
 656             } else {
 657                 it.remove();
 658             }
 659         }
 660     }
 661
 662     @SuppressWarnings("illegalCatch")
 663     private ServiceInfo ensureStopping(final ClusterSingletonServiceRegistration reg, final ServiceInfo info) {
 664         switch (info.getState()) {
 665             case STARTED:
 666                 final ClusterSingletonService service = reg.getInstance();
 667
 668                 LOG.debug("Service group {} stopping service {}", identifier, service);
 669                 final @NonNull ListenableFuture<?> future;
 670                 try {
 671                     future = verifyNotNull(service.closeServiceInstance());
 672                 } catch (Exception e) {
 673                     LOG.warn("Service group {} service {} failed to stop, attempting to continue", identifier, service,
 674                         e);
 675                     return null;
 676                 }
 677
 678                 Futures.addCallback(future, new FutureCallback<Object>() {
 679                     @Override
 680                     public void onSuccess(final Object result) {
 681                         LOG.debug("Service group {} service {} stopped successfully", identifier, service);
 682                         serviceTransitionCompleted();
 683                     }
 684
 685                     @Override
 686                     public void onFailure(final Throwable cause) {
 687                         LOG.debug("Service group {} service {} stopped with error", identifier, service, cause);
 688                         serviceTransitionCompleted();
 689                     }
 690                 }, MoreExecutors.directExecutor());
 691                 return info.toState(ServiceState.STOPPING, future);
 692             case STOPPING:
 693                 if (info.getFuture().isDone()) {
 694                     LOG.debug("Service group {} removed stopped service {}", identifier, reg.getInstance());
 695                     return null;
 696                 }
 697                 return info;
 698             default:
 699                 throw new IllegalStateException("Unhandled state " + info.getState());
 700         }
 701     }
 702
 703     private void markDirty() {
 704         dirty = 1;
 705     }
 706
 707     private boolean isDirty() {
 708         return dirty != 0;
 709     }
 710
 711     private boolean conditionalClean() {
 712         return DIRTY_UPDATER.compareAndSet(this, 1, 0);
 713     }
 714
 715     private boolean tryLock() {
 716         return LOCK_UPDATER.compareAndSet(this, 0, 1);
 717     }
 718
 719     private boolean unlock() {
 720         verify(LOCK_UPDATER.compareAndSet(this, 1, 0));
 721         return true;
 722     }
 723
 724     @Override
 725     public String toString() {
 726         return MoreObjects.toStringHelper(this).add("identifier", identifier).toString();
 727     }
 728 }