singleton-service/mdsal-singleton-dom-impl/src/main/java/org/opendaylight/mdsal/singleton/dom/impl/ClusterSingletonServiceGroupImpl.java

   1 /*
   2  * Copyright (c) 2016 Cisco Systems, Inc. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.mdsal.singleton.dom.impl;
  10
  11 import com.google.common.annotations.VisibleForTesting;
  12 import com.google.common.base.MoreObjects;
  13 import com.google.common.base.Preconditions;
  14 import com.google.common.base.Verify;
  15 import com.google.common.util.concurrent.FutureCallback;
  16 import com.google.common.util.concurrent.Futures;
  17 import com.google.common.util.concurrent.ListenableFuture;
  18 import com.google.common.util.concurrent.MoreExecutors;
  19 import com.google.common.util.concurrent.SettableFuture;
  20 import java.util.ArrayList;
  21 import java.util.List;
  22 import java.util.concurrent.atomic.AtomicReference;
  23 import java.util.concurrent.locks.ReentrantLock;
  24 import javax.annotation.CheckReturnValue;
  25 import javax.annotation.concurrent.GuardedBy;
  26 import javax.annotation.concurrent.ThreadSafe;
  27 import org.opendaylight.mdsal.eos.common.api.CandidateAlreadyRegisteredException;
  28 import org.opendaylight.mdsal.eos.common.api.EntityOwnershipChangeState;
  29 import org.opendaylight.mdsal.eos.common.api.GenericEntity;
  30 import org.opendaylight.mdsal.eos.common.api.GenericEntityOwnershipCandidateRegistration;
  31 import org.opendaylight.mdsal.eos.common.api.GenericEntityOwnershipChange;
  32 import org.opendaylight.mdsal.eos.common.api.GenericEntityOwnershipListener;
  33 import org.opendaylight.mdsal.eos.common.api.GenericEntityOwnershipService;
  34 import org.opendaylight.mdsal.singleton.common.api.ClusterSingletonService;
  35 import org.opendaylight.yangtools.concepts.Path;
  36 import org.slf4j.Logger;
  37 import org.slf4j.LoggerFactory;
  38
  39 /**
  40  * Implementation of {@link ClusterSingletonServiceGroup} on top of the Entitiy Ownership Service. Since EOS is atomic
  41  * in its operation and singleton services incur startup and most notably cleanup, we need to do something smart here.
  42  *
  43  * <p>
  44  * The implementation takes advantage of the fact that EOS provides stable ownership, i.e. owners are not moved as
  45  * a result on new candidates appearing. We use two entities:
  46  * - service entity, to which all nodes register
  47  * - cleanup entity, which only the service entity owner registers to
  48  *
  49  * <p>
  50  * Once the cleanup entity ownership is acquired, services are started. As long as the cleanup entity is registered,
  51  * it should remain the owner. In case a new service owner emerges, the old owner will start the cleanup process,
  52  * eventually releasing the cleanup entity. The new owner registers for the cleanup entity -- but will not see it
  53  * granted until the old owner finishes the cleanup.
  54  *
  55  * @param <P> the instance identifier path type
  56  * @param <E> the GenericEntity type
  57  * @param <C> the GenericEntityOwnershipChange type
  58  * @param <G> the GenericEntityOwnershipListener type
  59  * @param <S> the GenericEntityOwnershipService type
  60  */
  61 @ThreadSafe
  62 final class ClusterSingletonServiceGroupImpl<P extends Path<P>, E extends GenericEntity<P>,
  63         C extends GenericEntityOwnershipChange<P, E>,  G extends GenericEntityOwnershipListener<P, C>,
  64         S extends GenericEntityOwnershipService<P, E, G>> extends ClusterSingletonServiceGroup<P, E, C> {
  65
  66     private enum EntityState {
  67         /**
  68          * This entity was never registered.
  69          */
  70         UNREGISTERED,
  71         /**
  72          * Registration exists, but we are waiting for it to resolve.
  73          */
  74         REGISTERED,
  75         /**
  76          * Registration indicated we are the owner.
  77          */
  78         OWNED,
  79         /**
  80          * Registration indicated we are the owner, but global state is uncertain -- meaning there can be owners in
  81          * another partition, for example.
  82          */
  83         OWNED_JEOPARDY,
  84         /**
  85          * Registration indicated we are not the owner. In this state we do not care about global state, therefore we
  86          * do not need an UNOWNED_JEOPARDY state.
  87          */
  88         UNOWNED,
  89     }
  90
  91     private enum ServiceState {
  92         /**
  93          * Local services are stopped.
  94          */
  95         STOPPED,
  96         /**
  97          * Local services are up and running.
  98          */
  99         // FIXME: we should support async startup, which will require a STARTING state.
 100         STARTED,
 101         /**
 102          * Local services are being stopped.
 103          */
 104         STOPPING,
 105     }
 106
 107     private static final Logger LOG = LoggerFactory.getLogger(ClusterSingletonServiceGroupImpl.class);
 108
 109     private final S entityOwnershipService;
 110     private final String identifier;
 111
 112     /* Entity instances */
 113     private final E serviceEntity;
 114     private final E cleanupEntity;
 115
 116     private final ReentrantLock lock = new ReentrantLock(true);
 117
 118     @GuardedBy("lock")
 119     private final List<ClusterSingletonService> serviceGroup;
 120
 121     /*
 122      * State tracking is quite involved, as we are tracking up to four asynchronous sources of events:
 123      * - user calling close()
 124      * - service entity ownership
 125      * - cleanup entity ownership
 126      * - service shutdown future
 127      *
 128      * Absolutely correct solution would be a set of behaviors, which govern each state, remembering where we want to
 129      * get to and what we are doing. That would result in ~15 classes which would quickly render this code unreadable
 130      * due to boilerplate overhead.
 131      *
 132      * We therefore take a different approach, tracking state directly in this class and evaluate state transitions
 133      * based on recorded bits -- without explicit representation of state machine state.
 134      */
 135     /**
 136      * Group close future. In can only go from null to non-null reference. Whenever it is non-null, it indicates that
 137      * the user has closed the group and we are converging to termination.
 138      */
 139     // We are using volatile get-and-set to support non-blocking close(). It may be more efficient to inline it here,
 140     // as we perform a volatile read after unlocking -- that volatile read may easier on L1 cache.
 141     // XXX: above needs a microbenchmark contention ever becomes a problem.
 142     private final AtomicReference<SettableFuture<Void>> closeFuture = new AtomicReference<>();
 143
 144     /**
 145      * Service (base) entity registration. This entity selects an owner candidate across nodes. Candidates proceed to
 146      * acquire {@link #cleanupEntity}.
 147      */
 148     @GuardedBy("lock")
 149     private GenericEntityOwnershipCandidateRegistration<P, E> serviceEntityReg = null;
 150     /**
 151      * Service (base) entity last reported state.
 152      */
 153     @GuardedBy("lock")
 154     private EntityState serviceEntityState = EntityState.UNREGISTERED;
 155
 156     /**
 157      * Cleanup (owner) entity registration. This entity guards access to service state and coordinates shutdown cleanup
 158      * and startup.
 159      */
 160     @GuardedBy("lock")
 161     private GenericEntityOwnershipCandidateRegistration<P, E> cleanupEntityReg;
 162     /**
 163      * Cleanup (owner) entity last reported state.
 164      */
 165     @GuardedBy("lock")
 166     private EntityState cleanupEntityState = EntityState.UNREGISTERED;
 167
 168     /**
 169      * Optional event capture list. This field is initialized when we interact with entity ownership service, to capture
 170      * events reported during EOS method invocation -- like immediate acquisition of entity when we register it. This
 171      * prevents bugs from recursion.
 172      */
 173     @GuardedBy("lock")
 174     private List<C> capture = null;
 175
 176     /**
 177      * State of local services.
 178      */
 179     @GuardedBy("lock")
 180     private ServiceState localServicesState = ServiceState.STOPPED;
 181
 182     /**
 183      * Class constructor. Note: last argument is reused as-is.
 184      *
 185      * @param identifier non-empty string as identifier
 186      * @param mainEntity as Entity instance
 187      * @param closeEntity as Entity instance
 188      * @param entityOwnershipService GenericEntityOwnershipService instance
 189      * @param parent parent service
 190      * @param services Services list
 191      */
 192     ClusterSingletonServiceGroupImpl(final String identifier, final S entityOwnershipService, final E mainEntity,
 193             final E closeEntity, final List<ClusterSingletonService> services) {
 194         Preconditions.checkArgument(!identifier.isEmpty(), "Identifier may not be empty");
 195         this.identifier = identifier;
 196         this.entityOwnershipService = Preconditions.checkNotNull(entityOwnershipService);
 197         this.serviceEntity = Preconditions.checkNotNull(mainEntity);
 198         this.cleanupEntity = Preconditions.checkNotNull(closeEntity);
 199         this.serviceGroup = Preconditions.checkNotNull(services);
 200         LOG.debug("Instantiated new service group for {}", identifier);
 201     }
 202
 203     @VisibleForTesting
 204     ClusterSingletonServiceGroupImpl(final String identifier, final E mainEntity,
 205             final E closeEntity, final S entityOwnershipService) {
 206         this(identifier, entityOwnershipService, mainEntity, closeEntity, new ArrayList<>(1));
 207     }
 208
 209     @Override
 210     public String getIdentifier() {
 211         return identifier;
 212     }
 213
 214     @Override
 215     ListenableFuture<?> closeClusterSingletonGroup() {
 216         // Assert our future first
 217         final SettableFuture<Void> future = SettableFuture.create();
 218         final SettableFuture<Void> existing = closeFuture.getAndSet(future);
 219         if (existing != null) {
 220             return existing;
 221         }
 222
 223         if (!lock.tryLock()) {
 224             // The lock is held, the cleanup will be finished by the owner thread
 225             LOG.debug("Singleton group {} cleanup postponed", identifier);
 226             return future;
 227         }
 228
 229         try {
 230             lockedClose(future);
 231         } finally {
 232             lock.unlock();
 233         }
 234
 235         LOG.debug("Service group {} {}", identifier, future.isDone() ? "closed" : "closing");
 236         return future;
 237     }
 238
 239     private boolean isClosed() {
 240         return closeFuture.get() != null;
 241     }
 242
 243     @GuardedBy("lock")
 244     private void startCapture() {
 245         Verify.verify(capture == null, "Service group {} is already capturing events {}", identifier, capture);
 246         capture = new ArrayList<>(0);
 247         LOG.debug("Service group {} started capturing events", identifier);
 248     }
 249
 250     private List<C> endCapture() {
 251         final List<C> ret = Verify.verifyNotNull(capture, "Service group {} is not currently capturing", identifier);
 252         capture = null;
 253         LOG.debug("Service group {} finished capturing events, {} events to process", identifier, ret.size());
 254         return ret;
 255     }
 256
 257     @GuardedBy("lock")
 258     private void lockedClose(final SettableFuture<Void> future) {
 259         if (serviceEntityReg != null) {
 260             // We are still holding the service registration, close it now...
 261             LOG.debug("Service group {} unregistering service entity {}", identifier, serviceEntity);
 262             startCapture();
 263             serviceEntityReg.close();
 264             serviceEntityReg = null;
 265
 266             // This can potentially mutate our state, so all previous checks need to be re-validated.
 267             endCapture().forEach(this::lockedOwnershipChanged);
 268         }
 269
 270         // Now check service entity state: if it is still owned, we need to wait until it is acknowledged as
 271         // unregistered.
 272         switch (serviceEntityState) {
 273             case REGISTERED:
 274             case UNOWNED:
 275             case UNREGISTERED:
 276                 // We have either successfully shut down, or have never started up, proceed with termination
 277                 break;
 278             case OWNED:
 279                 // We have unregistered, but EOS has not reported our loss of ownership. We will continue with shutdown
 280                 // when that loss is reported.
 281                 LOG.debug("Service group {} is still owned, postponing termination", identifier);
 282                 return;
 283             case OWNED_JEOPARDY:
 284                 // This is a significant event, as it relates to cluster split/join operations, operators need to know
 285                 // we are waiting for cluster join event.
 286                 LOG.info("Service group {} is still owned with split cluster, postponing termination", identifier);
 287                 return;
 288             default:
 289                 throw new IllegalStateException("Unhandled service entity state " + serviceEntityState);
 290         }
 291
 292         // We do not own service entity state: we need to ensure services are stopped.
 293         if (stopServices()) {
 294             LOG.debug("Service group {} started shutting down services, postponing termination", identifier);
 295             return;
 296         }
 297
 298         // Local cleanup completed, release cleanup entity if needed
 299         if (cleanupEntityReg != null) {
 300             LOG.debug("Service group {} unregistering cleanup entity {}", identifier, cleanupEntity);
 301             startCapture();
 302             cleanupEntityReg.close();
 303             cleanupEntityReg = null;
 304
 305             // This can potentially mutate our state, so all previous checks need to be re-validated.
 306             endCapture().forEach(this::lockedOwnershipChanged);
 307         }
 308
 309         switch (cleanupEntityState) {
 310             case REGISTERED:
 311             case UNOWNED:
 312             case UNREGISTERED:
 313                 // We have either successfully shut down, or have never started up, proceed with termination
 314                 break;
 315             case OWNED:
 316                 // We have unregistered, but EOS has not reported our loss of ownership. We will continue with shutdown
 317                 // when that loss is reported.
 318                 LOG.debug("Service group {} is still owns cleanup, postponing termination", identifier);
 319                 return;
 320             case OWNED_JEOPARDY:
 321                 // This is a significant event, as it relates to cluster split/join operations, operators need to know
 322                 // we are waiting for cluster join event.
 323                 LOG.info("Service group {} is still owns cleanup with split cluster, postponing termination",
 324                     identifier);
 325                 return;
 326             default:
 327                 throw new IllegalStateException("Unhandled cleanup entity state " + serviceEntityState);
 328         }
 329
 330         // No registrations left and no service operations pending, we are done
 331         LOG.debug("Service group {} completing termination", identifier);
 332         future.set(null);
 333     }
 334
 335     @Override
 336     void initialize() throws CandidateAlreadyRegisteredException {
 337         lock.lock();
 338         try {
 339             Preconditions.checkState(serviceEntityState == EntityState.UNREGISTERED,
 340                     "Singleton group %s was already initilized", identifier);
 341
 342             LOG.debug("Initializing service group {} with services {}", identifier, serviceGroup);
 343             startCapture();
 344             serviceEntityReg = entityOwnershipService.registerCandidate(serviceEntity);
 345             serviceEntityState = EntityState.REGISTERED;
 346             endCapture().forEach(this::lockedOwnershipChanged);
 347         } finally {
 348             lock.unlock();
 349         }
 350     }
 351
 352     private void checkNotClosed() {
 353         Preconditions.checkState(closeFuture.get() == null, "Service group %s has already been closed",
 354                 identifier);
 355     }
 356
 357     @Override
 358     void registerService(final ClusterSingletonService service) {
 359         Verify.verify(identifier.equals(service.getIdentifier().getValue()));
 360         checkNotClosed();
 361
 362         lock.lock();
 363         try {
 364             Preconditions.checkState(serviceEntityState != EntityState.UNREGISTERED,
 365                     "Service group %s is not initialized yet", identifier);
 366
 367             LOG.debug("Adding service {} to service group {}", service, identifier);
 368             serviceGroup.add(service);
 369
 370             switch (localServicesState) {
 371                 case STARTED:
 372                     LOG.debug("Service group {} starting late-registered service {}", identifier, service);
 373                     service.instantiateServiceInstance();
 374                     break;
 375                 case STOPPED:
 376                 case STOPPING:
 377                     break;
 378                 default:
 379                     throw new IllegalStateException("Unhandled local services state " + localServicesState);
 380             }
 381         } finally {
 382             lock.unlock();
 383             finishCloseIfNeeded();
 384         }
 385     }
 386
 387     @CheckReturnValue
 388     @Override
 389     boolean unregisterService(final ClusterSingletonService service) {
 390         Verify.verify(identifier.equals(service.getIdentifier().getValue()));
 391         checkNotClosed();
 392
 393         lock.lock();
 394         try {
 395             // There is a slight problem here, as the type does not match the list type, hence we need to tread
 396             // carefully.
 397             if (serviceGroup.size() == 1) {
 398                 Verify.verify(serviceGroup.contains(service));
 399                 return true;
 400             }
 401
 402             Verify.verify(serviceGroup.remove(service));
 403             LOG.debug("Service {} was removed from group.", service.getIdentifier().getValue());
 404
 405             switch (localServicesState) {
 406                 case STARTED:
 407                     LOG.warn("Service group {} stopping unregistered service {}", identifier, service);
 408                     service.closeServiceInstance();
 409                     break;
 410                 case STOPPED:
 411                 case STOPPING:
 412                     break;
 413                 default:
 414                     throw new IllegalStateException("Unhandled local services state " + localServicesState);
 415             }
 416
 417             return false;
 418         } finally {
 419             lock.unlock();
 420             finishCloseIfNeeded();
 421         }
 422     }
 423
 424     @Override
 425     void ownershipChanged(final C ownershipChange) {
 426         LOG.debug("Ownership change {} for ClusterSingletonServiceGroup {}", ownershipChange, identifier);
 427
 428         lock.lock();
 429         try {
 430             if (capture != null) {
 431                 capture.add(ownershipChange);
 432             } else {
 433                 lockedOwnershipChanged(ownershipChange);
 434             }
 435         } finally {
 436             lock.unlock();
 437             finishCloseIfNeeded();
 438         }
 439     }
 440
 441     /**
 442      * Handle an ownership change with the lock held. Callers are expected to handle termination conditions, this method
 443      * and anything it calls must not call {@link #lockedClose(SettableFuture)}.
 444      *
 445      * @param ownershipChange reported change
 446      */
 447     @GuardedBy("lock")
 448     private void lockedOwnershipChanged(final C ownershipChange) {
 449         final E entity = ownershipChange.getEntity();
 450         if (serviceEntity.equals(entity)) {
 451             serviceOwnershipChanged(ownershipChange.getState(), ownershipChange.inJeopardy());
 452         } else if (cleanupEntity.equals(entity)) {
 453             cleanupCandidateOwnershipChanged(ownershipChange.getState(), ownershipChange.inJeopardy());
 454         } else {
 455             LOG.warn("Group {} received unrecognized change {}", identifier, ownershipChange);
 456         }
 457     }
 458
 459     private void cleanupCandidateOwnershipChanged(final EntityOwnershipChangeState state, final boolean jeopardy) {
 460         if (jeopardy) {
 461             switch (state) {
 462                 case LOCAL_OWNERSHIP_GRANTED:
 463                 case LOCAL_OWNERSHIP_RETAINED_WITH_NO_CHANGE:
 464                     if (cleanupEntityReg == null) {
 465                         LOG.debug("Service group {} ignoring cleanup entity ownership when unregistered", identifier);
 466                         return;
 467                     }
 468
 469                     LOG.warn("Service group {} cleanup entity owned without certainty", identifier);
 470                     cleanupEntityState = EntityState.OWNED_JEOPARDY;
 471                     break;
 472                 case LOCAL_OWNERSHIP_LOST_NEW_OWNER:
 473                 case LOCAL_OWNERSHIP_LOST_NO_OWNER:
 474                 case REMOTE_OWNERSHIP_CHANGED:
 475                 case REMOTE_OWNERSHIP_LOST_NO_OWNER:
 476                     LOG.info("Service group {} cleanup entity ownership uncertain", identifier);
 477                     cleanupEntityState = EntityState.UNOWNED;
 478                     break;
 479                 default:
 480                     throw new IllegalStateException("Unhandled cleanup entity jeopardy change " + state);
 481             }
 482
 483             stopServices();
 484             return;
 485         }
 486
 487         if (cleanupEntityState == EntityState.OWNED_JEOPARDY) {
 488             // Pair info message with previous jeopardy
 489             LOG.info("Service group {} cleanup entity ownership ascertained", identifier);
 490         }
 491
 492         switch (state) {
 493             case LOCAL_OWNERSHIP_GRANTED:
 494             case LOCAL_OWNERSHIP_RETAINED_WITH_NO_CHANGE:
 495                 if (cleanupEntityReg == null) {
 496                     LOG.debug("Service group {} ignoring cleanup entity ownership when unregistered", identifier);
 497                     return;
 498                 }
 499
 500                 cleanupEntityState = EntityState.OWNED;
 501                 switch (localServicesState) {
 502                     case STARTED:
 503                         LOG.debug("Service group {} already has local services running", identifier);
 504                         break;
 505                     case STOPPED:
 506                         startServices();
 507                         break;
 508                     case STOPPING:
 509                         LOG.debug("Service group {} has local services stopping, postponing startup", identifier);
 510                         break;
 511                     default:
 512                         throw new IllegalStateException("Unhandled local services state " + localServicesState);
 513                 }
 514                 break;
 515             case LOCAL_OWNERSHIP_LOST_NEW_OWNER:
 516             case LOCAL_OWNERSHIP_LOST_NO_OWNER:
 517                 cleanupEntityState = EntityState.UNOWNED;
 518                 stopServices();
 519                 break;
 520             case REMOTE_OWNERSHIP_LOST_NO_OWNER:
 521             case REMOTE_OWNERSHIP_CHANGED:
 522                 cleanupEntityState = EntityState.UNOWNED;
 523                 break;
 524             default:
 525                 LOG.warn("Service group {} ignoring unhandled cleanup entity change {}", identifier, state);
 526                 break;
 527         }
 528     }
 529
 530     private void serviceOwnershipChanged(final EntityOwnershipChangeState state, final boolean jeopardy) {
 531         if (jeopardy) {
 532             LOG.info("Service group {} service entity ownership uncertain", identifier);
 533
 534             // Service entity ownership is uncertain, which means we want to record the state, but we do not want
 535             // to stop local services nor do anything with the cleanup entity.
 536             switch (state) {
 537                 case LOCAL_OWNERSHIP_GRANTED:
 538                 case LOCAL_OWNERSHIP_RETAINED_WITH_NO_CHANGE:
 539                     if (serviceEntityReg == null) {
 540                         LOG.debug("Service group {} ignoring service entity ownership when unregistered", identifier);
 541                         return;
 542                     }
 543
 544                     serviceEntityState = EntityState.OWNED_JEOPARDY;
 545                     break;
 546                 case LOCAL_OWNERSHIP_LOST_NEW_OWNER:
 547                 case LOCAL_OWNERSHIP_LOST_NO_OWNER:
 548                 case REMOTE_OWNERSHIP_CHANGED:
 549                 case REMOTE_OWNERSHIP_LOST_NO_OWNER:
 550                     serviceEntityState = EntityState.UNOWNED;
 551                     break;
 552                 default:
 553                     throw new IllegalStateException("Unhandled cleanup entity jeopardy change " + state);
 554             }
 555             return;
 556         }
 557
 558         if (serviceEntityState == EntityState.OWNED_JEOPARDY) {
 559             // Pair info message with previous jeopardy
 560             LOG.info("Service group {} service entity ownership ascertained", identifier);
 561         }
 562
 563         switch (state) {
 564             case LOCAL_OWNERSHIP_GRANTED:
 565             case LOCAL_OWNERSHIP_RETAINED_WITH_NO_CHANGE:
 566                 if (serviceEntityReg == null) {
 567                     LOG.debug("Service group {} ignoring service entity ownership when unregistered", identifier);
 568                     return;
 569                 }
 570
 571                 serviceEntityState = EntityState.OWNED;
 572                 takeOwnership();
 573                 break;
 574             case LOCAL_OWNERSHIP_LOST_NEW_OWNER:
 575             case LOCAL_OWNERSHIP_LOST_NO_OWNER:
 576                 LOG.debug("Service group {} lost service entity ownership", identifier);
 577                 serviceEntityState = EntityState.UNOWNED;
 578                 if (stopServices()) {
 579                     LOG.debug("Service group {} already stopping services, postponing cleanup", identifier);
 580                     return;
 581                 }
 582
 583                 if (cleanupEntityReg != null) {
 584                     cleanupEntityReg.close();
 585                     cleanupEntityReg = null;
 586                 }
 587                 break;
 588             case REMOTE_OWNERSHIP_CHANGED:
 589             case REMOTE_OWNERSHIP_LOST_NO_OWNER:
 590                 // No need to react, just update the state
 591                 serviceEntityState = EntityState.UNOWNED;
 592                 break;
 593             default:
 594                 LOG.warn("Service group {} ignoring unhandled cleanup entity change {}", identifier, state);
 595                 break;
 596         }
 597     }
 598
 599     private void finishCloseIfNeeded() {
 600         final SettableFuture<Void> future = closeFuture.get();
 601         if (future != null) {
 602             lock.lock();
 603             try {
 604                 lockedClose(future);
 605             } finally {
 606                 lock.unlock();
 607             }
 608         }
 609     }
 610
 611     /*
 612      * Help method to registered DoubleCandidateEntity. It is first step
 613      * before the actual instance take Leadership.
 614      */
 615     private void takeOwnership() {
 616         if (isClosed()) {
 617             LOG.debug("Service group {} is closed, skipping cleanup ownership bid", identifier);
 618             return;
 619         }
 620
 621         LOG.debug("Service group {} registering cleanup entity", identifier);
 622
 623         startCapture();
 624         try {
 625             cleanupEntityReg = entityOwnershipService.registerCandidate(cleanupEntity);
 626             cleanupEntityState = EntityState.REGISTERED;
 627         } catch (CandidateAlreadyRegisteredException e) {
 628             LOG.error("Service group {} failed to take ownership", identifier, e);
 629         }
 630
 631         endCapture().forEach(this::lockedOwnershipChanged);
 632     }
 633
 634     /*
 635      * Help method calls instantiateServiceInstance method for create single cluster-wide service instance.
 636      */
 637     @SuppressWarnings("checkstyle:IllegalCatch")
 638     private void startServices() {
 639         if (isClosed()) {
 640             LOG.debug("Service group {} is closed, not starting services", identifier);
 641             return;
 642         }
 643
 644         LOG.debug("Service group {} starting services", identifier);
 645         serviceGroup.forEach(service -> {
 646             LOG.debug("Starting service {}", service);
 647             try {
 648                 service.instantiateServiceInstance();
 649             } catch (Exception e) {
 650                 LOG.warn("Service group {} service {} failed to start, attempting to continue", identifier, service, e);
 651             }
 652         });
 653
 654         localServicesState = ServiceState.STARTED;
 655         LOG.debug("Service group {} services started", identifier);
 656     }
 657
 658     @SuppressWarnings("checkstyle:IllegalCatch")
 659     boolean stopServices() {
 660         switch (localServicesState) {
 661             case STARTED:
 662                 localServicesState = ServiceState.STOPPING;
 663
 664                 final List<ListenableFuture<?>> serviceCloseFutureList = new ArrayList<>(serviceGroup.size());
 665                 for (final ClusterSingletonService service : serviceGroup) {
 666                     final ListenableFuture<?> future;
 667
 668                     try {
 669                         future = service.closeServiceInstance();
 670                     } catch (Exception e) {
 671                         LOG.warn("Service group {} service {} failed to stop, attempting to continue", identifier,
 672                             service, e);
 673                         continue;
 674                     }
 675
 676                     serviceCloseFutureList.add(future);
 677                 }
 678
 679                 LOG.debug("Service group {} initiated service shutdown", identifier);
 680
 681                 Futures.addCallback(Futures.allAsList(serviceCloseFutureList), new FutureCallback<List<?>>() {
 682                     @Override
 683                     public void onFailure(final Throwable cause) {
 684                         LOG.warn("Service group {} service stopping reported error", identifier, cause);
 685                         onServicesStopped();
 686                     }
 687
 688                     @Override
 689                     public void onSuccess(final List<?> nulls) {
 690                         onServicesStopped();
 691                     }
 692                 }, MoreExecutors.directExecutor());
 693
 694                 return localServicesState == ServiceState.STOPPING;
 695             case STOPPED:
 696                 LOG.debug("Service group {} has already stopped services", identifier);
 697                 return false;
 698             case STOPPING:
 699                 LOG.debug("Service group {} is already stopping services", identifier);
 700                 return true;
 701             default:
 702                 throw new IllegalStateException("Unhandled local services state " + localServicesState);
 703         }
 704     }
 705
 706     void onServicesStopped() {
 707         LOG.debug("Service group {} finished stopping services", identifier);
 708         lock.lock();
 709         try {
 710             localServicesState = ServiceState.STOPPED;
 711
 712             if (isClosed()) {
 713                 LOG.debug("Service group {} closed, skipping service restart check", identifier);
 714                 return;
 715             }
 716
 717             // If we lost the service entity while services were stopping, we need to unregister cleanup entity
 718             switch (serviceEntityState) {
 719                 case OWNED:
 720                 case OWNED_JEOPARDY:
 721                     // No need to churn cleanup entity
 722                     break;
 723                 case REGISTERED:
 724                 case UNOWNED:
 725                 case UNREGISTERED:
 726                     if (cleanupEntityReg != null) {
 727                         startCapture();
 728                         cleanupEntityReg.close();
 729                         cleanupEntityReg = null;
 730                         endCapture().forEach(this::lockedOwnershipChanged);
 731                     }
 732                     break;
 733                 default:
 734                     throw new IllegalStateException("Unhandled service entity state" + serviceEntityState);
 735             }
 736
 737             if (cleanupEntityReg == null) {
 738                 LOG.debug("Service group {} does not have cleanup entity registered, skipping restart check",
 739                     identifier);
 740                 return;
 741             }
 742
 743             // Double-check if the services should really be down
 744             switch (cleanupEntityState) {
 745                 case OWNED:
 746                     // We have finished stopping services, but we own cleanup, e.g. we should start them again.
 747                     startServices();
 748                     return;
 749                 case UNOWNED:
 750                 case OWNED_JEOPARDY:
 751                 case REGISTERED:
 752                 case UNREGISTERED:
 753                     break;
 754                 default:
 755                     throw new IllegalStateException("Unhandled cleanup entity state" + cleanupEntityState);
 756             }
 757         } finally {
 758             lock.unlock();
 759             finishCloseIfNeeded();
 760         }
 761     }
 762
 763     @Override
 764     public String toString() {
 765         return MoreObjects.toStringHelper(this).add("identifier", identifier).toString();
 766     }
 767 }