Bug 6710 - Close ClusterSingletonServiceRegistration fix 61/46761/4
authorVaclav Demcak <vdemcak@cisco.com>
Mon, 26 Sep 2016 11:30:41 +0000 (13:30 +0200)
committerTom Pantelis <tpanteli@brocade.com>
Mon, 14 Nov 2016 12:03:10 +0000 (12:03 +0000)
Note: Closing process for CSSRegistration has to follow same
way as a lostLeadership and the main diferences between this
two ways is a not null MainEntityCandidate registration.

* fix using closeClusterSingletonGroup method
  - this method has to be use ONLY without active EOS listener
* fix closeResources method
  - MainEntityCandidate has to be close before lock.release
* fix removing actual CSSGroup instance from EOS listener map
  - close proces has to always finish with removing parth and
    we have to check guardEntityCandidate instece only. Because
    we don't get any aditional notif. from Eos without
    GuardEntityCandidate registration.
* add and modify test
* add async Test Case for SingletonServiceGroup

Change-Id: I15f89cbd43310b0a8e4b7ac78595eeaaa82a65ee
Signed-off-by: Vaclav Demcak <vdemcak@cisco.com>
singleton-service/mdsal-singleton-common-api/src/main/java/org/opendaylight/mdsal/singleton/common/api/ClusterSingletonServiceProvider.java
singleton-service/mdsal-singleton-dom-impl/src/main/java/org/opendaylight/mdsal/singleton/dom/impl/ClusterSingletonServiceGroup.java
singleton-service/mdsal-singleton-dom-impl/src/main/java/org/opendaylight/mdsal/singleton/dom/impl/ClusterSingletonServiceGroupImpl.java
singleton-service/mdsal-singleton-dom-impl/src/test/java/org/opendaylight/mdsal/singleton/dom/impl/ClusterSingletonServiceGroupImplTest.java
singleton-service/mdsal-singleton-dom-impl/src/test/java/org/opendaylight/mdsal/singleton/dom/impl/DOMClusterSingletonServiceProviderAsyncImplTest.java [new file with mode: 0644]
singleton-service/mdsal-singleton-dom-impl/src/test/java/org/opendaylight/mdsal/singleton/dom/impl/DOMClusterSingletonServiceProviderImplTest.java

index f1696c984a816ecec73878661a5600322bfbd772..531483d043880da2bdd44a731dbd7fd6b0974403 100644 (file)
@@ -23,6 +23,13 @@ public interface ClusterSingletonServiceProvider extends AutoCloseable {
 
     /**
      * Method registers {@link ClusterSingletonService} to Provider.
+     * Method returns {@link java.lang.RuntimeException} for unexpected state, so be careful with implementation.
+     * Note: RuntimeException is implemented as a notification about some problems with registration and client
+     * has to implement some strategy for handling this issue.
+     * TODO: RuntimeException is not a transparent contract for handling unexpected state and it needs to be
+     * replaced with a specific documented Exception or it needs to add another contract definition for a client
+     * notification about the unexpected state reason in {@link ClusterSingletonService}.
+     * RuntimeException implementation is a hotfix for an unwanted API contract changes in boron release only.
      *
      * @param service ClusterSingletonService instance
      * @return {@link AutoCloseable} registration
index b9383a32569dd4b94f5d435eece1237fdcd6268e..fd1b2e1500b90ef222864af7833560271d466590 100644 (file)
@@ -66,14 +66,15 @@ interface ClusterSingletonServiceGroup<P extends Path<P>, E extends GenericEntit
     void unregisterService(ClusterSingletonService service);
 
     /**
-     * Method implementation has to apply ownershipChange for all registred services.
+     * Method implementation has to apply ownershipChange for all registered services.
      *
      * @param ownershipChange change role for ClusterSingletonServiceGroup
      */
     void ownershipChanged(final C ownershipChange);
 
     /**
-     * Closes this service group. All registered service providers are also closed.
+     * Closes this service group. All registered service providers are also closed. Please be careful
+     * and use this method without active EOS listener only.
      *
      * @return {@link ListenableFuture} in list for all Future from closing {@link ClusterSingletonService}
      */
index af18888ef24dddd43799754d35041df6cf276887..90ffc10e7a78d0977de5287c30301f93db754ab8 100644 (file)
@@ -56,12 +56,18 @@ final class ClusterSingletonServiceGroupImpl<P extends Path<P>, E extends Generi
 
     private final S entityOwnershipService;
     private final String clusterSingletonGroupIdentifier;
-    private final Semaphore clusterLock = new Semaphore(1);
+    private final Semaphore clusterLock = new Semaphore(1, true);
 
     /* Entity instances */
     private final E serviceEntity;
     private final E doubleCandidateEntity;
 
+    // TODO :it needs to rewrite for StateMachine (INITIALIZED, TRY_TO_TAKE_LEADERSHIP, LEADER, FOLLOWER, TERMINATED)
+    // INITIALIZED : we have registered baseCandidate and we are waiting for first EOS response (!do we really need it?)
+    // FOLLOWER : baseCandidate is registered correctly
+    // TRY_TO_TAKE_LEADERSHIP : guardCandidate is registered correctly
+    // LEADER : both candidate have mastership from EOS
+    // TERMINATED : service go down
     @GuardedBy("clusterLock")
     private boolean hasOwnership = false;
     @GuardedBy("clusterLock")
@@ -98,9 +104,10 @@ final class ClusterSingletonServiceGroupImpl<P extends Path<P>, E extends Generi
         boolean needReleaseLock = false;
         final ListenableFuture<List<Void>> destroyFuture;
         try {
-            needReleaseLock = clusterLock.tryAcquire(10, TimeUnit.SECONDS);
+            needReleaseLock = clusterLock.tryAcquire(1, TimeUnit.SECONDS);
         } catch (final Exception e) {
-            LOG.warn("Unexpected Exception for service Provider {} in closing phase.", clusterSingletonGroupIdentifier, e);
+            LOG.warn("Unexpected Exception for service Provider {} in closing phase.", clusterSingletonGroupIdentifier,
+                    e);
         } finally {
             if (serviceEntityCandidateReg != null) {
                 serviceEntityCandidateReg.close();
@@ -111,10 +118,11 @@ final class ClusterSingletonServiceGroupImpl<P extends Path<P>, E extends Generi
                 for (final ClusterSingletonServiceRegistrationDelegator service : serviceGroup) {
                     serviceCloseFutureList.add(service.closeServiceInstance());
                 }
+                hasOwnership = false;
             }
             destroyFuture = Futures.allAsList(serviceCloseFutureList);
             final Semaphore finalRelease = needReleaseLock ? clusterLock : null;
-            Futures.addCallback(destroyFuture, newAsyncCloseCallback(finalRelease));
+            Futures.addCallback(destroyFuture, newAsyncCloseCallback(finalRelease, true));
         }
         return destroyFuture;
     }
@@ -134,6 +142,7 @@ final class ClusterSingletonServiceGroupImpl<P extends Path<P>, E extends Generi
         } catch (final Exception e) {
             LOG.debug("Unexpected error by registration service Provider {}", clusterSingletonGroupIdentifier, e);
             needCloseProviderInstance = true;
+            throw new RuntimeException(e);
         } finally {
             closeResources(needReleaseLock, needCloseProviderInstance);
         }
@@ -158,6 +167,7 @@ final class ClusterSingletonServiceGroupImpl<P extends Path<P>, E extends Generi
         } catch (final Exception e) {
             LOG.debug("Unexpected error by registration service Provider {}", clusterSingletonGroupIdentifier, e);
             needCloseProviderInstance = true;
+            throw new RuntimeException(e);
         } finally {
             closeResources(needReleaseLock, needCloseProviderInstance);
         }
@@ -173,24 +183,27 @@ final class ClusterSingletonServiceGroupImpl<P extends Path<P>, E extends Generi
         try {
             clusterLock.acquire();
             needReleaseLock = true;
-            serviceGroup.remove(service);
-            if (hasOwnership) {
-                service.closeServiceInstance();
+            if (serviceGroup.size() > 1) {
+                if (hasOwnership) {
+                    service.closeServiceInstance();
+                }
+                serviceGroup.remove(service);
+                LOG.debug("Service {} was removed from group.", service.getIdentifier().getValue());
+            } else {
+                needCloseProviderInstance = true;
             }
         } catch (final Exception e) {
             LOG.debug("Unexpected error by registration service Provider {}", clusterSingletonGroupIdentifier, e);
             needCloseProviderInstance = true;
+            throw new RuntimeException(e);
         } finally {
             closeResources(needReleaseLock, needCloseProviderInstance);
-            if (serviceGroup.isEmpty()) {
-                this.closeClusterSingletonGroup();
-            }
         }
     }
 
     @Override
-    public final void ownershipChanged(final C ownershipChange) {
-        LOG.debug("Ownership change {} for ClusterSingletonServiceGrou {}", ownershipChange,
+    public void ownershipChanged(final C ownershipChange) {
+        LOG.debug("Ownership change {} for ClusterSingletonServiceGroup {}", ownershipChange,
                 clusterSingletonGroupIdentifier);
         try {
             if (ownershipChange.inJeopardy()) {
@@ -222,15 +235,6 @@ final class ClusterSingletonServiceGroupImpl<P extends Path<P>, E extends Generi
                      * SLAVE to MASTER : ownershipChange.getState().isOwner() && !ownershipChange.getState().wasOwner()
                      */
                     takeOwnership();
-                } else if (EntityOwnershipChangeState.LOCAL_OWNERSHIP_LOST_NEW_OWNER.equals(ownershipChange.getState())
-                        || EntityOwnershipChangeState.LOCAL_OWNERSHIP_LOST_NO_OWNER
-                                .equals(ownershipChange.getState())) {
-                    /*
-                     * MASTER to SLAVE : !ownershipChange.getState().isOwner() && ownershipChange.getState().wasOwner()
-                     */
-                    LOG.warn("Unexpected lost doubleCandidate {} leadership. Close service instance {}",
-                            doubleCandidateEntity, clusterSingletonGroupIdentifier);
-                    lostOwnership();
                 } else {
                     /* Not needed notifications */
                     LOG.debug("Not processed doubleCandidate OwnershipChange {} in service Provider {}",
@@ -256,9 +260,12 @@ final class ClusterSingletonServiceGroupImpl<P extends Path<P>, E extends Generi
         try {
             clusterLock.acquire();
             needReleaseLock = true;
-            Verify.verify(serviceEntityCandidateReg != null);
-            Verify.verify(asyncCloseEntityCandidateReg == null);
-            asyncCloseEntityCandidateReg = entityOwnershipService.registerCandidate(doubleCandidateEntity);
+            if (serviceEntityCandidateReg != null) {
+                Verify.verify(asyncCloseEntityCandidateReg == null);
+                asyncCloseEntityCandidateReg = entityOwnershipService.registerCandidate(doubleCandidateEntity);
+            } else {
+                LOG.debug("Service {} is closed, so don't to tryTakeLeadership", clusterSingletonGroupIdentifier);
+            }
         } catch (final Exception e) {
             LOG.error("Unexpected exception state for service Provider {} in TryToTakeLeadership",
                     clusterSingletonGroupIdentifier, e);
@@ -278,12 +285,15 @@ final class ClusterSingletonServiceGroupImpl<P extends Path<P>, E extends Generi
         try {
             clusterLock.acquire();
             needReleaseLock = true;
-            Verify.verify(serviceEntityCandidateReg != null);
-            Verify.verify(asyncCloseEntityCandidateReg != null);
-            for (final ClusterSingletonServiceRegistrationDelegator service : serviceGroup) {
-                service.instantiateServiceInstance();
+            if (serviceEntityCandidateReg != null) {
+                Verify.verify(asyncCloseEntityCandidateReg != null);
+                for (final ClusterSingletonServiceRegistrationDelegator service : serviceGroup) {
+                    service.instantiateServiceInstance();
+                }
+                hasOwnership = true;
+            } else {
+                LOG.debug("Service {} is closed, so don't take leadership", clusterSingletonGroupIdentifier);
             }
-            hasOwnership = true;
         } catch (final Exception e) {
             LOG.error("Unexpected exception state for service Provider {} in TakeLeadership",
                     clusterSingletonGroupIdentifier, e);
@@ -305,17 +315,23 @@ final class ClusterSingletonServiceGroupImpl<P extends Path<P>, E extends Generi
         try {
             clusterLock.acquire();
             needReleaseLock = true;
-            Verify.verify(serviceEntityCandidateReg != null);
             final List<ListenableFuture<Void>> serviceCloseFutureList = new ArrayList<>();
             if (hasOwnership) {
                 Verify.verify(asyncCloseEntityCandidateReg != null);
                 for (final ClusterSingletonServiceRegistrationDelegator service : serviceGroup) {
                     serviceCloseFutureList.add(service.closeServiceInstance());
                 }
+                hasOwnership = false;
             }
 
             final ListenableFuture<List<Void>> destroyFuture = Futures.allAsList(serviceCloseFutureList);
-            Futures.addCallback(destroyFuture, newAsyncCloseCallback(clusterLock));
+            if (serviceEntityCandidateReg != null) {
+                // we don't want to remove this instance from map
+                Futures.addCallback(destroyFuture, newAsyncCloseCallback(clusterLock, false));
+            } else {
+                // we have to remove this ClusterSingletonServiceGroup instance from map
+                Futures.addCallback(destroyFuture, newAsyncCloseCallback(clusterLock, true));
+            }
             /*
              * We wish to stop all possible EOS activities before we don't close
              * a close candidate registration that acts as a guard. So we don't want
@@ -336,46 +352,33 @@ final class ClusterSingletonServiceGroupImpl<P extends Path<P>, E extends Generi
     /*
      * Help method for finalization every acquired functionality
      */
+    @GuardedBy("clusterLock")
     private void closeResources(final boolean needReleaseLock, final boolean needCloseProvider) {
-        if (needReleaseLock) {
-            clusterLock.release();
-        }
         if (needCloseProvider) {
-            final ListenableFuture<List<Void>> closeFutureList = this.closeClusterSingletonGroup();
-            Futures.addCallback(closeFutureList, new FutureCallback<List<Void>>() {
-
-                @Override
-                public void onSuccess(final List<Void> result) {
-                    removeThisGroupFromProvider(null);
-                }
-
-                @Override
-                public void onFailure(final Throwable t) {
-                    removeThisGroupFromProvider(t);
-                }
-            });
+            // The Game Over for this ClusterSingletonServiceGroup instance
+            if (serviceEntityCandidateReg != null) {
+                serviceEntityCandidateReg.close();
+                serviceEntityCandidateReg = null;
+            }
+            // Remove instance immediately because actual state is follower or initialization
+            if (asyncCloseEntityCandidateReg == null) {
+                allServiceGroups.remove(clusterSingletonGroupIdentifier, this);
+            }
         }
-    }
 
-    /*
-     * Help method for final ClusterSingletonGroup removing
-     */
-    protected final void removeThisGroupFromProvider(@Nullable final Throwable t) {
-        LOG.debug("Removing ClusterSingletonServiceGroup {}", clusterSingletonGroupIdentifier);
-        if (t != null) {
-            LOG.warn("Unexpected problem by closingResources ClusterSingletonServiceGroup {}",
-                    clusterSingletonGroupIdentifier);
+        if (needReleaseLock) {
+            clusterLock.release();
         }
-        allServiceGroups.remove(clusterSingletonGroupIdentifier, this);
     }
 
     /*
      * Help method creates FutureCallback for suspend Future
      */
-    private FutureCallback<List<Void>> newAsyncCloseCallback(@Nullable final Semaphore semaphore) {
-        final Consumer<Throwable> closeEntityCandidateRegistration = (@Nullable final Throwable t) -> {
-            if (t != null) {
-                LOG.warn("Unexpected error closing service instance {}", clusterSingletonGroupIdentifier, t);
+    private FutureCallback<List<Void>> newAsyncCloseCallback(@Nullable final Semaphore semaphore,
+            final boolean isInCloseProcess) {
+        final Consumer<Throwable> closeEntityCandidateRegistration = (@Nullable final Throwable throwable) -> {
+            if (throwable != null) {
+                LOG.warn("Unexpected error closing service instance {}", clusterSingletonGroupIdentifier, throwable);
             } else {
                 LOG.debug("Destroy service Instance {} is success", clusterSingletonGroupIdentifier);
             }
@@ -383,10 +386,12 @@ final class ClusterSingletonServiceGroupImpl<P extends Path<P>, E extends Generi
                 asyncCloseEntityCandidateReg.close();
                 asyncCloseEntityCandidateReg = null;
             }
+            if (isInCloseProcess) {
+                allServiceGroups.remove(clusterSingletonGroupIdentifier, this);
+            }
             if (semaphore != null) {
                 semaphore.release();
             }
-            allServiceGroups.remove(clusterSingletonGroupIdentifier, this);
         };
 
         return new FutureCallback<List<Void>>() {
index bf1421435458268edf8edb406631cd0ae3b0d2cb..5019815d5e2e00048c04320689e6e5b890431390 100644 (file)
@@ -8,6 +8,7 @@
 
 package org.opendaylight.mdsal.singleton.dom.impl;
 
+import static org.mockito.Mockito.atLeastOnce;
 import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.never;
@@ -208,7 +209,7 @@ public class ClusterSingletonServiceGroupImplTest {
         Assert.assertNotNull(reg);
         singletonServiceGroup.ownershipChanged(getEntityToJeopardy());
         final ClusterSingletonServiceGroup<?, ?, ?> serviceGroup = map.get(SERVICE_IDENTIFIER);
-        Assert.assertNull(serviceGroup);
+        Assert.assertNotNull(serviceGroup);
         verify(mockClusterSingletonService, never()).instantiateServiceInstance();
         verify(mockEosService, never()).registerCandidate(closeEntity);
     }
@@ -383,7 +384,7 @@ public class ClusterSingletonServiceGroupImplTest {
         verify(mockClusterSingletonService, never()).instantiateServiceInstance();
         verify(mockClusterSingletonService, never()).closeServiceInstance();
         final ClusterSingletonServiceGroup<?, ?, ?> serviceGroup = map.get(SERVICE_IDENTIFIER);
-        Assert.assertNull(serviceGroup);
+        Assert.assertNotNull(serviceGroup);
     }
 
     /**
@@ -409,7 +410,7 @@ public class ClusterSingletonServiceGroupImplTest {
         verify(mockClusterSingletonService, never()).instantiateServiceInstance();
         verify(mockClusterSingletonService, never()).closeServiceInstance();
         final ClusterSingletonServiceGroup<?, ?, ?> serviceGroup = map.get(SERVICE_IDENTIFIER);
-        Assert.assertNull(serviceGroup);
+        Assert.assertNotNull(serviceGroup);
     }
 
     /**
@@ -433,7 +434,7 @@ public class ClusterSingletonServiceGroupImplTest {
         singletonServiceGroup.ownershipChanged(getEntityToJeopardy());
         verify(mockClusterSingletonService).closeServiceInstance();
         final ClusterSingletonServiceGroup<?, ?, ?> serviceGroup = map.get(SERVICE_IDENTIFIER);
-        Assert.assertNull(serviceGroup);
+        Assert.assertNotNull(serviceGroup);
     }
 
     /**
@@ -464,7 +465,7 @@ public class ClusterSingletonServiceGroupImplTest {
      *
      * @throws Exception - unexpected exception
      */
-    @Test
+    @Test(expected = RuntimeException.class)
     public void tryToTakeLeaderForNotInitializedGroupTest() throws Exception {
         map.putIfAbsent(SERVICE_IDENTIFIER, singletonServiceGroup);
         final ClusterSingletonServiceRegistration reg = singletonServiceGroup
@@ -493,7 +494,11 @@ public class ClusterSingletonServiceGroupImplTest {
         singletonServiceGroup.ownershipChanged(getDoubleEntityToMaster());
         verify(mockClusterSingletonService).instantiateServiceInstance();
         reg.close();
-        verify(mockClusterSingletonService).closeServiceInstance();
+        verify(mockClusterSingletonService, never()).closeServiceInstance();
+        singletonServiceGroup.ownershipChanged(getEntityToSlaveNoMaster());
+        verify(mockClusterSingletonService, atLeastOnce()).closeServiceInstance();
+        final ClusterSingletonServiceGroup<?, ?, ?> serviceGroup = map.get(SERVICE_IDENTIFIER);
+        Assert.assertNull(serviceGroup);
     }
 
     /**
@@ -515,9 +520,9 @@ public class ClusterSingletonServiceGroupImplTest {
         singletonServiceGroup.ownershipChanged(getDoubleEntityToMaster());
         verify(mockClusterSingletonService).instantiateServiceInstance();
         singletonServiceGroup.ownershipChanged(getDoubleEntityToSlave());
-        verify(mockClusterSingletonService).closeServiceInstance();
+        verify(mockClusterSingletonService, never()).closeServiceInstance();
         final ClusterSingletonServiceGroup<?, ?, ?> serviceGroup = map.get(SERVICE_IDENTIFIER);
-        Assert.assertNull(serviceGroup);
+        Assert.assertNotNull(serviceGroup);
     }
 
     /**
@@ -540,7 +545,7 @@ public class ClusterSingletonServiceGroupImplTest {
         singletonServiceGroup.ownershipChanged(getDoubleEntityToSlave());
         verify(mockClusterSingletonService, never()).closeServiceInstance();
         final ClusterSingletonServiceGroup<?, ?, ?> serviceGroup = map.get(SERVICE_IDENTIFIER);
-        Assert.assertNull(serviceGroup);
+        Assert.assertNotNull(serviceGroup);
     }
 
     private GenericEntityOwnershipChange<TestInstanceIdentifier, TestEntity> getEntityToMaster() {
diff --git a/singleton-service/mdsal-singleton-dom-impl/src/test/java/org/opendaylight/mdsal/singleton/dom/impl/DOMClusterSingletonServiceProviderAsyncImplTest.java b/singleton-service/mdsal-singleton-dom-impl/src/test/java/org/opendaylight/mdsal/singleton/dom/impl/DOMClusterSingletonServiceProviderAsyncImplTest.java
new file mode 100644 (file)
index 0000000..9636252
--- /dev/null
@@ -0,0 +1,834 @@
+/*
+ * Copyright (c) 2016 Cisco Systems, Inc. and others.  All rights reserved.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v1.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v10.html
+ */
+
+package org.opendaylight.mdsal.singleton.dom.impl;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.atLeastOnce;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
+import java.util.Timer;
+import java.util.TimerTask;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.mockito.Mock;
+import org.mockito.MockitoAnnotations;
+import org.opendaylight.mdsal.eos.common.api.CandidateAlreadyRegisteredException;
+import org.opendaylight.mdsal.eos.common.api.EntityOwnershipChangeState;
+import org.opendaylight.mdsal.eos.dom.api.DOMEntity;
+import org.opendaylight.mdsal.eos.dom.api.DOMEntityOwnershipCandidateRegistration;
+import org.opendaylight.mdsal.eos.dom.api.DOMEntityOwnershipChange;
+import org.opendaylight.mdsal.eos.dom.api.DOMEntityOwnershipListenerRegistration;
+import org.opendaylight.mdsal.eos.dom.api.DOMEntityOwnershipService;
+import org.opendaylight.mdsal.singleton.common.api.ClusterSingletonService;
+import org.opendaylight.mdsal.singleton.common.api.ClusterSingletonServiceRegistration;
+import org.opendaylight.mdsal.singleton.common.api.ServiceGroupIdentifier;
+
+/*
+ * Testing {@link DOMClusterSingletonServiceProviderImpl} implementation
+ */
+public final class DOMClusterSingletonServiceProviderAsyncImplTest {
+
+    private static final String SERVICE_ENTITY_TYPE = "org.opendaylight.mdsal.ServiceEntityType";
+    private static final String CLOSE_SERVICE_ENTITY_TYPE = "org.opendaylight.mdsal.AsyncServiceCloseEntityType";
+    private static final String SERVICE_NAME = "testServiceName";
+
+    @Mock
+    private DOMEntityOwnershipService mockEos;
+    @Mock
+    private DOMEntityOwnershipCandidateRegistration mockEntityCandReg;
+    @Mock
+    private DOMEntityOwnershipCandidateRegistration mockDoubleEntityCandReg;
+    @Mock
+    private DOMEntityOwnershipListenerRegistration mockEosEntityListReg;
+    @Mock
+    private DOMEntityOwnershipListenerRegistration mockEosDoubleEntityListReg;
+
+    private DOMClusterSingletonServiceProviderImpl clusterSingletonServiceProvider;
+    private TestClusterSingletonAsyncServiceInstance clusterSingletonService;
+    private TestClusterSingletonAsyncServiceInstance clusterSingletonService2;
+
+    private final DOMEntity entity = new DOMEntity(SERVICE_ENTITY_TYPE, SERVICE_NAME);
+    private final DOMEntity doubleEntity = new DOMEntity(CLOSE_SERVICE_ENTITY_TYPE, SERVICE_NAME);
+
+    protected static Timer timer;
+    protected static long ASYNC_TIME_DELAY_SEC = 100L;
+
+    @BeforeClass
+    public static void asyncInitTest() {
+        timer = new Timer();
+    }
+
+    @AfterClass
+    public static void cleanTest() {
+        timer.cancel();
+    }
+
+    /**
+     * Initialization functionality for every Tests in this suite.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Before
+    public void setup() throws Exception {
+        MockitoAnnotations.initMocks(this);
+
+        doNothing().when(mockEosEntityListReg).close();
+        doNothing().when(mockEosDoubleEntityListReg).close();
+        doNothing().when(mockEntityCandReg).close();
+        doNothing().when(mockDoubleEntityCandReg).close();
+        doReturn(mockEosEntityListReg).when(mockEos).registerListener(eq(SERVICE_ENTITY_TYPE),
+                any(DOMClusterSingletonServiceProviderImpl.class));
+        doReturn(mockEosDoubleEntityListReg).when(mockEos).registerListener(eq(CLOSE_SERVICE_ENTITY_TYPE),
+                any(DOMClusterSingletonServiceProviderImpl.class));
+        doReturn(mockEntityCandReg).when(mockEos).registerCandidate(entity);
+        doReturn(mockDoubleEntityCandReg).when(mockEos).registerCandidate(doubleEntity);
+
+        clusterSingletonServiceProvider = new DOMClusterSingletonServiceProviderImpl(mockEos);
+        clusterSingletonServiceProvider.initializeProvider();
+        verify(mockEos).registerListener(SERVICE_ENTITY_TYPE, clusterSingletonServiceProvider);
+        verify(mockEos).registerListener(CLOSE_SERVICE_ENTITY_TYPE, clusterSingletonServiceProvider);
+
+        clusterSingletonService = new TestClusterSingletonAsyncServiceInstance();
+        clusterSingletonService2 = new TestClusterSingletonAsyncServiceInstance();
+
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService2.getServiceState());
+    }
+
+    /**
+     * Test checks NullPointer for null {@link DOMEntityOwnershipService} input value.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test(expected = NullPointerException.class)
+    public void initializationClusterSingletonServiceProviderNullInputTest() throws Exception {
+        clusterSingletonServiceProvider = new DOMClusterSingletonServiceProviderImpl(null);
+    }
+
+    /**
+     * Test GoldPath for close {@link DOMClusterSingletonServiceProviderImpl}.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void closeClusterSingletonServiceProviderTest() throws Exception {
+        verify(mockEos).registerListener(SERVICE_ENTITY_TYPE, clusterSingletonServiceProvider);
+        verify(mockEos).registerListener(CLOSE_SERVICE_ENTITY_TYPE, clusterSingletonServiceProvider);
+        clusterSingletonServiceProvider.close();
+        verify(mockEosEntityListReg).close();
+        verify(mockEosDoubleEntityListReg).close();
+    }
+
+    /**
+     * Test parser ServiceIdentifier from Entity.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void makeEntityClusterSingletonServiceProviderTest() throws Exception {
+        final DOMEntity testEntity = clusterSingletonServiceProvider.createEntity(SERVICE_ENTITY_TYPE, SERVICE_NAME);
+        Assert.assertEquals(entity, testEntity);
+        final DOMEntity testDbEn = clusterSingletonServiceProvider.createEntity(CLOSE_SERVICE_ENTITY_TYPE,
+                SERVICE_NAME);
+        Assert.assertEquals(doubleEntity, testDbEn);
+    }
+
+    /**
+     * Test parser ServiceIdentifier from Entity.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void getIdentifierClusterSingletonServiceProviderTest() throws Exception {
+        final String entityIdentifier = clusterSingletonServiceProvider.getServiceIdentifierFromEntity(entity);
+        Assert.assertEquals(SERVICE_NAME, entityIdentifier);
+        final String doubleEntityId = clusterSingletonServiceProvider.getServiceIdentifierFromEntity(doubleEntity);
+        Assert.assertEquals(SERVICE_NAME, doubleEntityId);
+    }
+
+    /**
+     * Test GoldPath for initialization {@link ClusterSingletonService}.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void initializationClusterSingletonServiceTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        verify(mockEos).registerCandidate(entity);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+    }
+
+    /**
+     * Test GoldPath for initialization with init ownership result SLAVE {@link ClusterSingletonService}.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void slaveInitClusterSingletonServiceTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        verify(mockEos).registerCandidate(entity);
+        clusterSingletonServiceProvider.ownershipChanged(getInitEntityToSlave());
+        verify(mockEos, never()).registerCandidate(doubleEntity);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+    }
+
+    /**
+     * Test GoldPath for initialization with init ownership result SLAVE, but NO-MASTER {@link ClusterSingletonService}.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void slaveInitNoMasterClusterSingletonServiceTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        verify(mockEos).registerCandidate(entity);
+        clusterSingletonServiceProvider.ownershipChanged(getInitEntityToSlaveNoMaster());
+        verify(mockEos, never()).registerCandidate(doubleEntity);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+    }
+
+    /**
+     * Test GoldPath for initialization with init ownership result MASTER {@link ClusterSingletonService}.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void masterInitClusterSingletonServiceTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        verify(mockEos).registerCandidate(entity);
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToMaster());
+        verify(mockEos).registerCandidate(doubleEntity);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+    }
+
+    /**
+     * Test GoldPath for initialization with init ownership result MASTER {@link ClusterSingletonService}.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void masterInitSlaveDoubleCandidateClusterSingletonServiceTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        verify(mockEos).registerCandidate(entity);
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToMaster());
+        verify(mockEos).registerCandidate(doubleEntity);
+        clusterSingletonServiceProvider.ownershipChanged(getInitDoubleEntityToSlave());
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+    }
+
+    /**
+     * Test GoldPath for takeLeadership with ownership result MASTER {@link ClusterSingletonService}.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void takeLeadershipClusterSingletonServiceTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        verify(mockEos).registerCandidate(entity);
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToMaster());
+        verify(mockEos).registerCandidate(doubleEntity);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getDoubleEntityToMaster());
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
+    }
+
+    /**
+     * Test GoldPath for takeLeadership with ownership result MASTER {@link ClusterSingletonService}.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void takeDoubleLeadershipClusterSingletonServiceTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        verify(mockEos).registerCandidate(entity);
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToMaster());
+        verify(mockEos).registerCandidate(doubleEntity);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getDoubleEntityToMaster());
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToSlave());
+        Assert.assertEquals(TestClusterSingletonServiceState.DESTROYED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToMaster());
+        Assert.assertEquals(TestClusterSingletonServiceState.DESTROYED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getInitDoubleEntityToSlave());
+        clusterSingletonServiceProvider.ownershipChanged(getDoubleEntityToMaster());
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
+        verify(mockEosDoubleEntityListReg, never()).close();
+        verify(mockEosEntityListReg, never()).close();
+        verify(mockEntityCandReg, never()).close();
+        verify(mockDoubleEntityCandReg).close();
+    }
+
+    /**
+     * Test GoldPath for initialization with init ownership result MASTER {@link ClusterSingletonService}.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void masterInitClusterSingletonServiceTwoServicesTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        verify(mockEos).registerCandidate(entity);
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToMaster());
+        verify(mockEos).registerCandidate(doubleEntity);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        final ClusterSingletonServiceRegistration reg2 = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService2);
+        Assert.assertNotNull(reg2);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService2.getServiceState());
+    }
+
+    /**
+     * Test GoldPath for takeLeadership with ownership result MASTER {@link ClusterSingletonService}.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void takeLeadershipClusterSingletonServiceTwoAddDuringWaitPhaseServicesTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        verify(mockEos).registerCandidate(entity);
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToMaster());
+        verify(mockEos).registerCandidate(doubleEntity);
+        final ClusterSingletonServiceRegistration reg2 = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService2);
+        Assert.assertNotNull(reg2);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService2.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getDoubleEntityToMaster());
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService2.getServiceState());
+    }
+
+    /**
+     * Test GoldPath for takeLeadership with ownership result MASTER {@link ClusterSingletonService}.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void takeLeadershipClusterSingletonServiceTowServicesTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        verify(mockEos).registerCandidate(entity);
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToMaster());
+        verify(mockEos).registerCandidate(doubleEntity);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getDoubleEntityToMaster());
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
+        final ClusterSingletonServiceRegistration reg2 = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService2);
+        Assert.assertNotNull(reg2);
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService2.getServiceState());
+    }
+
+    /**
+     * Test checks CandidateAlreadyRegisteredException processing in initialization phase.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test(expected = RuntimeException.class)
+    public void initializationClusterSingletonServiceCandidateAlreadyRegistredTest() throws Exception {
+        doThrow(CandidateAlreadyRegisteredException.class).when(mockEos).registerCandidate(entity);
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNull(reg);
+    }
+
+    /**
+     * Test GoldPath for lostLeadership during tryToTakeLeadership with ownership result MASTER
+     * {@link ClusterSingletonService}.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void lostLeadershipDuringTryToTakeLeadershipClusterSingletonServiceTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        verify(mockEos).registerCandidate(entity);
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToMaster());
+        verify(mockEos).registerCandidate(doubleEntity);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToSlave());
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getDoubleEntityToMaster());
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+    }
+
+    /**
+     * Test GoldPath for lostLeadership with ownership result MASTER-TO-SLAVE {@link ClusterSingletonService}.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void lostLeadershipClusterSingletonServiceTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        verify(mockEos).registerCandidate(entity);
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToMaster());
+        verify(mockEos).registerCandidate(doubleEntity);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getInitDoubleEntityToSlave());
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getDoubleEntityToMaster());
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToSlave());
+        Assert.assertEquals(TestClusterSingletonServiceState.DESTROYED, clusterSingletonService.getServiceState());
+    }
+
+    /**
+     * Test checks unexpected change for MASTER-TO-SLAVE double Candidate role change.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void unexpectedLostLeadershipDoubleCandidateTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        verify(mockEos).registerCandidate(entity);
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToMaster());
+        verify(mockEos).registerCandidate(doubleEntity);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getInitDoubleEntityToSlave());
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getDoubleEntityToMaster());
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getDoubleEntityToSlave());
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
+        Thread.sleep(ASYNC_TIME_DELAY_SEC * 2);
+        verify(mockEosDoubleEntityListReg, never()).close();
+        verify(mockEntityCandReg, never()).close();
+        verify(mockDoubleEntityCandReg, never()).close();
+        reg.close();
+        verify(mockEosDoubleEntityListReg, never()).close();
+        verify(mockEntityCandReg, atLeastOnce()).close();
+        verify(mockDoubleEntityCandReg, never()).close();
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToSlave());
+        Thread.sleep(ASYNC_TIME_DELAY_SEC * 2);
+        verify(mockEntityCandReg, atLeastOnce()).close();
+        verify(mockDoubleEntityCandReg, atLeastOnce()).close();
+        verify(mockEosDoubleEntityListReg, never()).close();
+        Assert.assertEquals(TestClusterSingletonServiceState.DESTROYED, clusterSingletonService.getServiceState());
+    }
+
+    /**
+     * Test checks inJeopardy Cluster Node state for Master Instance.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void inJeopardyMasterTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        verify(mockEos).registerCandidate(entity);
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToMaster());
+        verify(mockEos).registerCandidate(doubleEntity);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getInitDoubleEntityToSlave());
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getDoubleEntityToMaster());
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToJeopardy());
+        Assert.assertEquals(TestClusterSingletonServiceState.DESTROYED, clusterSingletonService.getServiceState());
+        Thread.sleep(ASYNC_TIME_DELAY_SEC * 2);
+        verify(mockEosEntityListReg, never()).close();
+        verify(mockEosDoubleEntityListReg, never()).close();
+        verify(mockEntityCandReg, never()).close();
+        verify(mockDoubleEntityCandReg).close();
+    }
+
+    /**
+     * Test checks inJeopardy Cluster Node state for Slave Instance.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void inJeopardySlaveTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        verify(mockEos).registerCandidate(entity);
+        clusterSingletonServiceProvider.ownershipChanged(getInitEntityToSlave());
+        verify(mockEos, never()).registerCandidate(doubleEntity);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToJeopardy());
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        verify(mockEosEntityListReg, never()).close();
+        verify(mockEosDoubleEntityListReg, never()).close();
+        verify(mockEntityCandReg, never()).close();
+        verify(mockDoubleEntityCandReg, never()).close();
+    }
+
+    /**
+     * Test checks close processing for {@link ClusterSingletonServiceRegistration}.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void closeClusterSingletonServiceRegistrationNoRoleTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        verify(mockEos).registerCandidate(entity);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        reg.close();
+        verify(mockEosEntityListReg, never()).close();
+        verify(mockEosDoubleEntityListReg, never()).close();
+        verify(mockEntityCandReg).close();
+        verify(mockDoubleEntityCandReg, never()).close();
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+    }
+
+    /**
+     * Test checks close processing for {@link ClusterSingletonServiceRegistration}.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void closeClusterSingletonServiceRegistrationNoRoleTwoServicesTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        verify(mockEos).registerCandidate(entity);
+        final ClusterSingletonServiceRegistration reg2 = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService2);
+        Assert.assertNotNull(reg2);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService2.getServiceState());
+        reg.close();
+        verify(mockEosEntityListReg, never()).close();
+        verify(mockEosDoubleEntityListReg, never()).close();
+        verify(mockEntityCandReg, never()).close();
+        verify(mockDoubleEntityCandReg, never()).close();
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService2.getServiceState());
+    }
+
+    /**
+     * Test checks close processing for {@link ClusterSingletonServiceRegistration}.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void closeClusterSingletonServiceRegistrationSlaveTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        verify(mockEos).registerCandidate(entity);
+        clusterSingletonServiceProvider.ownershipChanged(getInitEntityToSlave());
+        reg.close();
+        verify(mockEosEntityListReg, never()).close();
+        verify(mockEosDoubleEntityListReg, never()).close();
+        verify(mockEntityCandReg).close();
+        verify(mockDoubleEntityCandReg, never()).close();
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+    }
+
+    /**
+     * Test checks close processing for {@link ClusterSingletonServiceRegistration}.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void closeClusterSingletonServiceRegistrationSlaveTwoServicesTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        verify(mockEos).registerCandidate(entity);
+        final ClusterSingletonServiceRegistration reg2 = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService2);
+        Assert.assertNotNull(reg2);
+        clusterSingletonServiceProvider.ownershipChanged(getInitEntityToSlave());
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService2.getServiceState());
+        reg.close();
+        verify(mockEosEntityListReg, never()).close();
+        verify(mockEosDoubleEntityListReg, never()).close();
+        verify(mockEntityCandReg, never()).close();
+        verify(mockDoubleEntityCandReg, never()).close();
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService2.getServiceState());
+    }
+
+    /**
+     * Test checks close processing for {@link ClusterSingletonServiceRegistration}.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void closeClusterSingletonServiceRegistrationMasterTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        verify(mockEos).registerCandidate(entity);
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToMaster());
+        verify(mockEos).registerCandidate(doubleEntity);
+        clusterSingletonServiceProvider.ownershipChanged(getDoubleEntityToMaster());
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
+        reg.close();
+        Thread.sleep(ASYNC_TIME_DELAY_SEC * 2);
+        verify(mockEosEntityListReg, never()).close();
+        verify(mockEosDoubleEntityListReg, never()).close();
+        verify(mockEntityCandReg, atLeastOnce()).close();
+        verify(mockDoubleEntityCandReg, never()).close();
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToSlave());
+        Thread.sleep(ASYNC_TIME_DELAY_SEC * 2);
+        verify(mockEntityCandReg, atLeastOnce()).close();
+        verify(mockDoubleEntityCandReg, atLeastOnce()).close();
+        verify(mockEosDoubleEntityListReg, never()).close();
+        Assert.assertEquals(TestClusterSingletonServiceState.DESTROYED, clusterSingletonService.getServiceState());
+    }
+
+    /**
+     * Test checks close processing for {@link ClusterSingletonServiceRegistration}.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void closeClusterSingletonServiceRegistrationMasterCloseWithNotificationTimesTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        verify(mockEos).registerCandidate(entity);
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToMaster());
+        verify(mockEos).registerCandidate(doubleEntity);
+        clusterSingletonServiceProvider.ownershipChanged(getDoubleEntityToMaster());
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
+        reg.close();
+        Thread.sleep(ASYNC_TIME_DELAY_SEC * 2);
+        verify(mockEosEntityListReg, never()).close();
+        verify(mockEosDoubleEntityListReg, never()).close();
+        verify(mockEntityCandReg, atLeastOnce()).close();
+        verify(mockDoubleEntityCandReg, never()).close();
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToSlave());
+        Thread.sleep(ASYNC_TIME_DELAY_SEC * 2);
+        verify(mockEntityCandReg, atLeastOnce()).close();
+        verify(mockDoubleEntityCandReg, atLeastOnce()).close();
+        verify(mockEosDoubleEntityListReg, never()).close();
+        Assert.assertEquals(TestClusterSingletonServiceState.DESTROYED, clusterSingletonService.getServiceState());
+    }
+
+    /**
+     * Test checks close processing for {@link ClusterSingletonServiceRegistration}.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void closeClusterSingletonServiceRegistrationMasterCloseCoupleTimesTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        verify(mockEos).registerCandidate(entity);
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToMaster());
+        verify(mockEos).registerCandidate(doubleEntity);
+        clusterSingletonServiceProvider.ownershipChanged(getDoubleEntityToMaster());
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
+        reg.close();
+        reg.close();
+        Thread.sleep(ASYNC_TIME_DELAY_SEC * 2);
+        verify(mockEosEntityListReg, never()).close();
+        verify(mockEosDoubleEntityListReg, never()).close();
+        verify(mockEntityCandReg, atLeastOnce()).close();
+        verify(mockDoubleEntityCandReg, never()).close();
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToSlave());
+        Thread.sleep(ASYNC_TIME_DELAY_SEC * 2);
+        verify(mockEntityCandReg, atLeastOnce()).close();
+        verify(mockDoubleEntityCandReg, atLeastOnce()).close();
+        verify(mockEosDoubleEntityListReg, never()).close();
+        Assert.assertEquals(TestClusterSingletonServiceState.DESTROYED, clusterSingletonService.getServiceState());
+    }
+
+    /**
+     * Test checks close processing for {@link ClusterSingletonServiceRegistration}.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void closeClusterSingletonServiceRegistrationMasterTwoServicesTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        final ClusterSingletonServiceRegistration reg2 = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService2);
+        Assert.assertNotNull(reg2);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService2.getServiceState());
+        verify(mockEos).registerCandidate(entity);
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToMaster());
+        verify(mockEos).registerCandidate(doubleEntity);
+        clusterSingletonServiceProvider.ownershipChanged(getDoubleEntityToMaster());
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService2.getServiceState());
+        reg.close();
+        verify(mockEosEntityListReg, never()).close();
+        verify(mockEosDoubleEntityListReg, never()).close();
+        verify(mockEntityCandReg, never()).close();
+        verify(mockDoubleEntityCandReg, never()).close();
+        Assert.assertEquals(TestClusterSingletonServiceState.DESTROYED, clusterSingletonService.getServiceState());
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService2.getServiceState());
+    }
+
+    /**
+     * Test checks validation Error processing for SLAVE-TO-MASTER entity Candidate role change.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void tryToTakeLeaderForClosedServiceRegistrationTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        final ClusterSingletonServiceRegistration reg2 = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService2);
+        Assert.assertNotNull(reg2);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService2.getServiceState());
+        verify(mockEos).registerCandidate(entity);
+        reg.close();
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToMaster());
+        verify(mockEos).registerCandidate(doubleEntity);
+        clusterSingletonServiceProvider.ownershipChanged(getDoubleEntityToMaster());
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService2.getServiceState());
+        verify(mockEosEntityListReg, never()).close();
+        verify(mockEosDoubleEntityListReg, never()).close();
+        verify(mockEntityCandReg, never()).close();
+        verify(mockDoubleEntityCandReg, never()).close();
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService2.getServiceState());
+    }
+
+    private DOMEntityOwnershipChange getEntityToMaster() {
+        return new DOMEntityOwnershipChange(entity, EntityOwnershipChangeState.from(false, true, true));
+    }
+
+    private DOMEntityOwnershipChange getEntityToSlave() {
+        return new DOMEntityOwnershipChange(entity, EntityOwnershipChangeState.from(true, false, true));
+    }
+
+    private DOMEntityOwnershipChange getInitEntityToSlave() {
+        return new DOMEntityOwnershipChange(entity, EntityOwnershipChangeState.from(false, false, true));
+    }
+
+    private DOMEntityOwnershipChange getInitEntityToSlaveNoMaster() {
+        return new DOMEntityOwnershipChange(entity, EntityOwnershipChangeState.from(false, false, false));
+    }
+
+    private DOMEntityOwnershipChange getDoubleEntityToMaster() {
+        return new DOMEntityOwnershipChange(doubleEntity, EntityOwnershipChangeState.from(false, true, true));
+    }
+
+    private DOMEntityOwnershipChange getInitDoubleEntityToSlave() {
+        return new DOMEntityOwnershipChange(doubleEntity, EntityOwnershipChangeState.from(false, false, true));
+    }
+
+    private DOMEntityOwnershipChange getDoubleEntityToSlave() {
+        return new DOMEntityOwnershipChange(doubleEntity, EntityOwnershipChangeState.from(true, false, true));
+    }
+
+    private DOMEntityOwnershipChange getEntityToJeopardy() {
+        return new DOMEntityOwnershipChange(entity, EntityOwnershipChangeState.from(false, false, false), true);
+    }
+
+    /*
+     * Base states for AbstractClusterProjectProvider
+     */
+    enum TestClusterSingletonServiceState {
+        /*
+         * State represents a correct Instantiated process
+         */
+        STARTED,
+        /*
+         * State represents a correct call abstract method instantiatingProject
+         */
+        INITIALIZED,
+        /*
+         * State represents a correct call abstract method destryingProject
+         */
+        DESTROYED;
+    }
+
+    /*
+     * Test implementation of {@link ClusterSingletonService}
+     */
+    class TestClusterSingletonAsyncServiceInstance implements ClusterSingletonService {
+
+        private final ServiceGroupIdentifier serviceId = ServiceGroupIdentifier.create(SERVICE_NAME);
+        private TestClusterSingletonServiceState serviceState;
+        protected SettableFuture<Void> future;
+
+        TestClusterSingletonAsyncServiceInstance() {
+            this.serviceState = TestClusterSingletonServiceState.INITIALIZED;
+        }
+
+        @Override
+        public void instantiateServiceInstance() {
+            this.serviceState = TestClusterSingletonServiceState.STARTED;
+        }
+
+        @Override
+        public ListenableFuture<Void> closeServiceInstance() {
+            this.serviceState = TestClusterSingletonServiceState.DESTROYED;
+            future = SettableFuture.create();
+            timer.schedule(new TimerTask() {
+
+                @Override
+                public void run() {
+                    future.set(null);
+                }
+            }, ASYNC_TIME_DELAY_SEC);
+            return future;
+        }
+
+        public TestClusterSingletonServiceState getServiceState() {
+            return serviceState;
+        }
+
+        @Override
+        public ServiceGroupIdentifier getIdentifier() {
+            return serviceId;
+        }
+    }
+}
index 6a05eb1cbbc644a9353ab219c7f4cd3de995a331..da9cd477ef1c81ff400e40a304d676bc0f806f20 100644 (file)
@@ -10,6 +10,7 @@ package org.opendaylight.mdsal.singleton.dom.impl;
 
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.atLeastOnce;
 import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.doThrow;
@@ -266,6 +267,35 @@ public class DOMClusterSingletonServiceProviderImplTest {
      * @throws Exception
      */
     @Test
+    public void takeDoubleLeadershipClusterSingletonServiceTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        verify(mockEos).registerCandidate(entity);
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToMaster());
+        verify(mockEos).registerCandidate(doubleEntity);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getDoubleEntityToMaster());
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToSlave());
+        Assert.assertEquals(TestClusterSingletonServiceState.DESTROYED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToMaster());
+        Assert.assertEquals(TestClusterSingletonServiceState.DESTROYED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getInitDoubleEntityToSlave());
+        clusterSingletonServiceProvider.ownershipChanged(getDoubleEntityToMaster());
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
+        verify(mockEosDoubleEntityListReg, never()).close();
+        verify(mockEosEntityListReg, never()).close();
+        verify(mockEntityCandReg, never()).close();
+        verify(mockDoubleEntityCandReg).close();
+    }
+
+    /**
+     * Test GoldPath for takeLeadership with ownership result MASTER {@link ClusterSingletonService}.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
     public void takeLeadershipClusterSingletonServiceTwoAddDuringWaitPhaseServicesTest() throws Exception {
         final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
                 .registerClusterSingletonService(clusterSingletonService);
@@ -310,7 +340,7 @@ public class DOMClusterSingletonServiceProviderImplTest {
      *
      * @throws Exception
      */
-    @Test
+    @Test(expected = RuntimeException.class)
     public void initializationClusterSingletonServiceCandidateAlreadyRegistredTest() throws Exception {
         doThrow(CandidateAlreadyRegisteredException.class).when(mockEos).registerCandidate(entity);
         final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
@@ -362,7 +392,7 @@ public class DOMClusterSingletonServiceProviderImplTest {
     }
 
     /**
-     * Test checks umexpected change for MASTER-TO-SLAVE double Candidate role change
+     * Test checks unexpected change for MASTER-TO-SLAVE double Candidate role change.
      *
      * @throws Exception
      */
@@ -380,10 +410,21 @@ public class DOMClusterSingletonServiceProviderImplTest {
         clusterSingletonServiceProvider.ownershipChanged(getDoubleEntityToMaster());
         Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
         clusterSingletonServiceProvider.ownershipChanged(getDoubleEntityToSlave());
-        Assert.assertEquals(TestClusterSingletonServiceState.DESTROYED, clusterSingletonService.getServiceState());
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
         verify(mockEosDoubleEntityListReg, never()).close();
         verify(mockEntityCandReg, never()).close();
-        verify(mockDoubleEntityCandReg).close();
+        verify(mockDoubleEntityCandReg, never()).close();
+        reg.close();
+        verify(mockEosEntityListReg, never()).close();
+        verify(mockEosDoubleEntityListReg, never()).close();
+        verify(mockEntityCandReg, atLeastOnce()).close();
+        verify(mockDoubleEntityCandReg, never()).close();
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToSlave());
+        verify(mockEntityCandReg, atLeastOnce()).close();
+        verify(mockDoubleEntityCandReg, atLeastOnce()).close();
+        verify(mockEosDoubleEntityListReg, never()).close();
+        Assert.assertEquals(TestClusterSingletonServiceState.DESTROYED, clusterSingletonService.getServiceState());
     }
 
     /**
@@ -543,6 +584,7 @@ public class DOMClusterSingletonServiceProviderImplTest {
         clusterSingletonServiceProvider.ownershipChanged(getDoubleEntityToMaster());
         Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
         reg.close();
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToSlave());
         verify(mockEosEntityListReg, never()).close();
         verify(mockEosDoubleEntityListReg, never()).close();
         verify(mockEntityCandReg).close();
@@ -551,7 +593,66 @@ public class DOMClusterSingletonServiceProviderImplTest {
     }
 
     /**
-     * Test checks close procesing for {@link ClusterSingletonServiceRegistration}
+     * Test checks close processing for {@link ClusterSingletonServiceRegistration}.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void closeClusterSingletonServiceRegistrationMasterCloseWithNotificationTimesTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        verify(mockEos).registerCandidate(entity);
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToMaster());
+        verify(mockEos).registerCandidate(doubleEntity);
+        clusterSingletonServiceProvider.ownershipChanged(getDoubleEntityToMaster());
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
+        reg.close();
+        verify(mockEosEntityListReg, never()).close();
+        verify(mockEosDoubleEntityListReg, never()).close();
+        verify(mockEntityCandReg, atLeastOnce()).close();
+        verify(mockDoubleEntityCandReg, never()).close();
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToSlave());
+        verify(mockEntityCandReg, atLeastOnce()).close();
+        verify(mockDoubleEntityCandReg, atLeastOnce()).close();
+        verify(mockEosDoubleEntityListReg, never()).close();
+        Assert.assertEquals(TestClusterSingletonServiceState.DESTROYED, clusterSingletonService.getServiceState());
+    }
+
+    /**
+     * Test checks close processing for {@link ClusterSingletonServiceRegistration}.
+     *
+     * @throws Exception if the condition does not meet
+     */
+    @Test
+    public void closeClusterSingletonServiceRegistrationMasterCloseCoupleTimesTest() throws Exception {
+        final ClusterSingletonServiceRegistration reg = clusterSingletonServiceProvider
+                .registerClusterSingletonService(clusterSingletonService);
+        Assert.assertNotNull(reg);
+        Assert.assertEquals(TestClusterSingletonServiceState.INITIALIZED, clusterSingletonService.getServiceState());
+        verify(mockEos).registerCandidate(entity);
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToMaster());
+        verify(mockEos).registerCandidate(doubleEntity);
+        clusterSingletonServiceProvider.ownershipChanged(getDoubleEntityToMaster());
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
+        reg.close();
+        reg.close();
+        verify(mockEosEntityListReg, never()).close();
+        verify(mockEosDoubleEntityListReg, never()).close();
+        verify(mockEntityCandReg, atLeastOnce()).close();
+        verify(mockDoubleEntityCandReg, never()).close();
+        Assert.assertEquals(TestClusterSingletonServiceState.STARTED, clusterSingletonService.getServiceState());
+        clusterSingletonServiceProvider.ownershipChanged(getEntityToSlave());
+        verify(mockEntityCandReg, atLeastOnce()).close();
+        verify(mockDoubleEntityCandReg, atLeastOnce()).close();
+        verify(mockEosDoubleEntityListReg, never()).close();
+        Assert.assertEquals(TestClusterSingletonServiceState.DESTROYED, clusterSingletonService.getServiceState());
+    }
+
+    /**
+     * Test checks close processing for {@link ClusterSingletonServiceRegistration}.
      *
      * @throws Exception
      */
@@ -645,15 +746,18 @@ public class DOMClusterSingletonServiceProviderImplTest {
     /**
      * Base states for AbstractClusterProjectProvider
      */
-    static enum TestClusterSingletonServiceState {
+    enum TestClusterSingletonServiceState {
+
         /**
          * State represents a correct Instantiated process
          */
+
         STARTED,
         /**
          * State represents a correct call abstract method instantiatingProject
          */
         INITIALIZED,
+
         /**
          * State represents a correct call abstract method destryingProject
          */