idmanager/idmanager-impl/src/main/java/org/opendaylight/genius/idmanager/IdManager.java

   1 /*
   2  * Copyright (c) 2016 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.genius.idmanager;
   9
  10 import static java.util.Comparator.comparing;
  11 import static java.util.stream.Collectors.toCollection;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.mdsal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  14 import static org.opendaylight.yangtools.yang.binding.CodeHelpers.nonnull;
  15
  16 import com.google.common.util.concurrent.Futures;
  17 import com.google.common.util.concurrent.ListenableFuture;
  18 import com.google.common.util.concurrent.MoreExecutors;
  19 import java.util.ArrayList;
  20 import java.util.Collections;
  21 import java.util.HashMap;
  22 import java.util.LinkedList;
  23 import java.util.List;
  24 import java.util.Map;
  25 import java.util.Objects;
  26 import java.util.Optional;
  27 import java.util.Timer;
  28 import java.util.TimerTask;
  29 import java.util.concurrent.CompletableFuture;
  30 import java.util.concurrent.ConcurrentHashMap;
  31 import java.util.concurrent.ConcurrentMap;
  32 import java.util.concurrent.CountDownLatch;
  33 import java.util.concurrent.ExecutionException;
  34 import java.util.concurrent.TimeUnit;
  35 import javax.annotation.PostConstruct;
  36 import javax.annotation.PreDestroy;
  37 import javax.inject.Inject;
  38 import javax.inject.Singleton;
  39 import org.apache.aries.blueprint.annotation.service.Reference;
  40 import org.opendaylight.daexim.DataImportBootReady;
  41 import org.opendaylight.genius.datastoreutils.ExpectedDataObjectNotFoundException;
  42 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  43 import org.opendaylight.genius.idmanager.ReleasedIdHolder.DelayedIdEntry;
  44 import org.opendaylight.genius.idmanager.api.IdManagerMonitor;
  45 import org.opendaylight.genius.idmanager.jobs.CleanUpJob;
  46 import org.opendaylight.genius.idmanager.jobs.IdHolderSyncJob;
  47 import org.opendaylight.genius.idmanager.jobs.LocalPoolCreateJob;
  48 import org.opendaylight.genius.idmanager.jobs.LocalPoolDeleteJob;
  49 import org.opendaylight.genius.idmanager.jobs.UpdateIdEntryJob;
  50 import org.opendaylight.genius.infra.Datastore.Configuration;
  51 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  52 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  53 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  54 import org.opendaylight.genius.infra.TypedWriteTransaction;
  55 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  56 import org.opendaylight.mdsal.binding.api.DataBroker;
  57 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  58 import org.opendaylight.mdsal.common.api.ReadFailedException;
  59 import org.opendaylight.serviceutils.tools.rpc.FutureRpcResults;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInput;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutput;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutputBuilder;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdRangeInput;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdRangeOutput;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdRangeOutputBuilder;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInput;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolOutput;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolOutputBuilder;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.DeleteIdPoolInput;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.DeleteIdPoolOutput;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdPools;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInput;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdOutput;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdOutputBuilder;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.id.pools.IdPool;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.id.pools.IdPoolBuilder;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.id.pools.IdPoolKey;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.id.pools.id.pool.AvailableIdsHolder;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.id.pools.id.pool.AvailableIdsHolderBuilder;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.id.pools.id.pool.ChildPools;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.id.pools.id.pool.IdEntries;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.id.pools.id.pool.ReleasedIdsHolder;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.id.pools.id.pool.ReleasedIdsHolderBuilder;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.released.ids.DelayedIdEntries;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.lockmanager.rev160413.LockManagerService;
  87 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  88 import org.opendaylight.yangtools.yang.common.OperationFailedException;
  89 import org.opendaylight.yangtools.yang.common.RpcResult;
  90 import org.opendaylight.yangtools.yang.common.Uint32;
  91 import org.slf4j.Logger;
  92 import org.slf4j.LoggerFactory;
  93
  94 @Singleton
  95 public class IdManager implements IdManagerService, IdManagerMonitor {
  96
  97     private static final Logger LOG = LoggerFactory.getLogger(IdManager.class);
  98     private static final long DEFAULT_IDLE_TIME = 24 * 60 * 60;
  99
 100     private final DataBroker broker;
 101     private final ManagedNewTransactionRunner txRunner;
 102     private final SingleTransactionDataBroker singleTxDB;
 103     private final LockManagerService lockManager;
 104     private final IdUtils idUtils;
 105     private final JobCoordinator jobCoordinator;
 106
 107     private final ConcurrentMap<String, IdLocalPool> localPool;
 108     private final Timer cleanJobTimer = new Timer();
 109
 110     @Inject
 111     public IdManager(DataBroker db, LockManagerService lockManager, IdUtils idUtils,
 112                      @Reference DataImportBootReady dataImportBootReady, @Reference JobCoordinator jobCoordinator)
 113                     throws ReadFailedException, InterruptedException {
 114         this.broker = db;
 115         this.txRunner = new ManagedNewTransactionRunnerImpl(db);
 116         this.singleTxDB = new SingleTransactionDataBroker(db);
 117         this.lockManager = lockManager;
 118         this.idUtils = idUtils;
 119         this.jobCoordinator = jobCoordinator;
 120
 121         // NB: We do not "use" the DataImportBootReady, but it's presence in the OSGi
 122         // Service Registry is the required "signal" that the Daexim "import on boot"
 123         // has fully completed (which we want to wait for).  Therefore, making this
 124         // dependent on that defers the Blueprint initialization, as we'd like to,
 125         // so that we do not start giving out new IDs before an import went in.
 126         // Thus, please DO NOT remove the DataImportBootReady argument, even if
 127         // it appears to be (is) un-used from a Java code PoV!
 128
 129         this.localPool = new ConcurrentHashMap<>();
 130         populateCache();
 131     }
 132
 133     @Override
 134     public Map<String, String> getLocalPoolsDetails() {
 135         Map<String, String> map = new HashMap<>();
 136         localPool.forEach((key, value) -> map.put(key, value.toString()));
 137         return map;
 138     }
 139
 140     @PostConstruct
 141     public void start() {
 142         LOG.info("{} start", getClass().getSimpleName());
 143     }
 144
 145     @PreDestroy
 146     public void close() {
 147         cleanJobTimer.cancel();
 148
 149         LOG.info("{} close", getClass().getSimpleName());
 150     }
 151
 152     private void populateCache() throws InterruptedException {
 153         // If IP changes during reboot, then there will be orphaned child pools.
 154         InstanceIdentifier<IdPools> idPoolsInstance = idUtils.getIdPools();
 155         Optional<IdPools> idPoolsOptional;
 156         while (true) {
 157             try {
 158                 idPoolsOptional = singleTxDB.syncReadOptional(LogicalDatastoreType.CONFIGURATION, idPoolsInstance);
 159                 break;
 160             } catch (ExecutionException e) {
 161                 LOG.error("Failed to read the id pools due to error. Retrying again...", e);
 162             }
 163             Thread.sleep(2000);
 164         }
 165         if (!idPoolsOptional.isPresent()) {
 166             return;
 167         }
 168         idPoolsOptional.get().nonnullIdPool()
 169                 .stream()
 170                 .filter(idPool -> idPool.getParentPoolName() != null
 171                         && !idPool.getParentPoolName().isEmpty()
 172                         && idUtils.getLocalPoolName(idPool.getParentPoolName())
 173                                 .equals(idPool.getPoolName()))
 174                 .forEach(
 175                     idPool -> updateLocalIdPoolCache(idPool,
 176                         idPool.getParentPoolName()));
 177     }
 178
 179     public boolean updateLocalIdPoolCache(IdPool idPool, String parentPoolName) {
 180         AvailableIdsHolder availableIdsHolder = idPool.getAvailableIdsHolder();
 181         AvailableIdHolder availableIdHolder = new AvailableIdHolder(idUtils, availableIdsHolder.getStart().toJava(),
 182                 availableIdsHolder.getEnd().toJava());
 183         availableIdHolder.setCur(availableIdsHolder.getCursor());
 184         ReleasedIdsHolder releasedIdsHolder = idPool.getReleasedIdsHolder();
 185         ReleasedIdHolder releasedIdHolder = new ReleasedIdHolder(idUtils,
 186                                                         releasedIdsHolder.getDelayedTimeSec().toJava());
 187         releasedIdHolder.setAvailableIdCount(releasedIdsHolder.getAvailableIdCount().toJava());
 188         List<DelayedIdEntry> delayedIdEntryInCache = releasedIdsHolder.nonnullDelayedIdEntries()
 189                 .stream()
 190                 .map(delayedIdEntry -> new DelayedIdEntry(delayedIdEntry
 191                         .getId().toJava(), delayedIdEntry.getReadyTimeSec().toJava()))
 192                 .sorted(comparing(DelayedIdEntry::getReadyTimeSec))
 193                 .collect(toCollection(ArrayList::new));
 194
 195         releasedIdHolder.replaceDelayedEntries(delayedIdEntryInCache);
 196
 197         IdLocalPool idLocalPool = new IdLocalPool(idUtils, idPool.getPoolName());
 198         idLocalPool.setAvailableIds(availableIdHolder);
 199         idLocalPool.setReleasedIds(releasedIdHolder);
 200         localPool.put(parentPoolName, idLocalPool);
 201         if (LOG.isDebugEnabled()) {
 202             LOG.debug("Populating cache for {} with {}", idLocalPool.getPoolName(), idLocalPool);
 203         }
 204         return true;
 205     }
 206
 207     @Override
 208     public ListenableFuture<RpcResult<CreateIdPoolOutput>> createIdPool(CreateIdPoolInput input) {
 209         LOG.info("createIdPool called with input {}", input);
 210         long low = input.getLow().toJava();
 211         long high = input.getHigh().toJava();
 212         long blockSize = idUtils.computeBlockSize(low, high);
 213         return FutureRpcResults.fromListenableFuture(LOG, "createIdPool", input, () -> {
 214             String poolName = input.getPoolName().intern();
 215             try {
 216                 idUtils.lock(lockManager, poolName);
 217                 return Futures.transform(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 218                     IdPool idPool = createGlobalPool(confTx, poolName, low, high, blockSize);
 219                     String localPoolName = idUtils.getLocalPoolName(poolName);
 220                     IdLocalPool idLocalPool = localPool.get(poolName);
 221                     if (idLocalPool == null) {
 222                         createLocalPool(confTx, localPoolName, idPool);
 223                         idUtils.updateChildPool(confTx, idPool.getPoolName(), localPoolName);
 224                     }
 225                 }), unused -> new CreateIdPoolOutputBuilder().build(), MoreExecutors.directExecutor());
 226             } finally {
 227                 idUtils.unlock(lockManager, poolName);
 228             }
 229         }).build();
 230     }
 231
 232     @Override
 233     public ListenableFuture<RpcResult<AllocateIdOutput>> allocateId(AllocateIdInput input) {
 234         String idKey = input.getIdKey();
 235         String poolName = input.getPoolName();
 236         return FutureRpcResults.fromBuilder(LOG, "allocateId", input, () -> {
 237             String localPoolName = idUtils.getLocalPoolName(poolName);
 238             // allocateIdFromLocalPool method returns a list of IDs with one element. This element is obtained by get(0)
 239             long newIdValue = allocateIdFromLocalPool(poolName, localPoolName, idKey, 1).get(0).toJava();
 240             return new AllocateIdOutputBuilder().setIdValue(newIdValue);
 241         }).onFailure(e -> completeExceptionallyIfPresent(poolName, idKey, e)).build();
 242     }
 243
 244     private void completeExceptionallyIfPresent(String poolName, String idKey, Throwable exception) {
 245         CompletableFuture<List<Uint32>> completableFuture =
 246                 idUtils.removeAllocatedIds(idUtils.getUniqueKey(poolName, idKey));
 247         if (completableFuture != null) {
 248             completableFuture.completeExceptionally(exception);
 249         }
 250     }
 251
 252     @Override
 253     public ListenableFuture<RpcResult<AllocateIdRangeOutput>> allocateIdRange(AllocateIdRangeInput input) {
 254         String idKey = input.getIdKey();
 255         String poolName = input.getPoolName();
 256         long size = input.getSize().toJava();
 257         String localPoolName = idUtils.getLocalPoolName(poolName);
 258         AllocateIdRangeOutputBuilder output = new AllocateIdRangeOutputBuilder();
 259         return FutureRpcResults.fromBuilder(LOG, "allocateIdRange", input, () -> {
 260             List<Uint32> newIdValuesList = allocateIdFromLocalPool(poolName, localPoolName, idKey, size);
 261             Collections.sort(newIdValuesList);
 262             output.setIdValues(newIdValuesList);
 263             return output;
 264         }).onFailure(e -> completeExceptionallyIfPresent(poolName, idKey, e)).build();
 265     }
 266
 267     @Override
 268     public ListenableFuture<RpcResult<DeleteIdPoolOutput>> deleteIdPool(DeleteIdPoolInput input) {
 269         return FutureRpcResults.fromListenableFuture(LOG, "deleteIdPool", input, () -> {
 270             String poolName = input.getPoolName().intern();
 271             InstanceIdentifier<IdPool> idPoolToBeDeleted = idUtils.getIdPoolInstance(poolName);
 272             synchronized (poolName) {
 273                 IdPool idPool = singleTxDB.syncRead(LogicalDatastoreType.CONFIGURATION, idPoolToBeDeleted);
 274                 List<ChildPools> childPoolList = idPool.getChildPools();
 275                 if (childPoolList != null) {
 276                     childPoolList.forEach(childPool -> deletePool(childPool.getChildPoolName()));
 277                 }
 278                 singleTxDB.syncDelete(LogicalDatastoreType.CONFIGURATION, idPoolToBeDeleted);
 279             }
 280             // TODO return the Future from a TBD asyncDelete instead.. BUT check that all callers @CheckReturnValue
 281             return Futures.immediateFuture((DeleteIdPoolOutput) null);
 282         }).build();
 283     }
 284
 285     @Override
 286     public ListenableFuture<RpcResult<ReleaseIdOutput>> releaseId(ReleaseIdInput input) {
 287         String poolName = input.getPoolName();
 288         String idKey = input.getIdKey();
 289         String uniqueKey = idUtils.getUniqueKey(poolName, idKey);
 290         return FutureRpcResults.fromBuilder(LOG, "releaseId", input, () -> {
 291             idUtils.lock(lockManager, uniqueKey);
 292             return releaseIdFromLocalPool(poolName, idUtils.getLocalPoolName(poolName), idKey);
 293         }).onFailureLogLevel(FutureRpcResults.LogLevel.NONE)
 294                 .onFailure(e -> {
 295                     if (e instanceof IdDoesNotExistException) {
 296                         // Do not log full stack trace in case ID does not exist
 297                         LOG.error("RPC releaseId() failed due to IdDoesNotExistException; input = {}", input);
 298                     } else {
 299                         // But for all other cases do:
 300                         LOG.error("RPC releaseId() failed; input = {}", input, e);
 301                     }
 302                     idUtils.unlock(lockManager, uniqueKey);
 303                 }).build();
 304     }
 305
 306     private List<Uint32> allocateIdFromLocalPool(String parentPoolName, String localPoolName,
 307             String idKey, long size) throws OperationFailedException, IdManagerException, ExecutionException,
 308             InterruptedException {
 309         LOG.debug("Allocating id from local pool {}. Parent pool {}. Idkey {}", localPoolName, parentPoolName, idKey);
 310         String uniqueIdKey = idUtils.getUniqueKey(parentPoolName, idKey);
 311         CompletableFuture<List<Uint32>> futureIdValues = new CompletableFuture<>();
 312         CompletableFuture<List<Uint32>> existingFutureIdValue =
 313                 idUtils.putAllocatedIdsIfAbsent(uniqueIdKey, futureIdValues);
 314         if (existingFutureIdValue != null) {
 315             try {
 316                 return existingFutureIdValue.get();
 317             } catch (InterruptedException | ExecutionException e) {
 318                 LOG.warn("Could not obtain id from existing futureIdValue for idKey {} and pool {}.",
 319                         idKey, parentPoolName);
 320                 throw new IdManagerException(e.getMessage(), e);
 321             }
 322         }
 323         try {
 324             List<Uint32> newIdValuesList = checkForIdInIdEntries(parentPoolName, idKey, uniqueIdKey, futureIdValues,
 325                     false);
 326             if (!newIdValuesList.isEmpty()) {
 327                 return newIdValuesList;
 328             }
 329             //This get will not help in concurrent reads. Hence the same read needs to be done again.
 330             IdLocalPool localIdPool = getOrCreateLocalIdPool(parentPoolName, localPoolName);
 331             LOG.debug("Got pool {}", localIdPool);
 332             long newIdValue = -1;
 333             localPoolName = localPoolName.intern();
 334             if (size == 1) {
 335                 newIdValue = getIdFromLocalPoolCache(localIdPool, parentPoolName);
 336                 newIdValuesList.add(Uint32.valueOf(newIdValue));
 337             } else {
 338                 getRangeOfIds(parentPoolName, localPoolName, size, newIdValuesList, localIdPool, newIdValue);
 339             }
 340             LOG.debug("The newIdValues {} for the idKey {}", newIdValuesList, idKey);
 341             idUtils.putReleaseIdLatch(uniqueIdKey, new CountDownLatch(1));
 342             UpdateIdEntryJob job = new UpdateIdEntryJob(parentPoolName, localPoolName, idKey, newIdValuesList, txRunner,
 343                     idUtils, lockManager);
 344             jobCoordinator.enqueueJob(parentPoolName, job, IdUtils.RETRY_COUNT);
 345             futureIdValues.complete(newIdValuesList);
 346             return newIdValuesList;
 347         } catch (OperationFailedException | IdManagerException e) {
 348             idUtils.unlock(lockManager, uniqueIdKey);
 349             throw e;
 350         }
 351     }
 352
 353     private Long getIdFromLocalPoolCache(IdLocalPool localIdPool, String parentPoolName)
 354             throws IdManagerException {
 355         while (true) {
 356             IdHolder availableIds = localIdPool.getAvailableIds();
 357             if (availableIds != null) {
 358                 Optional<Long> availableId = availableIds.allocateId();
 359                 if (availableId.isPresent()) {
 360                     IdHolderSyncJob poolSyncJob =
 361                             new IdHolderSyncJob(localIdPool.getPoolName(), localIdPool.getAvailableIds(), txRunner,
 362                                     idUtils);
 363                     jobCoordinator.enqueueJob(localIdPool.getPoolName(), poolSyncJob, IdUtils.RETRY_COUNT);
 364                     return availableId.get();
 365                 }
 366             }
 367             IdHolder releasedIds = localIdPool.getReleasedIds();
 368             Optional<Long> releasedId = releasedIds.allocateId();
 369             if (releasedId.isPresent()) {
 370                 IdHolderSyncJob poolSyncJob =
 371                         new IdHolderSyncJob(localIdPool.getPoolName(), localIdPool.getReleasedIds(), txRunner,
 372                                 idUtils);
 373                 jobCoordinator.enqueueJob(localIdPool.getPoolName(), poolSyncJob, IdUtils.RETRY_COUNT);
 374                 return releasedId.get();
 375             }
 376             long idCount = getIdBlockFromParentPool(parentPoolName, localIdPool);
 377             if (idCount <= 0) {
 378                 if (LOG.isDebugEnabled()) {
 379                     LOG.debug("Unable to allocate Id block from global pool {}", parentPoolName);
 380                 }
 381                 throw new IdManagerException(String.format("Ids exhausted for pool : %s", parentPoolName));
 382             }
 383         }
 384     }
 385
 386     /**
 387      * Changes made to availableIds and releasedIds will not be persisted to the datastore.
 388      */
 389     private long getIdBlockFromParentPool(String parentPoolName, IdLocalPool localIdPool)
 390             throws IdManagerException {
 391         if (LOG.isDebugEnabled()) {
 392             LOG.debug("Allocating block of id from parent pool {}", parentPoolName);
 393         }
 394         InstanceIdentifier<IdPool> idPoolInstanceIdentifier = idUtils.getIdPoolInstance(parentPoolName);
 395         parentPoolName = parentPoolName.intern();
 396         idUtils.lock(lockManager, parentPoolName);
 397         try {
 398             // Check if the childpool already got id block.
 399             long availableIdCount =
 400                     localIdPool.getAvailableIds().getAvailableIdCount()
 401                             + localIdPool.getReleasedIds().getAvailableIdCount();
 402             if (availableIdCount > 0) {
 403                 return availableIdCount;
 404             }
 405             return txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 406                 Optional<IdPool> parentIdPool = confTx.read(idPoolInstanceIdentifier).get();
 407                 if (parentIdPool.isPresent()) {
 408                     return allocateIdBlockFromParentPool(localIdPool, parentIdPool.get(), confTx);
 409                 } else {
 410                     throw new ExpectedDataObjectNotFoundException(LogicalDatastoreType.CONFIGURATION,
 411                             idPoolInstanceIdentifier);
 412                 }
 413             }).get();
 414         } catch (InterruptedException | ExecutionException e) {
 415             throw new IdManagerException("Error getting id block from parent pool", e);
 416         } finally {
 417             idUtils.unlock(lockManager, parentPoolName);
 418         }
 419     }
 420
 421     private long allocateIdBlockFromParentPool(IdLocalPool localPoolCache, IdPool parentIdPool,
 422             TypedWriteTransaction<Configuration> confTx)
 423             throws OperationFailedException, IdManagerException {
 424
 425         long idCount = allocateIdBlockFromAvailableIdsHolder(localPoolCache, parentIdPool, confTx);
 426         if (idCount > 0) {
 427             return idCount;
 428         }
 429
 430         ReleasedIdsHolderBuilder releasedIdsBuilderParent = IdUtils.getReleaseIdsHolderBuilder(parentIdPool);
 431         final List<DelayedIdEntries> delayedEntries = releasedIdsBuilderParent.getDelayedIdEntries();
 432         if (delayedEntries != null) {
 433             releasedIdsBuilderParent.setDelayedIdEntries(new ArrayList<>(delayedEntries));
 434         }
 435
 436         while (true) {
 437             idCount = allocateIdBlockFromReleasedIdsHolder(localPoolCache, releasedIdsBuilderParent, parentIdPool,
 438                     confTx);
 439             if (idCount > 0) {
 440                 return idCount;
 441             }
 442             idCount = getIdsFromOtherChildPools(releasedIdsBuilderParent, parentIdPool);
 443             if (idCount <= 0) {
 444                 if (LOG.isDebugEnabled()) {
 445                     LOG.debug("Unable to allocate Id block from global pool");
 446                 }
 447                 throw new IdManagerException(String.format("Ids exhausted for pool : %s", parentIdPool.getPoolName()));
 448             }
 449
 450             idCount = allocateIdBlockFromAvailableIdsHolder(localPoolCache, parentIdPool, confTx);
 451             if (idCount > 0) {
 452                 return idCount;
 453             }
 454         }
 455     }
 456
 457     private long getIdsFromOtherChildPools(ReleasedIdsHolderBuilder releasedIdsBuilderParent, IdPool parentIdPool)
 458             throws OperationFailedException {
 459         List<ChildPools> childPoolsList = parentIdPool.nonnullChildPools();
 460         // Sorting the child pools on last accessed time so that the pool that
 461         // was not accessed for a long time comes first.
 462         childPoolsList.sort(comparing(ChildPools::getLastAccessTime));
 463         long currentTime = System.currentTimeMillis() / 1000;
 464         for (ChildPools childPools : childPoolsList) {
 465             if (childPools.getLastAccessTime().toJava() + DEFAULT_IDLE_TIME > currentTime) {
 466                 break;
 467             }
 468             if (!Objects.equals(childPools.getChildPoolName(), idUtils.getLocalPoolName(parentIdPool.getPoolName()))) {
 469                 InstanceIdentifier<IdPool> idPoolInstanceIdentifier = idUtils
 470                         .getIdPoolInstance(childPools.getChildPoolName());
 471                 IdPool otherChildPool =
 472                         singleTxDB.syncRead(LogicalDatastoreType.CONFIGURATION, idPoolInstanceIdentifier);
 473                 ReleasedIdsHolderBuilder releasedIds = IdUtils.getReleaseIdsHolderBuilder(otherChildPool);
 474
 475                 List<DelayedIdEntries> delayedIdEntriesChild = releasedIds.getDelayedIdEntries();
 476                 List<DelayedIdEntries> delayedIdEntriesParent = releasedIdsBuilderParent.getDelayedIdEntries();
 477                 if (delayedIdEntriesParent == null) {
 478                     delayedIdEntriesParent = new LinkedList<>();
 479                 }
 480                 delayedIdEntriesParent.addAll(delayedIdEntriesChild);
 481                 delayedIdEntriesChild.clear();
 482
 483                 AvailableIdsHolderBuilder availableIds = idUtils.getAvailableIdsHolderBuilder(otherChildPool);
 484                 while (idUtils.isIdAvailable(availableIds)) {
 485                     long cursor = availableIds.getCursor() + 1;
 486                     delayedIdEntriesParent.add(idUtils.createDelayedIdEntry(cursor, currentTime));
 487                     availableIds.setCursor(cursor);
 488                 }
 489
 490                 long totalAvailableIdCount = releasedIds.getDelayedIdEntries().size()
 491                         + idUtils.getAvailableIdsCount(availableIds);
 492                 long count = releasedIdsBuilderParent.getAvailableIdCount().toJava() + totalAvailableIdCount;
 493                 releasedIdsBuilderParent.setDelayedIdEntries(delayedIdEntriesParent).setAvailableIdCount(count);
 494                 singleTxDB.syncUpdate(LogicalDatastoreType.CONFIGURATION, idPoolInstanceIdentifier,
 495                         new IdPoolBuilder().withKey(new IdPoolKey(otherChildPool.getPoolName()))
 496                                 .setAvailableIdsHolder(availableIds.build()).setReleasedIdsHolder(releasedIds.build())
 497                                 .build());
 498                 return totalAvailableIdCount;
 499             }
 500         }
 501         return 0;
 502     }
 503
 504     private long allocateIdBlockFromReleasedIdsHolder(IdLocalPool localIdPool,
 505             ReleasedIdsHolderBuilder releasedIdsBuilderParent, IdPool parentIdPool,
 506             TypedWriteTransaction<Configuration> confTx) {
 507         if (releasedIdsBuilderParent.getAvailableIdCount().toJava() == 0) {
 508             LOG.debug("Ids unavailable in releasedIds of parent pool {}", parentIdPool);
 509             return 0;
 510         }
 511         List<DelayedIdEntries> delayedIdEntriesParent = releasedIdsBuilderParent.getDelayedIdEntries();
 512         int idCount = Math.min(delayedIdEntriesParent.size(), parentIdPool.getBlockSize().toJava());
 513         List<DelayedIdEntries> idEntriesToBeRemoved = delayedIdEntriesParent.subList(0, idCount);
 514         ReleasedIdHolder releasedIds = (ReleasedIdHolder) localIdPool.getReleasedIds();
 515         List<DelayedIdEntry> delayedIdEntriesLocalCache = releasedIds.getDelayedEntries();
 516         List<DelayedIdEntry> delayedIdEntriesFromParentPool = idEntriesToBeRemoved
 517                 .stream()
 518                 .map(delayedIdEntry -> new DelayedIdEntry(delayedIdEntry
 519                         .getId().toJava(), delayedIdEntry.getReadyTimeSec().toJava()))
 520                 .sorted(comparing(DelayedIdEntry::getReadyTimeSec))
 521                 .collect(toCollection(ArrayList::new));
 522         delayedIdEntriesFromParentPool.addAll(delayedIdEntriesLocalCache);
 523         releasedIds.replaceDelayedEntries(delayedIdEntriesFromParentPool);
 524         releasedIds.setAvailableIdCount(releasedIds.getAvailableIdCount() + idCount);
 525         localIdPool.setReleasedIds(releasedIds);
 526         delayedIdEntriesParent.removeAll(idEntriesToBeRemoved);
 527         releasedIdsBuilderParent.setDelayedIdEntries(delayedIdEntriesParent);
 528         InstanceIdentifier<ReleasedIdsHolder> releasedIdsHolderInstanceIdentifier = InstanceIdentifier
 529                 .builder(IdPools.class).child(IdPool.class,
 530                         new IdPoolKey(parentIdPool.getPoolName())).child(ReleasedIdsHolder.class).build();
 531         releasedIdsBuilderParent.setAvailableIdCount(releasedIdsBuilderParent.getAvailableIdCount().toJava() - idCount);
 532         LOG.debug("Allocated {} ids from releasedIds of parent pool {}", idCount, parentIdPool);
 533         confTx.merge(releasedIdsHolderInstanceIdentifier, releasedIdsBuilderParent.build(), CREATE_MISSING_PARENTS);
 534         return idCount;
 535     }
 536
 537     private long allocateIdBlockFromAvailableIdsHolder(IdLocalPool localIdPool, IdPool parentIdPool,
 538             TypedWriteTransaction<Configuration> confTx) {
 539         long idCount = 0;
 540         AvailableIdsHolderBuilder availableIdsBuilderParent = idUtils.getAvailableIdsHolderBuilder(parentIdPool);
 541         long end = availableIdsBuilderParent.getEnd().toJava();
 542         long cur = availableIdsBuilderParent.getCursor();
 543         if (!idUtils.isIdAvailable(availableIdsBuilderParent)) {
 544             if (LOG.isDebugEnabled()) {
 545                 LOG.debug("Ids exhausted in parent pool {}", parentIdPool);
 546             }
 547             return idCount;
 548         }
 549         // Update availableIdsHolder of Local Pool
 550         idCount = Math.min(end - cur, parentIdPool.getBlockSize().toJava());
 551         AvailableIdHolder availableIds = new AvailableIdHolder(idUtils, cur + 1, cur + idCount);
 552         localIdPool.setAvailableIds(availableIds);
 553         // Update availableIdsHolder of Global Pool
 554         InstanceIdentifier<AvailableIdsHolder> availableIdsHolderInstanceIdentifier = InstanceIdentifier
 555                 .builder(IdPools.class).child(IdPool.class,
 556                         new IdPoolKey(parentIdPool.getPoolName())).child(AvailableIdsHolder.class).build();
 557         availableIdsBuilderParent.setCursor(cur + idCount);
 558         if (LOG.isDebugEnabled()) {
 559             LOG.debug("Allocated {} ids from availableIds of global pool {}", idCount, parentIdPool);
 560         }
 561         confTx.merge(availableIdsHolderInstanceIdentifier, availableIdsBuilderParent.build(), CREATE_MISSING_PARENTS);
 562         return idCount;
 563     }
 564
 565     private ReleaseIdOutputBuilder releaseIdFromLocalPool(String parentPoolName, String localPoolName, String idKey)
 566             throws IdManagerException, ReadFailedException, ExecutionException, InterruptedException {
 567         String idLatchKey = idUtils.getUniqueKey(parentPoolName, idKey);
 568         LOG.debug("Releasing ID {} from pool {}", idKey, localPoolName);
 569         CountDownLatch latch = idUtils.getReleaseIdLatch(idLatchKey);
 570         if (latch != null) {
 571             try {
 572                 if (!latch.await(10, TimeUnit.SECONDS)) {
 573                     LOG.warn("Timed out while releasing id {} from id pool {}", idKey, parentPoolName);
 574                 }
 575             } catch (InterruptedException ignored) {
 576                 LOG.warn("Thread interrupted while releasing id {} from id pool {}", idKey, parentPoolName);
 577             } finally {
 578                 idUtils.removeReleaseIdLatch(idLatchKey);
 579             }
 580         }
 581         localPoolName = localPoolName.intern();
 582         InstanceIdentifier<IdPool> parentIdPoolInstanceIdentifier = idUtils.getIdPoolInstance(parentPoolName);
 583         IdPool parentIdPool = singleTxDB.syncRead(LogicalDatastoreType.CONFIGURATION, parentIdPoolInstanceIdentifier);
 584         List<IdEntries> idEntries = parentIdPool.getIdEntries();
 585         if (idEntries == null) {
 586             throw new IdDoesNotExistException(parentPoolName, idKey);
 587         }
 588         InstanceIdentifier<IdEntries> existingId = idUtils.getIdEntry(parentIdPoolInstanceIdentifier, idKey);
 589         Optional<IdEntries> existingIdEntryObject =
 590                 singleTxDB.syncReadOptional(LogicalDatastoreType.CONFIGURATION, existingId);
 591         if (!existingIdEntryObject.isPresent()) {
 592             LOG.info("Specified Id key {} does not exist in id pool {}", idKey, parentPoolName);
 593             idUtils.unlock(lockManager, idLatchKey);
 594             throw new IdManagerException(String.format("Specified Id key %s does not exist in id pool %s",
 595                     idKey, parentPoolName));
 596         }
 597         IdEntries existingIdEntry = existingIdEntryObject.get();
 598         List<Uint32> idValuesList = nonnull(existingIdEntry.getIdValue());
 599         IdLocalPool localIdPoolCache = localPool.get(parentPoolName);
 600         boolean isRemoved = idEntries.contains(existingIdEntry);
 601         LOG.debug("The entry {} is removed {}", existingIdEntry, isRemoved);
 602         updateDelayedEntriesInLocalCache(idValuesList, parentPoolName, localIdPoolCache);
 603         IdHolderSyncJob poolSyncJob = new IdHolderSyncJob(localPoolName, localIdPoolCache.getReleasedIds(), txRunner,
 604                 idUtils);
 605         jobCoordinator.enqueueJob(localPoolName, poolSyncJob, IdUtils.RETRY_COUNT);
 606         scheduleCleanUpTask(localIdPoolCache, parentPoolName, parentIdPool.getBlockSize().toJava());
 607         LOG.debug("Released id ({}, {}) from pool {}", idKey, idValuesList, localPoolName);
 608         // Updating id entries in the parent pool. This will be used for restart scenario
 609         UpdateIdEntryJob job = new UpdateIdEntryJob(parentPoolName, localPoolName, idKey, null, txRunner, idUtils,
 610                         lockManager);
 611         jobCoordinator.enqueueJob(parentPoolName, job, IdUtils.RETRY_COUNT);
 612         return new ReleaseIdOutputBuilder().setIdValues(idValuesList);
 613     }
 614
 615     private void scheduleCleanUpTask(final IdLocalPool localIdPoolCache,
 616             final String parentPoolName, final int blockSize) {
 617         TimerTask scheduledTask = new TimerTask() {
 618             @Override
 619             public void run() {
 620                 CleanUpJob job =
 621                         new CleanUpJob(localIdPoolCache, txRunner, broker, parentPoolName, blockSize, lockManager,
 622                                 idUtils, jobCoordinator);
 623                 jobCoordinator.enqueueJob(localIdPoolCache.getPoolName(), job, IdUtils.RETRY_COUNT);
 624             }
 625         };
 626         cleanJobTimer.schedule(scheduledTask, IdUtils.DEFAULT_DELAY_TIME * 1000);
 627     }
 628
 629     private IdPool createGlobalPool(TypedReadWriteTransaction<Configuration> confTx, String poolName, long low,
 630             long high, long blockSize) throws IdManagerException {
 631         IdPool idPool;
 632         InstanceIdentifier<IdPool> idPoolInstanceIdentifier = idUtils.getIdPoolInstance(poolName);
 633         try {
 634             Optional<IdPool> existingIdPool = confTx.read(idPoolInstanceIdentifier).get();
 635             if (!existingIdPool.isPresent()) {
 636                 if (LOG.isDebugEnabled()) {
 637                     LOG.debug("Creating new global pool {}", poolName);
 638                 }
 639                 idPool = idUtils.createGlobalPool(poolName, low, high, blockSize);
 640                 confTx.put(idPoolInstanceIdentifier, idPool, CREATE_MISSING_PARENTS);
 641             } else {
 642                 idPool = existingIdPool.get();
 643                 if (LOG.isDebugEnabled()) {
 644                     LOG.debug("GlobalPool exists {}", idPool);
 645                 }
 646             }
 647             return idPool;
 648         } catch (ExecutionException | InterruptedException e) {
 649             throw new IdManagerException("Error retrieving the existing id pool for " + poolName, e);
 650         }
 651     }
 652
 653     private IdLocalPool createLocalPool(TypedWriteTransaction<Configuration> confTx, String localPoolName,
 654             IdPool idPool)
 655             throws OperationFailedException, IdManagerException {
 656         localPoolName = localPoolName.intern();
 657         IdLocalPool idLocalPool = new IdLocalPool(idUtils, localPoolName);
 658         allocateIdBlockFromParentPool(idLocalPool, idPool, confTx);
 659         String parentPool = idPool.getPoolName();
 660         localPool.put(parentPool, idLocalPool);
 661         LocalPoolCreateJob job = new LocalPoolCreateJob(idLocalPool, txRunner, idPool.getPoolName(),
 662                 idPool.getBlockSize().toJava(), idUtils);
 663         jobCoordinator.enqueueJob(localPoolName, job, IdUtils.RETRY_COUNT);
 664         return idLocalPool;
 665     }
 666
 667     private void deletePool(String poolName) {
 668         LocalPoolDeleteJob job = new LocalPoolDeleteJob(poolName, txRunner, idUtils);
 669         jobCoordinator.enqueueJob(poolName, job, IdUtils.RETRY_COUNT);
 670     }
 671
 672     public void poolDeleted(String parentPoolName, String poolName) {
 673         IdLocalPool idLocalPool = localPool.get(parentPoolName);
 674         if (idLocalPool != null) {
 675             if (idLocalPool.getPoolName().equals(poolName)) {
 676                 localPool.remove(parentPoolName);
 677             }
 678         }
 679     }
 680
 681     private void updateDelayedEntriesInLocalCache(List<Uint32> idsList, String parentPoolName,
 682             IdLocalPool localPoolCache) {
 683         for (Uint32 idValue : idsList) {
 684             localPoolCache.getReleasedIds().addId(idValue.toJava());
 685         }
 686         localPool.put(parentPoolName, localPoolCache);
 687     }
 688
 689     private List<Uint32> checkForIdInIdEntries(String parentPoolName, String idKey, String uniqueIdKey,
 690             CompletableFuture<List<Uint32>> futureIdValues, boolean hasExistingFutureIdValues)
 691             throws IdManagerException, InterruptedException , ExecutionException {
 692         InstanceIdentifier<IdPool> parentIdPoolInstanceIdentifier = idUtils.getIdPoolInstance(parentPoolName);
 693         InstanceIdentifier<IdEntries> existingId = idUtils.getIdEntry(parentIdPoolInstanceIdentifier, idKey);
 694         idUtils.lock(lockManager, uniqueIdKey);
 695         List<Uint32> newIdValuesList = new ArrayList<>();
 696         Optional<IdEntries> existingIdEntry =
 697                 singleTxDB.syncReadOptional(LogicalDatastoreType.CONFIGURATION, existingId);
 698         if (existingIdEntry.isPresent()) {
 699             newIdValuesList = existingIdEntry.get().getIdValue();
 700             LOG.debug("Existing ids {} for the key {} ", newIdValuesList, idKey);
 701             // Inform other waiting threads about this new value.
 702             futureIdValues.complete(newIdValuesList);
 703             // This is to avoid stale entries in the map. If this thread had populated the map,
 704             // then the entry should be removed.
 705             if (!hasExistingFutureIdValues) {
 706                 idUtils.removeAllocatedIds(uniqueIdKey);
 707             }
 708             idUtils.unlock(lockManager, uniqueIdKey);
 709             return newIdValuesList;
 710         }
 711         return newIdValuesList;
 712     }
 713
 714     private IdLocalPool getOrCreateLocalIdPool(String parentPoolName, String localPoolName)
 715         throws IdManagerException {
 716         IdLocalPool localIdPool = localPool.get(parentPoolName);
 717         if (localIdPool == null) {
 718             idUtils.lock(lockManager, parentPoolName);
 719             try {
 720                 // Check if a previous thread that got the cluster-wide lock
 721                 // first, has created the localPool
 722                 try {
 723                     InstanceIdentifier<IdPool> childIdPoolInstanceIdentifier = idUtils
 724                             .getIdPoolInstance(localPoolName);
 725                     Optional<IdPool> childIdPoolOpt = singleTxDB.syncReadOptional(LogicalDatastoreType.CONFIGURATION,
 726                             childIdPoolInstanceIdentifier);
 727                     if (childIdPoolOpt.isPresent()) {
 728                         updateLocalIdPoolCache(childIdPoolOpt.get(), parentPoolName);
 729                     }
 730                 }
 731                 catch (ExecutionException | InterruptedException ex) {
 732                     LOG.debug("Failed to read id pool {} due to {}", localPoolName, ex.getMessage());
 733                 }
 734                 if (localPool.get(parentPoolName) == null) {
 735                     try {
 736                         return txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
 737                             InstanceIdentifier<IdPool> parentIdPoolInstanceIdentifier = idUtils
 738                                     .getIdPoolInstance(parentPoolName);
 739                             Optional<IdPool> parentIdPool = confTx.read(parentIdPoolInstanceIdentifier).get();
 740                             if (parentIdPool.isPresent()) {
 741                                 // Return localIdPool
 742                                 return createLocalPool(confTx, localPoolName, parentIdPool.get());
 743                             } else {
 744                                 throw new ExpectedDataObjectNotFoundException(LogicalDatastoreType.CONFIGURATION,
 745                                         parentIdPoolInstanceIdentifier);
 746                             }
 747                         }).get();
 748                     } catch (InterruptedException | ExecutionException e) {
 749                         throw new IdManagerException("Error creating a local id pool", e);
 750                     }
 751                 } else {
 752                     localIdPool = localPool.get(parentPoolName);
 753                 }
 754             } finally {
 755                 idUtils.unlock(lockManager, parentPoolName);
 756             }
 757         }
 758         return localIdPool;
 759     }
 760
 761     private void getRangeOfIds(String parentPoolName, String localPoolName, long size, List<Uint32> newIdValuesList,
 762             IdLocalPool localIdPool, long newIdValue) throws ReadFailedException, IdManagerException {
 763         InstanceIdentifier<IdPool> parentIdPoolInstanceIdentifier1 = idUtils.getIdPoolInstance(parentPoolName);
 764         IdPool parentIdPool = singleTxDB.syncRead(LogicalDatastoreType.CONFIGURATION, parentIdPoolInstanceIdentifier1);
 765         long totalAvailableIdCount = localIdPool.getAvailableIds().getAvailableIdCount()
 766                 + localIdPool.getReleasedIds().getAvailableIdCount();
 767         AvailableIdsHolderBuilder availableParentIds = idUtils.getAvailableIdsHolderBuilder(parentIdPool);
 768         ReleasedIdsHolderBuilder releasedParentIds = IdUtils.getReleaseIdsHolderBuilder(parentIdPool);
 769         totalAvailableIdCount = totalAvailableIdCount + releasedParentIds.getAvailableIdCount().toJava()
 770                 + idUtils.getAvailableIdsCount(availableParentIds);
 771         if (totalAvailableIdCount > size) {
 772             while (size > 0) {
 773                 try {
 774                     newIdValue = getIdFromLocalPoolCache(localIdPool, parentPoolName);
 775                 } catch (IdManagerException e) {
 776                     if (LOG.isDebugEnabled()) {
 777                         LOG.debug("Releasing IDs to pool {}", localPoolName);
 778                     }
 779                     // Releasing the IDs added in newIdValuesList since
 780                     // a null list would be returned now, as the
 781                     // requested size of list IDs exceeds the number of
 782                     // available IDs.
 783                     updateDelayedEntriesInLocalCache(newIdValuesList, parentPoolName, localIdPool);
 784                 }
 785                 newIdValuesList.add(Uint32.valueOf(newIdValue));
 786                 size--;
 787             }
 788         } else {
 789             throw new IdManagerException(String.format("Ids exhausted for pool : %s", parentPoolName));
 790         }
 791     }
 792 }