natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/ha/WeightedCentralizedSwitchScheduler.java

   1 /*
   2  * Copyright (c) 2017 Red Hat, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.netvirt.natservice.ha;
  10
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13
  14 import java.util.ArrayList;
  15 import java.util.HashMap;
  16 import java.util.Iterator;
  17 import java.util.List;
  18 import java.util.Map;
  19 import java.util.Map.Entry;
  20 import java.util.Optional;
  21 import java.util.concurrent.ConcurrentHashMap;
  22 import java.util.concurrent.ExecutionException;
  23 import javax.inject.Inject;
  24 import javax.inject.Singleton;
  25 import org.eclipse.jdt.annotation.Nullable;
  26 import org.opendaylight.genius.datastoreutils.ExpectedDataObjectNotFoundException;
  27 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  28 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  29 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  30 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  31 import org.opendaylight.mdsal.binding.api.DataBroker;
  32 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  33 import org.opendaylight.mdsal.common.api.TransactionCommitFailedException;
  34 import org.opendaylight.netvirt.natservice.api.CentralizedSwitchScheduler;
  35 import org.opendaylight.netvirt.natservice.api.NatSwitchCache;
  36 import org.opendaylight.netvirt.natservice.api.NatSwitchCacheListener;
  37 import org.opendaylight.netvirt.natservice.api.SwitchInfo;
  38 import org.opendaylight.netvirt.natservice.internal.NatUtil;
  39 import org.opendaylight.netvirt.vpnmanager.api.IVpnFootprintService;
  40 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExtRouters;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.NaptSwitches;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.routers.ExternalIps;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.routers.ExternalIpsKey;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchBuilder;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
  55 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  56 import org.opendaylight.yangtools.yang.common.Uint64;
  57 import org.slf4j.Logger;
  58 import org.slf4j.LoggerFactory;
  59
  60 @Singleton
  61 public class WeightedCentralizedSwitchScheduler implements CentralizedSwitchScheduler, NatSwitchCacheListener {
  62     private static final Logger LOG = LoggerFactory.getLogger(WeightedCentralizedSwitchScheduler.class);
  63     private static final Integer INITIAL_SWITCH_WEIGHT = Integer.valueOf(0);
  64
  65     private final Map<String, Map<Uint64,Integer>> providerSwitchWeightsMap = new ConcurrentHashMap<>();
  66     private final Map<String,String> subnetIdToRouterPortMap = new ConcurrentHashMap<>();
  67     private final Map<String,String> subnetIdToElanInstanceMap = new ConcurrentHashMap<>();
  68     private final DataBroker dataBroker;
  69     private final ManagedNewTransactionRunner txRunner;
  70     private final OdlInterfaceRpcService interfaceManager;
  71     private final IVpnFootprintService vpnFootprintService;
  72     private final NatserviceConfig.NatMode natMode;
  73
  74     @Inject
  75     public WeightedCentralizedSwitchScheduler(final DataBroker dataBroker,
  76             final OdlInterfaceRpcService interfaceManager,
  77             final IVpnFootprintService vpnFootprintService, final NatserviceConfig config,
  78             final NatSwitchCache natSwitchCache) {
  79         this.dataBroker = dataBroker;
  80         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
  81         this.interfaceManager = interfaceManager;
  82         this.vpnFootprintService = vpnFootprintService;
  83         if (config != null) {
  84             this.natMode = config.getNatMode();
  85         } else {
  86             this.natMode = NatserviceConfig.NatMode.Controller;
  87         }
  88         natSwitchCache.register(this);
  89     }
  90
  91     @Override
  92     public boolean scheduleCentralizedSwitch(Routers router) {
  93         String providerNet = NatUtil.getElanInstancePhysicalNetwok(router.getNetworkId().getValue(),dataBroker);
  94         Uint64 nextSwitchId = getSwitchWithLowestWeight(providerNet);
  95         if (Uint64.ZERO.equals(nextSwitchId)) {
  96             LOG.error("In scheduleCentralizedSwitch, unable to schedule the router {} as there is no available switch.",
  97                     router.getRouterName());
  98             return false;
  99         }
 100
 101         LOG.info("scheduleCentralizedSwitch for router {} on switch {}", router.getRouterName(), nextSwitchId);
 102         String routerName = router.getRouterName();
 103         RouterToNaptSwitchBuilder routerToNaptSwitchBuilder =
 104                 new RouterToNaptSwitchBuilder().setRouterName(routerName);
 105         RouterToNaptSwitch id = routerToNaptSwitchBuilder.setPrimarySwitchId(nextSwitchId)
 106                 .setEnableSnat(router.isEnableSnat()).build();
 107         addToDpnMaps(routerName, router.getSubnetIds(), nextSwitchId);
 108         try {
 109             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
 110                     getNaptSwitchesIdentifier(routerName), id);
 111             Map<Uint64,Integer> switchWeightMap = providerSwitchWeightsMap.get(providerNet);
 112             switchWeightMap.put(nextSwitchId,switchWeightMap.get(nextSwitchId) + 1);
 113
 114         } catch (TransactionCommitFailedException e) {
 115             LOG.error("ScheduleCentralizedSwitch failed for {}", routerName);
 116         }
 117         return true;
 118
 119     }
 120
 121     @Override
 122     public boolean updateCentralizedSwitch(Routers oldRouter, Routers newRouter) {
 123         LOG.info("updateCentralizedSwitch for router {}", newRouter.getRouterName());
 124         String routerName = newRouter.getRouterName();
 125         List<Uuid> addedSubnetIds = getUpdatedSubnetIds(newRouter.getSubnetIds(), oldRouter.getSubnetIds());
 126         List<Uuid> deletedSubnetIds = getUpdatedSubnetIds(oldRouter.getSubnetIds(), newRouter.getSubnetIds());
 127         Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, newRouter.getRouterName());
 128         addToDpnMaps(routerName, addedSubnetIds, primarySwitchId);
 129         deleteFromDpnMaps(routerName, deletedSubnetIds, primarySwitchId);
 130         try {
 131             InstanceIdentifier<RouterToNaptSwitch> id  = NatUtil.buildNaptSwitchIdentifier(routerName);
 132             RouterToNaptSwitch routerToNaptSwitch = SingleTransactionDataBroker.syncRead(dataBroker,
 133                     LogicalDatastoreType.CONFIGURATION, id);
 134             boolean isSnatEnabled = newRouter.isEnableSnat();
 135             Map<ExternalIpsKey, ExternalIps> updateExternalIpsMap = newRouter.getExternalIps();
 136             if (updateExternalIpsMap == null || updateExternalIpsMap.isEmpty()) {
 137                 isSnatEnabled = false;
 138             }
 139             if (isSnatEnabled != routerToNaptSwitch.isEnableSnat()) {
 140                 RouterToNaptSwitchBuilder routerToNaptSwitchBuilder =
 141                         new RouterToNaptSwitchBuilder(routerToNaptSwitch);
 142                 routerToNaptSwitchBuilder.setEnableSnat(isSnatEnabled);
 143                 SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
 144                         getNaptSwitchesIdentifier(routerName), routerToNaptSwitchBuilder.build());
 145             }
 146         } catch (ExpectedDataObjectNotFoundException e) {
 147             LOG.error("updateCentralizedSwitch ReadFailedException for {}", routerName);
 148         } catch (TransactionCommitFailedException e) {
 149             LOG.error("updateCentralizedSwitch TransactionCommitFailedException for {}", routerName);
 150         }
 151         return true;
 152     }
 153
 154     @Override
 155     public boolean releaseCentralizedSwitch(Routers router) {
 156         String providerNet = NatUtil.getElanInstancePhysicalNetwok(router.getNetworkId().getValue(),dataBroker);
 157         String routerName = router.getRouterName();
 158         Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
 159         if (primarySwitchId == null || Uint64.ZERO.equals(primarySwitchId)) {
 160             LOG.info("releaseCentralizedSwitch: NAPT Switch is not allocated for router {}", router.getRouterName());
 161             return false;
 162         }
 163
 164         LOG.info("releaseCentralizedSwitch for router {} from switch {}", router.getRouterName(), primarySwitchId);
 165         deleteFromDpnMaps(routerName, router.getSubnetIds(), primarySwitchId);
 166         try {
 167             SingleTransactionDataBroker.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION,
 168                     getNaptSwitchesIdentifier(routerName));
 169             Map<Uint64,Integer> switchWeightMap = providerSwitchWeightsMap.get(providerNet);
 170             switchWeightMap.put(primarySwitchId, switchWeightMap.get(primarySwitchId) - 1);
 171         } catch (TransactionCommitFailedException e) {
 172             return false;
 173         }
 174         return true;
 175     }
 176
 177     private void addToDpnMaps(String routerName, List<Uuid> addedSubnetIds, Uint64 primarySwitchId) {
 178         if (addedSubnetIds == null || addedSubnetIds.isEmpty()) {
 179             LOG.debug("addToDpnMaps no subnets associated with {}", routerName);
 180             return;
 181         }
 182         Map<Uuid, Subnetmap> subnetMapEntries = new HashMap<>();
 183         try {
 184             String primaryRd = txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 185                 for (Uuid subnetUuid : addedSubnetIds) {
 186                     Subnetmap subnetMapEntry = tx.read(getSubnetMapIdentifier(subnetUuid)).get().orElse(null);
 187                     subnetMapEntries.put(subnetUuid, subnetMapEntry);
 188                     Uuid routerPortUuid = subnetMapEntry.getRouterInterfacePortId();
 189                     subnetIdToRouterPortMap.put(subnetUuid.getValue(), routerPortUuid.getValue());
 190                 }
 191                 return NatUtil.getPrimaryRd(routerName, tx);
 192             }).get();
 193             ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 194                 for (Uuid subnetUuid : addedSubnetIds) {
 195                     Subnetmap subnetMapEntry = subnetMapEntries.get(subnetUuid);
 196                     Uuid routerPortUuid = subnetMapEntry.getRouterInterfacePortId();
 197                     vpnFootprintService.updateVpnToDpnMapping(primarySwitchId, routerName, primaryRd,
 198                             routerPortUuid.getValue(), null, true);
 199                     NatUtil.addToNeutronRouterDpnsMap(routerName, routerPortUuid.getValue(), primarySwitchId, tx);
 200                     NatUtil.addToDpnRoutersMap(routerName, routerPortUuid.getValue(), primarySwitchId, tx);
 201                     if (subnetMapEntry.getNetworkType().equals(NetworkAttributes.NetworkType.VLAN)) {
 202                         String elanInstanceName = subnetMapEntry.getNetworkId().getValue();
 203                         subnetIdToElanInstanceMap.put(subnetUuid.getValue(), elanInstanceName);
 204                         NatUtil.addPseudoPortToElanDpn(elanInstanceName, elanInstanceName, primarySwitchId, dataBroker);
 205                     }
 206                 }
 207             }), LOG, "Error adding subnets to DPN maps for {}", routerName);
 208         } catch (InterruptedException | ExecutionException e) {
 209             LOG.error("Error adding subnets to DPN maps for {}", routerName);
 210         }
 211     }
 212
 213     private void deleteFromDpnMaps(String routerName, List<Uuid> deletedSubnetIds, Uint64 primarySwitchId) {
 214         if (deletedSubnetIds == null || deletedSubnetIds.isEmpty()) {
 215             LOG.debug("deleteFromDpnMaps no subnets associated with {}", routerName);
 216             return;
 217         }
 218         try {
 219             String primaryRd = txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 220                 tx -> NatUtil.getPrimaryRd(routerName, tx)).get();
 221             ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 222                 for (Uuid subnetUuid : deletedSubnetIds) {
 223                     String routerPort = subnetIdToRouterPortMap.remove(subnetUuid.getValue());
 224                     if (routerPort == null) {
 225                         LOG.error("The router port was not found for {}", subnetUuid.getValue());
 226                         continue;
 227                     }
 228                     vpnFootprintService.updateVpnToDpnMapping(primarySwitchId, routerName, primaryRd,
 229                             routerPort, null, false);
 230                     NatUtil.removeFromNeutronRouterDpnsMap(routerName, primarySwitchId, tx);
 231                     NatUtil.removeFromDpnRoutersMap(dataBroker, routerName, routerName,
 232                             primarySwitchId, interfaceManager, tx);
 233                     if (subnetIdToElanInstanceMap.containsKey(subnetUuid.getValue())) {
 234                         String elanInstanceName = subnetIdToElanInstanceMap.remove(subnetUuid.getValue());
 235                         NatUtil.removePseudoPortFromElanDpn(elanInstanceName, elanInstanceName, primarySwitchId,
 236                                 dataBroker);
 237                     }
 238                 }
 239             }), LOG, "Error deleting subnets from DPN maps for {}", routerName);
 240         } catch (InterruptedException | ExecutionException e) {
 241             LOG.error("Error deleting subnets from DPN maps for {}", routerName, e);
 242         }
 243     }
 244
 245     @Override
 246     public void switchAddedToCache(SwitchInfo switchInfo) {
 247         boolean scheduleRouters = (providerSwitchWeightsMap.size() == 0) ? true : false;
 248         for (String providerNet : switchInfo.getProviderNets()) {
 249             Map<Uint64,Integer> switchWeightMap = providerSwitchWeightsMap.get(providerNet);
 250             if (providerSwitchWeightsMap.get(providerNet) == null) {
 251                 switchWeightMap = new ConcurrentHashMap<>();
 252                 providerSwitchWeightsMap.put(providerNet, switchWeightMap);
 253             }
 254             LOG.info("addSwitch: Adding {} dpnId with provider mapping {} to switchWeightsMap",
 255                     switchInfo.getDpnId(), providerNet);
 256             switchWeightMap.put(switchInfo.getDpnId(), INITIAL_SWITCH_WEIGHT);
 257         }
 258         if (natMode == NatserviceConfig.NatMode.Conntrack && scheduleRouters) {
 259             Optional<ExtRouters> optRouters;
 260             try {
 261                 optRouters = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 262                         LogicalDatastoreType.CONFIGURATION, InstanceIdentifier.create(ExtRouters.class));
 263             } catch (InterruptedException | ExecutionException e) {
 264                 LOG.error("addSwitch: Error reading external routers", e);
 265                 return;
 266             }
 267
 268             if (optRouters.isPresent()) {
 269                 // Get the list of routers and verify if any routers do not have primarySwitch allocated.
 270                 for (Routers router : optRouters.get().nonnullRouters().values()) {
 271                     Map<ExternalIpsKey, ExternalIps> keyExternalIpsMap = router.getExternalIps();
 272                     if (router.isEnableSnat() && keyExternalIpsMap != null && !keyExternalIpsMap.isEmpty()) {
 273                         // Check if the primarySwitch is allocated for the router.
 274                         if (!isPrimarySwitchAllocatedForRouter(router.getRouterName())) {
 275                             scheduleCentralizedSwitch(router);
 276                         }
 277                     }
 278                 }
 279             }
 280         }
 281         return;
 282     }
 283
 284     private boolean isPrimarySwitchAllocatedForRouter(String routerName) {
 285         InstanceIdentifier<RouterToNaptSwitch> routerToNaptSwitch =
 286                 NatUtil.buildNaptSwitchRouterIdentifier(routerName);
 287         try {
 288             RouterToNaptSwitch rtrToNapt = SingleTransactionDataBroker.syncRead(dataBroker,
 289                     LogicalDatastoreType.CONFIGURATION, routerToNaptSwitch);
 290             Uint64 dpnId = rtrToNapt.getPrimarySwitchId();
 291             if (dpnId == null || dpnId.equals(Uint64.ZERO)) {
 292                 return false;
 293             }
 294         } catch (ExpectedDataObjectNotFoundException e) {
 295             LOG.error("isPrimarySwitchAllocatedForRouter: Error reading RouterToNaptSwitch model", e);
 296             return false;
 297         }
 298         return true;
 299     }
 300
 301     @Override
 302     public void switchRemovedFromCache(SwitchInfo switchInfo) {
 303         Uint64 dpnId = switchInfo.getDpnId();
 304         LOG.info("removeSwitch: Removing {} dpnId to switchWeightsMap", dpnId);
 305         for (Map.Entry<String,Map<Uint64,Integer>> providerNet : providerSwitchWeightsMap.entrySet()) {
 306             Map<Uint64,Integer> switchWeightMap = providerNet.getValue();
 307             if (natMode == NatserviceConfig.NatMode.Conntrack
 308                     && !INITIAL_SWITCH_WEIGHT.equals(switchWeightMap.get(dpnId))) {
 309                 NaptSwitches naptSwitches = getNaptSwitches();
 310                 for (RouterToNaptSwitch routerToNaptSwitch : naptSwitches.nonnullRouterToNaptSwitch().values()) {
 311                     if (dpnId.equals(routerToNaptSwitch.getPrimarySwitchId())) {
 312                         Routers router = NatUtil.getRoutersFromConfigDS(dataBroker, routerToNaptSwitch.getRouterName());
 313                         releaseCentralizedSwitch(router);
 314                         scheduleCentralizedSwitch(router);
 315                         break;
 316                     }
 317                 }
 318             }
 319             switchWeightMap.remove(dpnId);
 320         }
 321         return;
 322     }
 323
 324     private NaptSwitches getNaptSwitches() {
 325         InstanceIdentifier<NaptSwitches> id = InstanceIdentifier.builder(NaptSwitches.class).build();
 326         try {
 327             return SingleTransactionDataBroker.syncReadOptional(dataBroker,
 328                     LogicalDatastoreType.CONFIGURATION, id).orElse(null);
 329         } catch (ExecutionException | InterruptedException e) {
 330             LOG.error("getNaptSwitches: Exception while reading Napt-Switch DS", e);
 331         }
 332         return null;
 333     }
 334
 335     private Uint64 getSwitchWithLowestWeight(String providerNet) {
 336         int lowestWeight = Integer.MAX_VALUE;
 337         Uint64 nextSwitchId = Uint64.valueOf(0);
 338         Map<Uint64,Integer> switchWeightMap = providerSwitchWeightsMap.get(providerNet);
 339         if (null == switchWeightMap) {
 340             LOG.error("No switch have the provider mapping {}", providerNet);
 341             return nextSwitchId;
 342         }
 343         for (Entry<Uint64, Integer> entry : switchWeightMap.entrySet()) {
 344             Uint64 dpnId = entry.getKey();
 345             Integer weight = entry.getValue();
 346             if (lowestWeight > weight) {
 347                 lowestWeight = weight;
 348                 nextSwitchId =  dpnId;
 349             }
 350         }
 351         LOG.info("getSwitchWithLowestWeight: switchWeightsMap {}, returning nextSwitchId {} ",
 352                 providerSwitchWeightsMap, nextSwitchId);
 353         return nextSwitchId;
 354     }
 355
 356     private InstanceIdentifier<RouterToNaptSwitch> getNaptSwitchesIdentifier(String routerName) {
 357         return InstanceIdentifier.builder(NaptSwitches.class)
 358             .child(RouterToNaptSwitch.class, new RouterToNaptSwitchKey(routerName)).build();
 359     }
 360
 361     private InstanceIdentifier<Subnetmap> getSubnetMapIdentifier(Uuid subnetId) {
 362         return InstanceIdentifier.builder(Subnetmaps.class).child(Subnetmap.class,
 363                 new SubnetmapKey(subnetId)).build();
 364     }
 365
 366     @Nullable
 367     public Uint64 getCentralizedSwitch(String routerName) {
 368         try {
 369             Optional<RouterToNaptSwitch> naptSwitches = SingleTransactionDataBroker
 370                     .syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
 371                             getNaptSwitchesIdentifier(routerName));
 372             if (!naptSwitches.isPresent()) {
 373                 LOG.info("No Napt switch is scheduled for {}", routerName);
 374                 return null;
 375             }
 376             return naptSwitches.get().getPrimarySwitchId();
 377         } catch (InterruptedException | ExecutionException e) {
 378             LOG.error("Error reading RouterToNaptSwitch model", e);
 379             return null;
 380         }
 381     }
 382
 383     @Nullable
 384     private static List<Uuid> getUpdatedSubnetIds(List<Uuid> updatedSubnetIds, List<Uuid> currentSubnetIds) {
 385         if (updatedSubnetIds == null) {
 386             return null;
 387         }
 388         List<Uuid> newSubnetIds = new ArrayList<>(updatedSubnetIds);
 389         if (currentSubnetIds == null) {
 390             return newSubnetIds;
 391         }
 392         List<Uuid> origSubnetIds = new ArrayList<>(currentSubnetIds);
 393         for (Iterator<Uuid> iterator = newSubnetIds.iterator(); iterator.hasNext();) {
 394             Uuid updatedSubnetId = iterator.next();
 395             for (Uuid currentSubnetId : origSubnetIds) {
 396                 if (updatedSubnetId.getValue().equals(currentSubnetId.getValue())) {
 397                     iterator.remove();
 398                     break;
 399                 }
 400             }
 401         }
 402         return newSubnetIds;
 403     }
 404 }