natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/ha/WeightedCentralizedSwitchScheduler.java

   1 /*
   2  * Copyright (c) 2017 Red Hat, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.netvirt.natservice.ha;
  10
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13
  14 import com.google.common.base.Optional;
  15 import java.util.ArrayList;
  16 import java.util.HashMap;
  17 import java.util.Iterator;
  18 import java.util.List;
  19 import java.util.Map;
  20 import java.util.Map.Entry;
  21 import java.util.concurrent.ConcurrentHashMap;
  22 import java.util.concurrent.ExecutionException;
  23 import javax.inject.Inject;
  24 import javax.inject.Singleton;
  25 import org.eclipse.jdt.annotation.Nullable;
  26 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  27 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  28 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  29 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  30 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  31 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  32 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  33 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  34 import org.opendaylight.netvirt.natservice.api.CentralizedSwitchScheduler;
  35 import org.opendaylight.netvirt.natservice.api.NatSwitchCache;
  36 import org.opendaylight.netvirt.natservice.api.NatSwitchCacheListener;
  37 import org.opendaylight.netvirt.natservice.api.SwitchInfo;
  38 import org.opendaylight.netvirt.natservice.internal.NatUtil;
  39 import org.opendaylight.netvirt.vpnmanager.api.IVpnFootprintService;
  40 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExtRouters;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.NaptSwitches;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.routers.ExternalIps;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchBuilder;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
  54 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  55 import org.opendaylight.yangtools.yang.common.Uint64;
  56 import org.slf4j.Logger;
  57 import org.slf4j.LoggerFactory;
  58
  59 @Singleton
  60 public class WeightedCentralizedSwitchScheduler implements CentralizedSwitchScheduler, NatSwitchCacheListener {
  61     private static final Logger LOG = LoggerFactory.getLogger(WeightedCentralizedSwitchScheduler.class);
  62     private static final Integer INITIAL_SWITCH_WEIGHT = Integer.valueOf(0);
  63
  64     private final Map<String, Map<Uint64,Integer>> providerSwitchWeightsMap = new ConcurrentHashMap<>();
  65     private final Map<String,String> subnetIdToRouterPortMap = new ConcurrentHashMap<>();
  66     private final Map<String,String> subnetIdToElanInstanceMap = new ConcurrentHashMap<>();
  67     private final DataBroker dataBroker;
  68     private final ManagedNewTransactionRunner txRunner;
  69     private final OdlInterfaceRpcService interfaceManager;
  70     private final IVpnFootprintService vpnFootprintService;
  71     private final NatserviceConfig.NatMode natMode;
  72
  73     @Inject
  74     public WeightedCentralizedSwitchScheduler(final DataBroker dataBroker,
  75             final OdlInterfaceRpcService interfaceManager,
  76             final IVpnFootprintService vpnFootprintService, final NatserviceConfig config,
  77             final NatSwitchCache natSwitchCache) {
  78         this.dataBroker = dataBroker;
  79         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
  80         this.interfaceManager = interfaceManager;
  81         this.vpnFootprintService = vpnFootprintService;
  82         if (config != null) {
  83             this.natMode = config.getNatMode();
  84         } else {
  85             this.natMode = NatserviceConfig.NatMode.Controller;
  86         }
  87         natSwitchCache.register(this);
  88     }
  89
  90     @Override
  91     public boolean scheduleCentralizedSwitch(Routers router) {
  92         String providerNet = NatUtil.getElanInstancePhysicalNetwok(router.getNetworkId().getValue(),dataBroker);
  93         Uint64 nextSwitchId = getSwitchWithLowestWeight(providerNet);
  94         if (Uint64.ZERO.equals(nextSwitchId)) {
  95             LOG.error("In scheduleCentralizedSwitch, unable to schedule the router {} as there is no available switch.",
  96                     router.getRouterName());
  97             return false;
  98         }
  99
 100         LOG.info("scheduleCentralizedSwitch for router {} on switch {}", router.getRouterName(), nextSwitchId);
 101         String routerName = router.getRouterName();
 102         RouterToNaptSwitchBuilder routerToNaptSwitchBuilder =
 103                 new RouterToNaptSwitchBuilder().setRouterName(routerName);
 104         RouterToNaptSwitch id = routerToNaptSwitchBuilder.setPrimarySwitchId(nextSwitchId)
 105                 .setEnableSnat(router.isEnableSnat()).build();
 106         addToDpnMaps(routerName, router.getSubnetIds(), nextSwitchId);
 107         try {
 108             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
 109                     getNaptSwitchesIdentifier(routerName), id);
 110             Map<Uint64,Integer> switchWeightMap = providerSwitchWeightsMap.get(providerNet);
 111             switchWeightMap.put(nextSwitchId,switchWeightMap.get(nextSwitchId) + 1);
 112
 113         } catch (TransactionCommitFailedException e) {
 114             LOG.error("ScheduleCentralizedSwitch failed for {}", routerName);
 115         }
 116         return true;
 117
 118     }
 119
 120     @Override
 121     public boolean updateCentralizedSwitch(Routers oldRouter, Routers newRouter) {
 122         LOG.info("updateCentralizedSwitch for router {}", newRouter.getRouterName());
 123         String routerName = newRouter.getRouterName();
 124         List<Uuid> addedSubnetIds = getUpdatedSubnetIds(newRouter.getSubnetIds(), oldRouter.getSubnetIds());
 125         List<Uuid> deletedSubnetIds = getUpdatedSubnetIds(oldRouter.getSubnetIds(), newRouter.getSubnetIds());
 126         Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, newRouter.getRouterName());
 127         addToDpnMaps(routerName, addedSubnetIds, primarySwitchId);
 128         deleteFromDpnMaps(routerName, deletedSubnetIds, primarySwitchId);
 129         try {
 130             InstanceIdentifier<RouterToNaptSwitch> id  = NatUtil.buildNaptSwitchIdentifier(routerName);
 131             RouterToNaptSwitch routerToNaptSwitch = SingleTransactionDataBroker.syncRead(dataBroker,
 132                     LogicalDatastoreType.CONFIGURATION, id);
 133             boolean isSnatEnabled = newRouter.isEnableSnat();
 134             List<ExternalIps> updateExternalIps = newRouter.getExternalIps();
 135             if (updateExternalIps == null || updateExternalIps.isEmpty()) {
 136                 isSnatEnabled = false;
 137             }
 138             if (isSnatEnabled != routerToNaptSwitch.isEnableSnat()) {
 139                 RouterToNaptSwitchBuilder routerToNaptSwitchBuilder =
 140                         new RouterToNaptSwitchBuilder(routerToNaptSwitch);
 141                 routerToNaptSwitchBuilder.setEnableSnat(isSnatEnabled);
 142                 SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
 143                         getNaptSwitchesIdentifier(routerName), routerToNaptSwitchBuilder.build());
 144             }
 145         } catch (ReadFailedException e) {
 146             LOG.error("updateCentralizedSwitch ReadFailedException for {}", routerName);
 147         } catch (TransactionCommitFailedException e) {
 148             LOG.error("updateCentralizedSwitch TransactionCommitFailedException for {}", routerName);
 149         }
 150         return true;
 151     }
 152
 153     @Override
 154     public boolean releaseCentralizedSwitch(Routers router) {
 155         String providerNet = NatUtil.getElanInstancePhysicalNetwok(router.getNetworkId().getValue(),dataBroker);
 156         String routerName = router.getRouterName();
 157         Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
 158         if (primarySwitchId == null || Uint64.ZERO.equals(primarySwitchId)) {
 159             LOG.info("releaseCentralizedSwitch: NAPT Switch is not allocated for router {}", router.getRouterName());
 160             return false;
 161         }
 162
 163         LOG.info("releaseCentralizedSwitch for router {} from switch {}", router.getRouterName(), primarySwitchId);
 164         deleteFromDpnMaps(routerName, router.getSubnetIds(), primarySwitchId);
 165         try {
 166             SingleTransactionDataBroker.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION,
 167                     getNaptSwitchesIdentifier(routerName));
 168             Map<Uint64,Integer> switchWeightMap = providerSwitchWeightsMap.get(providerNet);
 169             switchWeightMap.put(primarySwitchId, switchWeightMap.get(primarySwitchId) - 1);
 170         } catch (TransactionCommitFailedException e) {
 171             return false;
 172         }
 173         return true;
 174     }
 175
 176     private void addToDpnMaps(String routerName, List<Uuid> addedSubnetIds, Uint64 primarySwitchId) {
 177         if (addedSubnetIds == null || addedSubnetIds.isEmpty()) {
 178             LOG.debug("addToDpnMaps no subnets associated with {}", routerName);
 179             return;
 180         }
 181         Map<Uuid, Subnetmap> subnetMapEntries = new HashMap<>();
 182         try {
 183             String primaryRd = txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 184                 for (Uuid subnetUuid : addedSubnetIds) {
 185                     Subnetmap subnetMapEntry = tx.read(getSubnetMapIdentifier(subnetUuid)).get().orNull();
 186                     subnetMapEntries.put(subnetUuid, subnetMapEntry);
 187                     Uuid routerPortUuid = subnetMapEntry.getRouterInterfacePortId();
 188                     subnetIdToRouterPortMap.put(subnetUuid.getValue(), routerPortUuid.getValue());
 189                 }
 190                 return NatUtil.getPrimaryRd(routerName, tx);
 191             }).get();
 192             ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 193                 for (Uuid subnetUuid : addedSubnetIds) {
 194                     Subnetmap subnetMapEntry = subnetMapEntries.get(subnetUuid);
 195                     Uuid routerPortUuid = subnetMapEntry.getRouterInterfacePortId();
 196                     vpnFootprintService.updateVpnToDpnMapping(primarySwitchId, routerName, primaryRd,
 197                             routerPortUuid.getValue(), null, true);
 198                     NatUtil.addToNeutronRouterDpnsMap(routerName, routerPortUuid.getValue(), primarySwitchId, tx);
 199                     NatUtil.addToDpnRoutersMap(routerName, routerPortUuid.getValue(), primarySwitchId, tx);
 200                     if (subnetMapEntry.getNetworkType().equals(NetworkAttributes.NetworkType.VLAN)) {
 201                         String elanInstanceName = subnetMapEntry.getNetworkId().getValue();
 202                         subnetIdToElanInstanceMap.put(subnetUuid.getValue(), elanInstanceName);
 203                         NatUtil.addPseudoPortToElanDpn(elanInstanceName, elanInstanceName, primarySwitchId, dataBroker);
 204                     }
 205                 }
 206             }), LOG, "Error adding subnets to DPN maps for {}", routerName);
 207         } catch (InterruptedException | ExecutionException e) {
 208             LOG.error("Error adding subnets to DPN maps for {}", routerName);
 209         }
 210     }
 211
 212     private void deleteFromDpnMaps(String routerName, List<Uuid> deletedSubnetIds, Uint64 primarySwitchId) {
 213         if (deletedSubnetIds == null || deletedSubnetIds.isEmpty()) {
 214             LOG.debug("deleteFromDpnMaps no subnets associated with {}", routerName);
 215             return;
 216         }
 217         try {
 218             String primaryRd = txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 219                 tx -> NatUtil.getPrimaryRd(routerName, tx)).get();
 220             ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 221                 for (Uuid subnetUuid : deletedSubnetIds) {
 222                     String routerPort = subnetIdToRouterPortMap.remove(subnetUuid.getValue());
 223                     if (routerPort == null) {
 224                         LOG.error("The router port was not found for {}", subnetUuid.getValue());
 225                         continue;
 226                     }
 227                     vpnFootprintService.updateVpnToDpnMapping(primarySwitchId, routerName, primaryRd,
 228                             routerPort, null, false);
 229                     NatUtil.removeFromNeutronRouterDpnsMap(routerName, primarySwitchId, tx);
 230                     NatUtil.removeFromDpnRoutersMap(dataBroker, routerName, routerName,
 231                             primarySwitchId, interfaceManager, tx);
 232                     if (subnetIdToElanInstanceMap.containsKey(subnetUuid.getValue())) {
 233                         String elanInstanceName = subnetIdToElanInstanceMap.remove(subnetUuid.getValue());
 234                         NatUtil.removePseudoPortFromElanDpn(elanInstanceName, elanInstanceName, primarySwitchId,
 235                                 dataBroker);
 236                     }
 237                 }
 238             }), LOG, "Error deleting subnets from DPN maps for {}", routerName);
 239         } catch (InterruptedException | ExecutionException e) {
 240             LOG.error("Error deleting subnets from DPN maps for {}", routerName, e);
 241         }
 242     }
 243
 244     @Override
 245     public void switchAddedToCache(SwitchInfo switchInfo) {
 246         boolean scheduleRouters = (providerSwitchWeightsMap.size() == 0) ? true : false;
 247         for (String providerNet : switchInfo.getProviderNets()) {
 248             Map<Uint64,Integer> switchWeightMap = providerSwitchWeightsMap.get(providerNet);
 249             if (providerSwitchWeightsMap.get(providerNet) == null) {
 250                 switchWeightMap = new ConcurrentHashMap<>();
 251                 providerSwitchWeightsMap.put(providerNet, switchWeightMap);
 252             }
 253             LOG.info("addSwitch: Adding {} dpnId with provider mapping {} to switchWeightsMap",
 254                     switchInfo.getDpnId(), providerNet);
 255             switchWeightMap.put(switchInfo.getDpnId(), INITIAL_SWITCH_WEIGHT);
 256         }
 257         if (natMode == NatserviceConfig.NatMode.Conntrack && scheduleRouters) {
 258             Optional<ExtRouters> optRouters;
 259             try {
 260                 optRouters = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 261                         LogicalDatastoreType.CONFIGURATION, InstanceIdentifier.create(ExtRouters.class));
 262             } catch (ReadFailedException e) {
 263                 LOG.error("addSwitch: Error reading external routers", e);
 264                 return;
 265             }
 266
 267             if (optRouters.isPresent()) {
 268                 // Get the list of routers and verify if any routers do not have primarySwitch allocated.
 269                 for (Routers router : optRouters.get().getRouters()) {
 270                     List<ExternalIps> externalIps = router.getExternalIps();
 271                     if (router.isEnableSnat() && externalIps != null && !externalIps.isEmpty()) {
 272                         // Check if the primarySwitch is allocated for the router.
 273                         if (!isPrimarySwitchAllocatedForRouter(router.getRouterName())) {
 274                             scheduleCentralizedSwitch(router);
 275                         }
 276                     }
 277                 }
 278             }
 279         }
 280         return;
 281     }
 282
 283     private boolean isPrimarySwitchAllocatedForRouter(String routerName) {
 284         InstanceIdentifier<RouterToNaptSwitch> routerToNaptSwitch =
 285                 NatUtil.buildNaptSwitchRouterIdentifier(routerName);
 286         try {
 287             RouterToNaptSwitch rtrToNapt = SingleTransactionDataBroker.syncRead(dataBroker,
 288                     LogicalDatastoreType.CONFIGURATION, routerToNaptSwitch);
 289             Uint64 dpnId = rtrToNapt.getPrimarySwitchId();
 290             if (dpnId == null || dpnId.equals(Uint64.ZERO)) {
 291                 return false;
 292             }
 293         } catch (ReadFailedException e) {
 294             LOG.error("isPrimarySwitchAllocatedForRouter: Error reading RouterToNaptSwitch model", e);
 295             return false;
 296         }
 297         return true;
 298     }
 299
 300     @Override
 301     public void switchRemovedFromCache(SwitchInfo switchInfo) {
 302         Uint64 dpnId = switchInfo.getDpnId();
 303         LOG.info("removeSwitch: Removing {} dpnId to switchWeightsMap", dpnId);
 304         for (Map.Entry<String,Map<Uint64,Integer>> providerNet : providerSwitchWeightsMap.entrySet()) {
 305             Map<Uint64,Integer> switchWeightMap = providerNet.getValue();
 306             if (natMode == NatserviceConfig.NatMode.Conntrack
 307                     && !INITIAL_SWITCH_WEIGHT.equals(switchWeightMap.get(dpnId))) {
 308                 NaptSwitches naptSwitches = getNaptSwitches();
 309                 for (RouterToNaptSwitch routerToNaptSwitch : naptSwitches.getRouterToNaptSwitch()) {
 310                     if (dpnId.equals(routerToNaptSwitch.getPrimarySwitchId())) {
 311                         Routers router = NatUtil.getRoutersFromConfigDS(dataBroker, routerToNaptSwitch.getRouterName());
 312                         releaseCentralizedSwitch(router);
 313                         scheduleCentralizedSwitch(router);
 314                         break;
 315                     }
 316                 }
 317             }
 318             switchWeightMap.remove(dpnId);
 319         }
 320         return;
 321     }
 322
 323     private NaptSwitches getNaptSwitches() {
 324         InstanceIdentifier<NaptSwitches> id = InstanceIdentifier.builder(NaptSwitches.class).build();
 325         return SingleTransactionDataBroker.syncReadOptionalAndTreatReadFailedExceptionAsAbsentOptional(dataBroker,
 326                 LogicalDatastoreType.CONFIGURATION, id).orNull();
 327     }
 328
 329     private Uint64 getSwitchWithLowestWeight(String providerNet) {
 330         int lowestWeight = Integer.MAX_VALUE;
 331         Uint64 nextSwitchId = Uint64.valueOf(0);
 332         Map<Uint64,Integer> switchWeightMap = providerSwitchWeightsMap.get(providerNet);
 333         if (null == switchWeightMap) {
 334             LOG.error("No switch have the provider mapping {}", providerNet);
 335             return nextSwitchId;
 336         }
 337         for (Entry<Uint64, Integer> entry : switchWeightMap.entrySet()) {
 338             Uint64 dpnId = entry.getKey();
 339             Integer weight = entry.getValue();
 340             if (lowestWeight > weight) {
 341                 lowestWeight = weight;
 342                 nextSwitchId =  dpnId;
 343             }
 344         }
 345         LOG.info("getSwitchWithLowestWeight: switchWeightsMap {}, returning nextSwitchId {} ",
 346                 providerSwitchWeightsMap, nextSwitchId);
 347         return nextSwitchId;
 348     }
 349
 350     private InstanceIdentifier<RouterToNaptSwitch> getNaptSwitchesIdentifier(String routerName) {
 351         return InstanceIdentifier.builder(NaptSwitches.class)
 352             .child(RouterToNaptSwitch.class, new RouterToNaptSwitchKey(routerName)).build();
 353     }
 354
 355     private InstanceIdentifier<Subnetmap> getSubnetMapIdentifier(Uuid subnetId) {
 356         return InstanceIdentifier.builder(Subnetmaps.class).child(Subnetmap.class,
 357                 new SubnetmapKey(subnetId)).build();
 358     }
 359
 360     @Nullable
 361     public Uint64 getCentralizedSwitch(String routerName) {
 362         try {
 363             Optional<RouterToNaptSwitch> naptSwitches = SingleTransactionDataBroker
 364                     .syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
 365                             getNaptSwitchesIdentifier(routerName));
 366             if (!naptSwitches.isPresent()) {
 367                 LOG.info("No Napt switch is scheduled for {}", routerName);
 368                 return null;
 369             }
 370             return naptSwitches.get().getPrimarySwitchId();
 371         } catch (ReadFailedException e) {
 372             LOG.error("Error reading RouterToNaptSwitch model", e);
 373             return null;
 374         }
 375     }
 376
 377     @Nullable
 378     private static List<Uuid> getUpdatedSubnetIds(List<Uuid> updatedSubnetIds, List<Uuid> currentSubnetIds) {
 379         if (updatedSubnetIds == null) {
 380             return null;
 381         }
 382         List<Uuid> newSubnetIds = new ArrayList<>(updatedSubnetIds);
 383         if (currentSubnetIds == null) {
 384             return newSubnetIds;
 385         }
 386         List<Uuid> origSubnetIds = new ArrayList<>(currentSubnetIds);
 387         for (Iterator<Uuid> iterator = newSubnetIds.iterator(); iterator.hasNext();) {
 388             Uuid updatedSubnetId = iterator.next();
 389             for (Uuid currentSubnetId : origSubnetIds) {
 390                 if (updatedSubnetId.getValue().equals(currentSubnetId.getValue())) {
 391                     iterator.remove();
 392                     break;
 393                 }
 394             }
 395         }
 396         return newSubnetIds;
 397     }
 398 }