natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/ha/WeightedCentralizedSwitchScheduler.java

   1 /*
   2  * Copyright (c) 2017 Red Hat, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.netvirt.natservice.ha;
  10
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13
  14 import com.google.common.base.Optional;
  15
  16 import java.math.BigInteger;
  17 import java.util.ArrayList;
  18 import java.util.HashMap;
  19 import java.util.Iterator;
  20 import java.util.List;
  21 import java.util.Map;
  22 import java.util.Map.Entry;
  23 import java.util.concurrent.ConcurrentHashMap;
  24 import java.util.concurrent.ExecutionException;
  25
  26 import javax.inject.Inject;
  27 import javax.inject.Singleton;
  28
  29 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  30 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  31 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  32 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  33 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  34 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  35 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  36 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  37 import org.opendaylight.netvirt.natservice.api.CentralizedSwitchScheduler;
  38 import org.opendaylight.netvirt.natservice.internal.NatUtil;
  39 import org.opendaylight.netvirt.vpnmanager.api.IVpnFootprintService;
  40 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExtRouters;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.NaptSwitches;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.routers.ExternalIps;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchBuilder;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
  54 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  55 import org.slf4j.Logger;
  56 import org.slf4j.LoggerFactory;
  57
  58 @Singleton
  59 public class WeightedCentralizedSwitchScheduler implements CentralizedSwitchScheduler {
  60     private static final Logger LOG = LoggerFactory.getLogger(WeightedCentralizedSwitchScheduler.class);
  61     private static final Integer INITIAL_SWITCH_WEIGHT = Integer.valueOf(0);
  62
  63     private final Map<String, Map<BigInteger,Integer>> providerSwitchWeightsMap = new ConcurrentHashMap<>();
  64     private final Map<String,String> subnetIdToRouterPortMap = new ConcurrentHashMap<>();
  65     private final Map<String,String> subnetIdToElanInstanceMap = new ConcurrentHashMap<>();
  66     private final DataBroker dataBroker;
  67     private final ManagedNewTransactionRunner txRunner;
  68     private final OdlInterfaceRpcService interfaceManager;
  69     private final IVpnFootprintService vpnFootprintService;
  70     private final NatserviceConfig.NatMode natMode;
  71
  72     @Inject
  73     public WeightedCentralizedSwitchScheduler(DataBroker dataBroker, OdlInterfaceRpcService interfaceManager,
  74             IVpnFootprintService vpnFootprintService, final NatserviceConfig config) {
  75         this.dataBroker = dataBroker;
  76         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
  77         this.interfaceManager = interfaceManager;
  78         this.vpnFootprintService = vpnFootprintService;
  79         if (config != null) {
  80             this.natMode = config.getNatMode();
  81         } else {
  82             this.natMode = NatserviceConfig.NatMode.Controller;
  83         }
  84     }
  85
  86     @Override
  87     public boolean scheduleCentralizedSwitch(Routers router) {
  88         String providerNet = NatUtil.getElanInstancePhysicalNetwok(router.getNetworkId().getValue(),dataBroker);
  89         BigInteger nextSwitchId = getSwitchWithLowestWeight(providerNet);
  90         if (nextSwitchId == BigInteger.valueOf(0)) {
  91             LOG.error("In scheduleCentralizedSwitch, unable to schedule the router {} as there is no available switch.",
  92                     router.getRouterName());
  93             return false;
  94         }
  95
  96         LOG.info("scheduleCentralizedSwitch for router {} on switch {}", router.getRouterName(), nextSwitchId);
  97         String routerName = router.getRouterName();
  98         RouterToNaptSwitchBuilder routerToNaptSwitchBuilder =
  99                 new RouterToNaptSwitchBuilder().setRouterName(routerName);
 100         RouterToNaptSwitch id = routerToNaptSwitchBuilder.setPrimarySwitchId(nextSwitchId)
 101                 .setEnableSnat(router.isEnableSnat()).build();
 102         addToDpnMaps(routerName, router.getSubnetIds(), nextSwitchId);
 103         try {
 104             SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
 105                     getNaptSwitchesIdentifier(routerName), id);
 106             Map<BigInteger,Integer> switchWeightMap = providerSwitchWeightsMap.get(providerNet);
 107             switchWeightMap.put(nextSwitchId,switchWeightMap.get(nextSwitchId) + 1);
 108
 109         } catch (TransactionCommitFailedException e) {
 110             LOG.error("ScheduleCentralizedSwitch failed for {}", routerName);
 111         }
 112         return true;
 113
 114     }
 115
 116     @Override
 117     public boolean updateCentralizedSwitch(Routers oldRouter, Routers newRouter) {
 118         LOG.info("updateCentralizedSwitch for router {}", newRouter.getRouterName());
 119         String routerName = newRouter.getRouterName();
 120         List<Uuid> addedSubnetIds = getUpdatedSubnetIds(newRouter.getSubnetIds(), oldRouter.getSubnetIds());
 121         List<Uuid> deletedSubnetIds = getUpdatedSubnetIds(oldRouter.getSubnetIds(), newRouter.getSubnetIds());
 122         BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, newRouter.getRouterName());
 123         addToDpnMaps(routerName, addedSubnetIds, primarySwitchId);
 124         deleteFromDpnMaps(routerName, deletedSubnetIds, primarySwitchId);
 125         try {
 126             InstanceIdentifier<RouterToNaptSwitch> id  = NatUtil.buildNaptSwitchIdentifier(routerName);
 127             RouterToNaptSwitch routerToNaptSwitch = SingleTransactionDataBroker.syncRead(dataBroker,
 128                     LogicalDatastoreType.CONFIGURATION, id);
 129             boolean isSnatEnabled = newRouter.isEnableSnat();
 130             List<ExternalIps> updateExternalIps = newRouter.getExternalIps();
 131             if (updateExternalIps == null || updateExternalIps.isEmpty()) {
 132                 isSnatEnabled = false;
 133             }
 134             if (isSnatEnabled != routerToNaptSwitch.isEnableSnat()) {
 135                 RouterToNaptSwitchBuilder routerToNaptSwitchBuilder =
 136                         new RouterToNaptSwitchBuilder(routerToNaptSwitch);
 137                 routerToNaptSwitchBuilder.setEnableSnat(isSnatEnabled);
 138                 SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
 139                         getNaptSwitchesIdentifier(routerName), routerToNaptSwitchBuilder.build());
 140             }
 141         } catch (ReadFailedException e) {
 142             LOG.error("updateCentralizedSwitch ReadFailedException for {}", routerName);
 143         } catch (TransactionCommitFailedException e) {
 144             LOG.error("updateCentralizedSwitch TransactionCommitFailedException for {}", routerName);
 145         }
 146         return true;
 147     }
 148
 149     @Override
 150     public boolean releaseCentralizedSwitch(Routers router) {
 151         String providerNet = NatUtil.getElanInstancePhysicalNetwok(router.getNetworkId().getValue(),dataBroker);
 152         String routerName = router.getRouterName();
 153         BigInteger primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
 154         if (primarySwitchId == null || primarySwitchId == BigInteger.valueOf(0)) {
 155             LOG.info("releaseCentralizedSwitch: NAPT Switch is not allocated for router {}", router.getRouterName());
 156             return false;
 157         }
 158
 159         LOG.info("releaseCentralizedSwitch for router {} from switch {}", router.getRouterName(), primarySwitchId);
 160         deleteFromDpnMaps(routerName, router.getSubnetIds(), primarySwitchId);
 161         try {
 162             SingleTransactionDataBroker.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION,
 163                     getNaptSwitchesIdentifier(routerName));
 164             Map<BigInteger,Integer> switchWeightMap = providerSwitchWeightsMap.get(providerNet);
 165             switchWeightMap.put(primarySwitchId, switchWeightMap.get(primarySwitchId) - 1);
 166         } catch (TransactionCommitFailedException e) {
 167             return false;
 168         }
 169         return true;
 170     }
 171
 172     private void addToDpnMaps(String routerName, List<Uuid> addedSubnetIds, BigInteger primarySwitchId) {
 173         if (addedSubnetIds == null || addedSubnetIds.isEmpty()) {
 174             LOG.debug("addToDpnMaps no subnets associated with {}", routerName);
 175             return;
 176         }
 177         Map<Uuid, Subnetmap> subnetMapEntries = new HashMap<>();
 178         try {
 179             String primaryRd = txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 180                 for (Uuid subnetUuid : addedSubnetIds) {
 181                     Subnetmap subnetMapEntry = tx.read(getSubnetMapIdentifier(subnetUuid)).get().orNull();
 182                     subnetMapEntries.put(subnetUuid, subnetMapEntry);
 183                     Uuid routerPortUuid = subnetMapEntry.getRouterInterfacePortId();
 184                     subnetIdToRouterPortMap.put(subnetUuid.getValue(), routerPortUuid.getValue());
 185                 }
 186                 return NatUtil.getPrimaryRd(routerName, tx);
 187             }).get();
 188             ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 189                 for (Uuid subnetUuid : addedSubnetIds) {
 190                     Subnetmap subnetMapEntry = subnetMapEntries.get(subnetUuid);
 191                     Uuid routerPortUuid = subnetMapEntry.getRouterInterfacePortId();
 192                     vpnFootprintService.updateVpnToDpnMapping(primarySwitchId, routerName, primaryRd,
 193                             routerPortUuid.getValue(), null, true);
 194                     NatUtil.addToNeutronRouterDpnsMap(routerName, routerPortUuid.getValue(), primarySwitchId, tx);
 195                     NatUtil.addToDpnRoutersMap(routerName, routerPortUuid.getValue(), primarySwitchId, tx);
 196                     if (subnetMapEntry.getNetworkType().equals(NetworkAttributes.NetworkType.VLAN)) {
 197                         String elanInstanceName = subnetMapEntry.getNetworkId().getValue();
 198                         subnetIdToElanInstanceMap.put(subnetUuid.getValue(), elanInstanceName);
 199                         NatUtil.addPseudoPortToElanDpn(elanInstanceName, elanInstanceName, primarySwitchId, dataBroker);
 200                     }
 201                 }
 202             }), LOG, "Error adding subnets to DPN maps for {}", routerName);
 203         } catch (InterruptedException | ExecutionException e) {
 204             LOG.error("Error adding subnets to DPN maps for {}", routerName);
 205         }
 206     }
 207
 208
 209
 210     private void deleteFromDpnMaps(String routerName, List<Uuid> deletedSubnetIds, BigInteger primarySwitchId) {
 211         if (deletedSubnetIds == null || deletedSubnetIds.isEmpty()) {
 212             LOG.debug("deleteFromDpnMaps no subnets associated with {}", routerName);
 213             return;
 214         }
 215         try {
 216             String primaryRd = txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 217                 tx -> NatUtil.getPrimaryRd(routerName, tx)).get();
 218             ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 219                 for (Uuid subnetUuid : deletedSubnetIds) {
 220                     String routerPort = subnetIdToRouterPortMap.remove(subnetUuid.getValue());
 221                     if (routerPort == null) {
 222                         LOG.error("The router port was not found for {}", subnetUuid.getValue());
 223                         continue;
 224                     }
 225                     vpnFootprintService.updateVpnToDpnMapping(primarySwitchId, routerName, primaryRd,
 226                             routerPort, null, false);
 227                     NatUtil.removeFromNeutronRouterDpnsMap(routerName, primarySwitchId, tx);
 228                     NatUtil.removeFromDpnRoutersMap(dataBroker, routerName, routerName,
 229                             primarySwitchId, interfaceManager, tx);
 230                     if (subnetIdToElanInstanceMap.containsKey(subnetUuid.getValue())) {
 231                         String elanInstanceName = subnetIdToElanInstanceMap.remove(subnetUuid.getValue());
 232                         NatUtil.removePseudoPortFromElanDpn(elanInstanceName, elanInstanceName, primarySwitchId,
 233                                 dataBroker);
 234                     }
 235                 }
 236             }), LOG, "Error deleting subnets from DPN maps for {}", routerName);
 237         } catch (InterruptedException | ExecutionException e) {
 238             LOG.error("Error deleting subnets from DPN maps for {}", routerName, e);
 239         }
 240     }
 241
 242     @Override
 243     public boolean addSwitch(BigInteger dpnId) {
 244         /* Initialize the switch in the map with weight 0 */
 245         LOG.info("addSwitch: Retrieving the provider config for {}", dpnId);
 246         boolean scheduleRouters = (providerSwitchWeightsMap.size() == 0) ? true : false;
 247         Map<String, String> providerMappingsMap = NatUtil.getOpenvswitchOtherConfigMap(dpnId, dataBroker);
 248         for (String providerNet : providerMappingsMap.keySet()) {
 249             Map<BigInteger,Integer> switchWeightMap = providerSwitchWeightsMap.get(providerNet);
 250             if (providerSwitchWeightsMap.get(providerNet) == null) {
 251                 switchWeightMap = new ConcurrentHashMap<>();
 252                 providerSwitchWeightsMap.put(providerNet, switchWeightMap);
 253             }
 254             LOG.info("addSwitch: Adding {} dpnId with provider mapping {} to switchWeightsMap", dpnId, providerNet);
 255             switchWeightMap.put(dpnId, INITIAL_SWITCH_WEIGHT);
 256         }
 257         if (natMode == NatserviceConfig.NatMode.Conntrack && scheduleRouters) {
 258             Optional<ExtRouters> optRouters;
 259             try {
 260                 optRouters = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 261                         LogicalDatastoreType.CONFIGURATION, InstanceIdentifier.create(ExtRouters.class));
 262             } catch (ReadFailedException e) {
 263                 LOG.error("addSwitch: Error reading external routers", e);
 264                 return false;
 265             }
 266
 267             if (optRouters.isPresent()) {
 268                 // Get the list of routers and verify if any routers do not have primarySwitch allocated.
 269                 for (Routers router : optRouters.get().getRouters()) {
 270                     List<ExternalIps> externalIps = router.getExternalIps();
 271                     if (router.isEnableSnat() && externalIps != null && !externalIps.isEmpty()) {
 272                         // Check if the primarySwitch is allocated for the router.
 273                         if (!isPrimarySwitchAllocatedForRouter(router.getRouterName())) {
 274                             scheduleCentralizedSwitch(router);
 275                         }
 276                     }
 277                 }
 278             }
 279         }
 280         return true;
 281     }
 282
 283     private boolean isPrimarySwitchAllocatedForRouter(String routerName) {
 284         InstanceIdentifier<RouterToNaptSwitch> routerToNaptSwitch =
 285                 NatUtil.buildNaptSwitchRouterIdentifier(routerName);
 286         try {
 287             RouterToNaptSwitch rtrToNapt = SingleTransactionDataBroker.syncRead(dataBroker,
 288                     LogicalDatastoreType.CONFIGURATION, routerToNaptSwitch);
 289             BigInteger dpnId = rtrToNapt.getPrimarySwitchId();
 290             if (dpnId == null || dpnId.equals(BigInteger.ZERO)) {
 291                 return false;
 292             }
 293         } catch (ReadFailedException e) {
 294             LOG.error("isPrimarySwitchAllocatedForRouter: Error reading RouterToNaptSwitch model", e);
 295             return false;
 296         }
 297         return true;
 298     }
 299
 300     @Override
 301     public boolean removeSwitch(BigInteger dpnId) {
 302         LOG.info("removeSwitch: Removing {} dpnId to switchWeightsMap", dpnId);
 303         for (Map.Entry<String,Map<BigInteger,Integer>> providerNet : providerSwitchWeightsMap.entrySet()) {
 304             Map<BigInteger,Integer> switchWeightMap = providerNet.getValue();
 305             if (natMode == NatserviceConfig.NatMode.Conntrack
 306                     && !INITIAL_SWITCH_WEIGHT.equals(switchWeightMap.get(dpnId))) {
 307                 NaptSwitches naptSwitches = getNaptSwitches();
 308                 for (RouterToNaptSwitch routerToNaptSwitch : naptSwitches.getRouterToNaptSwitch()) {
 309                     if (dpnId.equals(routerToNaptSwitch.getPrimarySwitchId())) {
 310                         Routers router = NatUtil.getRoutersFromConfigDS(dataBroker, routerToNaptSwitch.getRouterName());
 311                         releaseCentralizedSwitch(router);
 312                         scheduleCentralizedSwitch(router);
 313                         break;
 314                     }
 315                 }
 316             }
 317             switchWeightMap.remove(dpnId);
 318         }
 319         return true;
 320     }
 321
 322     private NaptSwitches getNaptSwitches() {
 323         InstanceIdentifier<NaptSwitches> id = InstanceIdentifier.builder(NaptSwitches.class).build();
 324         return SingleTransactionDataBroker.syncReadOptionalAndTreatReadFailedExceptionAsAbsentOptional(dataBroker,
 325                 LogicalDatastoreType.CONFIGURATION, id).orNull();
 326     }
 327
 328     private BigInteger getSwitchWithLowestWeight(String providerNet) {
 329         int lowestWeight = Integer.MAX_VALUE;
 330         BigInteger nextSwitchId = BigInteger.valueOf(0);
 331         Map<BigInteger,Integer> switchWeightMap = providerSwitchWeightsMap.get(providerNet);
 332         if (null == switchWeightMap) {
 333             LOG.error("No switch have the provider mapping {}", providerNet);
 334             return nextSwitchId;
 335         }
 336         for (Entry<BigInteger, Integer> entry : switchWeightMap.entrySet()) {
 337             BigInteger dpnId = entry.getKey();
 338             Integer weight = entry.getValue();
 339             if (lowestWeight > weight) {
 340                 lowestWeight = weight;
 341                 nextSwitchId =  dpnId;
 342             }
 343         }
 344         LOG.info("getSwitchWithLowestWeight: switchWeightsMap {}, returning nextSwitchId {} ",
 345                 providerSwitchWeightsMap, nextSwitchId);
 346         return nextSwitchId;
 347     }
 348
 349     private InstanceIdentifier<RouterToNaptSwitch> getNaptSwitchesIdentifier(String routerName) {
 350         return InstanceIdentifier.builder(NaptSwitches.class)
 351             .child(RouterToNaptSwitch.class, new RouterToNaptSwitchKey(routerName)).build();
 352     }
 353
 354     private InstanceIdentifier<Subnetmap> getSubnetMapIdentifier(Uuid subnetId) {
 355         return InstanceIdentifier.builder(Subnetmaps.class).child(Subnetmap.class,
 356                 new SubnetmapKey(subnetId)).build();
 357     }
 358
 359     public BigInteger getCentralizedSwitch(String routerName) {
 360         try {
 361             Optional<RouterToNaptSwitch> naptSwitches = SingleTransactionDataBroker
 362                     .syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
 363                             getNaptSwitchesIdentifier(routerName));
 364             if (!naptSwitches.isPresent()) {
 365                 LOG.info("No Napt switch is scheduled for {}", routerName);
 366                 return null;
 367             }
 368             return naptSwitches.get().getPrimarySwitchId();
 369         } catch (ReadFailedException e) {
 370             LOG.error("Error reading RouterToNaptSwitch model", e);
 371             return null;
 372         }
 373     }
 374
 375     @Override
 376     public boolean isSwitchConnectedToExternal(BigInteger dpnId, String providerNet) {
 377         Map<BigInteger,Integer> switchWeightMap = providerSwitchWeightsMap.get(providerNet);
 378         if (switchWeightMap != null) {
 379             return switchWeightMap.containsKey(dpnId);
 380         }
 381         return false;
 382     }
 383
 384     public static List<Uuid> getUpdatedSubnetIds(
 385             List<Uuid> updatedSubnetIds,
 386             List<Uuid> currentSubnetIds) {
 387         if (updatedSubnetIds == null) {
 388             return null;
 389         }
 390         List<Uuid> newSubnetIds = new ArrayList<>(updatedSubnetIds);
 391         if (currentSubnetIds == null) {
 392             return newSubnetIds;
 393         }
 394         List<Uuid> origSubnetIds = new ArrayList<>(currentSubnetIds);
 395         for (Iterator<Uuid> iterator = newSubnetIds.iterator(); iterator.hasNext();) {
 396             Uuid updatedSubnetId = iterator.next();
 397             for (Uuid currentSubnetId : origSubnetIds) {
 398                 if (updatedSubnetId.getValue().equals(currentSubnetId.getValue())) {
 399                     iterator.remove();
 400                     break;
 401                 }
 402             }
 403         }
 404         return newSubnetIds;
 405     }
 406 }