2 * Copyright (c) 2017 Red Hat, Inc. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
9 package org.opendaylight.netvirt.natservice.ha;
11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
14 import java.util.ArrayList;
15 import java.util.HashMap;
16 import java.util.Iterator;
17 import java.util.List;
19 import java.util.Map.Entry;
20 import java.util.Optional;
21 import java.util.concurrent.ConcurrentHashMap;
22 import java.util.concurrent.ExecutionException;
23 import javax.inject.Inject;
24 import javax.inject.Singleton;
25 import org.eclipse.jdt.annotation.Nullable;
26 import org.opendaylight.genius.datastoreutils.ExpectedDataObjectNotFoundException;
27 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
28 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
29 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
30 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
31 import org.opendaylight.mdsal.binding.api.DataBroker;
32 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
33 import org.opendaylight.mdsal.common.api.TransactionCommitFailedException;
34 import org.opendaylight.netvirt.natservice.api.CentralizedSwitchScheduler;
35 import org.opendaylight.netvirt.natservice.api.NatSwitchCache;
36 import org.opendaylight.netvirt.natservice.api.NatSwitchCacheListener;
37 import org.opendaylight.netvirt.natservice.api.SwitchInfo;
38 import org.opendaylight.netvirt.natservice.internal.NatUtil;
39 import org.opendaylight.netvirt.vpnmanager.api.IVpnFootprintService;
40 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
41 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
42 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.config.rev170206.NatserviceConfig;
43 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ExtRouters;
44 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.NaptSwitches;
45 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
46 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.routers.ExternalIps;
47 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.routers.ExternalIpsKey;
48 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitch;
49 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchBuilder;
50 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.napt.switches.RouterToNaptSwitchKey;
51 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes;
52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.Subnetmaps;
53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
54 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.SubnetmapKey;
55 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
56 import org.opendaylight.yangtools.yang.common.Uint64;
57 import org.slf4j.Logger;
58 import org.slf4j.LoggerFactory;
61 public class WeightedCentralizedSwitchScheduler implements CentralizedSwitchScheduler, NatSwitchCacheListener {
62 private static final Logger LOG = LoggerFactory.getLogger(WeightedCentralizedSwitchScheduler.class);
63 private static final Integer INITIAL_SWITCH_WEIGHT = Integer.valueOf(0);
65 private final Map<String, Map<Uint64,Integer>> providerSwitchWeightsMap = new ConcurrentHashMap<>();
66 private final Map<String,String> subnetIdToRouterPortMap = new ConcurrentHashMap<>();
67 private final Map<String,String> subnetIdToElanInstanceMap = new ConcurrentHashMap<>();
68 private final DataBroker dataBroker;
69 private final ManagedNewTransactionRunner txRunner;
70 private final OdlInterfaceRpcService interfaceManager;
71 private final IVpnFootprintService vpnFootprintService;
72 private final NatserviceConfig.NatMode natMode;
75 public WeightedCentralizedSwitchScheduler(final DataBroker dataBroker,
76 final OdlInterfaceRpcService interfaceManager,
77 final IVpnFootprintService vpnFootprintService, final NatserviceConfig config,
78 final NatSwitchCache natSwitchCache) {
79 this.dataBroker = dataBroker;
80 this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
81 this.interfaceManager = interfaceManager;
82 this.vpnFootprintService = vpnFootprintService;
84 this.natMode = config.getNatMode();
86 this.natMode = NatserviceConfig.NatMode.Controller;
88 natSwitchCache.register(this);
92 public boolean scheduleCentralizedSwitch(Routers router) {
93 String providerNet = NatUtil.getElanInstancePhysicalNetwok(router.getNetworkId().getValue(),dataBroker);
94 Uint64 nextSwitchId = getSwitchWithLowestWeight(providerNet);
95 if (Uint64.ZERO.equals(nextSwitchId)) {
96 LOG.error("In scheduleCentralizedSwitch, unable to schedule the router {} as there is no available switch.",
97 router.getRouterName());
101 LOG.info("scheduleCentralizedSwitch for router {} on switch {}", router.getRouterName(), nextSwitchId);
102 String routerName = router.getRouterName();
103 RouterToNaptSwitchBuilder routerToNaptSwitchBuilder =
104 new RouterToNaptSwitchBuilder().setRouterName(routerName);
105 RouterToNaptSwitch id = routerToNaptSwitchBuilder.setPrimarySwitchId(nextSwitchId)
106 .setEnableSnat(router.isEnableSnat()).build();
107 addToDpnMaps(routerName, router.getSubnetIds(), nextSwitchId);
109 SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
110 getNaptSwitchesIdentifier(routerName), id);
111 Map<Uint64,Integer> switchWeightMap = providerSwitchWeightsMap.get(providerNet);
112 switchWeightMap.put(nextSwitchId,switchWeightMap.get(nextSwitchId) + 1);
114 } catch (TransactionCommitFailedException e) {
115 LOG.error("ScheduleCentralizedSwitch failed for {}", routerName);
122 public boolean updateCentralizedSwitch(Routers oldRouter, Routers newRouter) {
123 LOG.info("updateCentralizedSwitch for router {}", newRouter.getRouterName());
124 String routerName = newRouter.getRouterName();
125 List<Uuid> addedSubnetIds = getUpdatedSubnetIds(newRouter.getSubnetIds(), oldRouter.getSubnetIds());
126 List<Uuid> deletedSubnetIds = getUpdatedSubnetIds(oldRouter.getSubnetIds(), newRouter.getSubnetIds());
127 Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, newRouter.getRouterName());
128 addToDpnMaps(routerName, addedSubnetIds, primarySwitchId);
129 deleteFromDpnMaps(routerName, deletedSubnetIds, primarySwitchId);
131 InstanceIdentifier<RouterToNaptSwitch> id = NatUtil.buildNaptSwitchIdentifier(routerName);
132 RouterToNaptSwitch routerToNaptSwitch = SingleTransactionDataBroker.syncRead(dataBroker,
133 LogicalDatastoreType.CONFIGURATION, id);
134 boolean isSnatEnabled = newRouter.isEnableSnat();
135 Map<ExternalIpsKey, ExternalIps> updateExternalIpsMap = newRouter.getExternalIps();
136 if (updateExternalIpsMap == null || updateExternalIpsMap.isEmpty()) {
137 isSnatEnabled = false;
139 if (isSnatEnabled != routerToNaptSwitch.isEnableSnat()) {
140 RouterToNaptSwitchBuilder routerToNaptSwitchBuilder =
141 new RouterToNaptSwitchBuilder(routerToNaptSwitch);
142 routerToNaptSwitchBuilder.setEnableSnat(isSnatEnabled);
143 SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION,
144 getNaptSwitchesIdentifier(routerName), routerToNaptSwitchBuilder.build());
146 } catch (ExpectedDataObjectNotFoundException e) {
147 LOG.error("updateCentralizedSwitch ReadFailedException for {}", routerName);
148 } catch (TransactionCommitFailedException e) {
149 LOG.error("updateCentralizedSwitch TransactionCommitFailedException for {}", routerName);
155 public boolean releaseCentralizedSwitch(Routers router) {
156 String providerNet = NatUtil.getElanInstancePhysicalNetwok(router.getNetworkId().getValue(),dataBroker);
157 String routerName = router.getRouterName();
158 Uint64 primarySwitchId = NatUtil.getPrimaryNaptfromRouterName(dataBroker, routerName);
159 if (primarySwitchId == null || Uint64.ZERO.equals(primarySwitchId)) {
160 LOG.info("releaseCentralizedSwitch: NAPT Switch is not allocated for router {}", router.getRouterName());
164 LOG.info("releaseCentralizedSwitch for router {} from switch {}", router.getRouterName(), primarySwitchId);
165 deleteFromDpnMaps(routerName, router.getSubnetIds(), primarySwitchId);
167 SingleTransactionDataBroker.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION,
168 getNaptSwitchesIdentifier(routerName));
169 Map<Uint64,Integer> switchWeightMap = providerSwitchWeightsMap.get(providerNet);
170 switchWeightMap.put(primarySwitchId, switchWeightMap.get(primarySwitchId) - 1);
171 } catch (TransactionCommitFailedException e) {
177 private void addToDpnMaps(String routerName, List<Uuid> addedSubnetIds, Uint64 primarySwitchId) {
178 if (addedSubnetIds == null || addedSubnetIds.isEmpty()) {
179 LOG.debug("addToDpnMaps no subnets associated with {}", routerName);
182 Map<Uuid, Subnetmap> subnetMapEntries = new HashMap<>();
184 String primaryRd = txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
185 for (Uuid subnetUuid : addedSubnetIds) {
186 Subnetmap subnetMapEntry = tx.read(getSubnetMapIdentifier(subnetUuid)).get().orElse(null);
187 subnetMapEntries.put(subnetUuid, subnetMapEntry);
188 Uuid routerPortUuid = subnetMapEntry.getRouterInterfacePortId();
189 subnetIdToRouterPortMap.put(subnetUuid.getValue(), routerPortUuid.getValue());
191 return NatUtil.getPrimaryRd(routerName, tx);
193 ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
194 for (Uuid subnetUuid : addedSubnetIds) {
195 Subnetmap subnetMapEntry = subnetMapEntries.get(subnetUuid);
196 Uuid routerPortUuid = subnetMapEntry.getRouterInterfacePortId();
197 vpnFootprintService.updateVpnToDpnMapping(primarySwitchId, routerName, primaryRd,
198 routerPortUuid.getValue(), null, true);
199 NatUtil.addToNeutronRouterDpnsMap(routerName, routerPortUuid.getValue(), primarySwitchId, tx);
200 NatUtil.addToDpnRoutersMap(routerName, routerPortUuid.getValue(), primarySwitchId, tx);
201 if (subnetMapEntry.getNetworkType().equals(NetworkAttributes.NetworkType.VLAN)) {
202 String elanInstanceName = subnetMapEntry.getNetworkId().getValue();
203 subnetIdToElanInstanceMap.put(subnetUuid.getValue(), elanInstanceName);
204 NatUtil.addPseudoPortToElanDpn(elanInstanceName, elanInstanceName, primarySwitchId, dataBroker);
207 }), LOG, "Error adding subnets to DPN maps for {}", routerName);
208 } catch (InterruptedException | ExecutionException e) {
209 LOG.error("Error adding subnets to DPN maps for {}", routerName);
213 private void deleteFromDpnMaps(String routerName, List<Uuid> deletedSubnetIds, Uint64 primarySwitchId) {
214 if (deletedSubnetIds == null || deletedSubnetIds.isEmpty()) {
215 LOG.debug("deleteFromDpnMaps no subnets associated with {}", routerName);
219 String primaryRd = txRunner.applyWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
220 tx -> NatUtil.getPrimaryRd(routerName, tx)).get();
221 ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
222 for (Uuid subnetUuid : deletedSubnetIds) {
223 String routerPort = subnetIdToRouterPortMap.remove(subnetUuid.getValue());
224 if (routerPort == null) {
225 LOG.error("The router port was not found for {}", subnetUuid.getValue());
228 vpnFootprintService.updateVpnToDpnMapping(primarySwitchId, routerName, primaryRd,
229 routerPort, null, false);
230 NatUtil.removeFromNeutronRouterDpnsMap(routerName, primarySwitchId, tx);
231 NatUtil.removeFromDpnRoutersMap(dataBroker, routerName, routerName,
232 primarySwitchId, interfaceManager, tx);
233 if (subnetIdToElanInstanceMap.containsKey(subnetUuid.getValue())) {
234 String elanInstanceName = subnetIdToElanInstanceMap.remove(subnetUuid.getValue());
235 NatUtil.removePseudoPortFromElanDpn(elanInstanceName, elanInstanceName, primarySwitchId,
239 }), LOG, "Error deleting subnets from DPN maps for {}", routerName);
240 } catch (InterruptedException | ExecutionException e) {
241 LOG.error("Error deleting subnets from DPN maps for {}", routerName, e);
246 public void switchAddedToCache(SwitchInfo switchInfo) {
247 boolean scheduleRouters = (providerSwitchWeightsMap.size() == 0) ? true : false;
248 for (String providerNet : switchInfo.getProviderNets()) {
249 Map<Uint64,Integer> switchWeightMap = providerSwitchWeightsMap.get(providerNet);
250 if (providerSwitchWeightsMap.get(providerNet) == null) {
251 switchWeightMap = new ConcurrentHashMap<>();
252 providerSwitchWeightsMap.put(providerNet, switchWeightMap);
254 LOG.info("addSwitch: Adding {} dpnId with provider mapping {} to switchWeightsMap",
255 switchInfo.getDpnId(), providerNet);
256 switchWeightMap.put(switchInfo.getDpnId(), INITIAL_SWITCH_WEIGHT);
258 if (natMode == NatserviceConfig.NatMode.Conntrack && scheduleRouters) {
259 Optional<ExtRouters> optRouters;
261 optRouters = SingleTransactionDataBroker.syncReadOptional(dataBroker,
262 LogicalDatastoreType.CONFIGURATION, InstanceIdentifier.create(ExtRouters.class));
263 } catch (InterruptedException | ExecutionException e) {
264 LOG.error("addSwitch: Error reading external routers", e);
268 if (optRouters.isPresent()) {
269 // Get the list of routers and verify if any routers do not have primarySwitch allocated.
270 for (Routers router : optRouters.get().getRouters().values()) {
271 Map<ExternalIpsKey, ExternalIps> keyExternalIpsMap = router.getExternalIps();
272 if (router.isEnableSnat() && keyExternalIpsMap != null && !keyExternalIpsMap.isEmpty()) {
273 // Check if the primarySwitch is allocated for the router.
274 if (!isPrimarySwitchAllocatedForRouter(router.getRouterName())) {
275 scheduleCentralizedSwitch(router);
284 private boolean isPrimarySwitchAllocatedForRouter(String routerName) {
285 InstanceIdentifier<RouterToNaptSwitch> routerToNaptSwitch =
286 NatUtil.buildNaptSwitchRouterIdentifier(routerName);
288 RouterToNaptSwitch rtrToNapt = SingleTransactionDataBroker.syncRead(dataBroker,
289 LogicalDatastoreType.CONFIGURATION, routerToNaptSwitch);
290 Uint64 dpnId = rtrToNapt.getPrimarySwitchId();
291 if (dpnId == null || dpnId.equals(Uint64.ZERO)) {
294 } catch (ExpectedDataObjectNotFoundException e) {
295 LOG.error("isPrimarySwitchAllocatedForRouter: Error reading RouterToNaptSwitch model", e);
302 public void switchRemovedFromCache(SwitchInfo switchInfo) {
303 Uint64 dpnId = switchInfo.getDpnId();
304 LOG.info("removeSwitch: Removing {} dpnId to switchWeightsMap", dpnId);
305 for (Map.Entry<String,Map<Uint64,Integer>> providerNet : providerSwitchWeightsMap.entrySet()) {
306 Map<Uint64,Integer> switchWeightMap = providerNet.getValue();
307 if (natMode == NatserviceConfig.NatMode.Conntrack
308 && !INITIAL_SWITCH_WEIGHT.equals(switchWeightMap.get(dpnId))) {
309 NaptSwitches naptSwitches = getNaptSwitches();
310 for (RouterToNaptSwitch routerToNaptSwitch : naptSwitches.getRouterToNaptSwitch().values()) {
311 if (dpnId.equals(routerToNaptSwitch.getPrimarySwitchId())) {
312 Routers router = NatUtil.getRoutersFromConfigDS(dataBroker, routerToNaptSwitch.getRouterName());
313 releaseCentralizedSwitch(router);
314 scheduleCentralizedSwitch(router);
319 switchWeightMap.remove(dpnId);
324 private NaptSwitches getNaptSwitches() {
325 InstanceIdentifier<NaptSwitches> id = InstanceIdentifier.builder(NaptSwitches.class).build();
327 return SingleTransactionDataBroker.syncReadOptional(dataBroker,
328 LogicalDatastoreType.CONFIGURATION, id).orElse(null);
329 } catch (ExecutionException | InterruptedException e) {
330 LOG.error("getNaptSwitches: Exception while reading Napt-Switch DS", e);
335 private Uint64 getSwitchWithLowestWeight(String providerNet) {
336 int lowestWeight = Integer.MAX_VALUE;
337 Uint64 nextSwitchId = Uint64.valueOf(0);
338 Map<Uint64,Integer> switchWeightMap = providerSwitchWeightsMap.get(providerNet);
339 if (null == switchWeightMap) {
340 LOG.error("No switch have the provider mapping {}", providerNet);
343 for (Entry<Uint64, Integer> entry : switchWeightMap.entrySet()) {
344 Uint64 dpnId = entry.getKey();
345 Integer weight = entry.getValue();
346 if (lowestWeight > weight) {
347 lowestWeight = weight;
348 nextSwitchId = dpnId;
351 LOG.info("getSwitchWithLowestWeight: switchWeightsMap {}, returning nextSwitchId {} ",
352 providerSwitchWeightsMap, nextSwitchId);
356 private InstanceIdentifier<RouterToNaptSwitch> getNaptSwitchesIdentifier(String routerName) {
357 return InstanceIdentifier.builder(NaptSwitches.class)
358 .child(RouterToNaptSwitch.class, new RouterToNaptSwitchKey(routerName)).build();
361 private InstanceIdentifier<Subnetmap> getSubnetMapIdentifier(Uuid subnetId) {
362 return InstanceIdentifier.builder(Subnetmaps.class).child(Subnetmap.class,
363 new SubnetmapKey(subnetId)).build();
367 public Uint64 getCentralizedSwitch(String routerName) {
369 Optional<RouterToNaptSwitch> naptSwitches = SingleTransactionDataBroker
370 .syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
371 getNaptSwitchesIdentifier(routerName));
372 if (!naptSwitches.isPresent()) {
373 LOG.info("No Napt switch is scheduled for {}", routerName);
376 return naptSwitches.get().getPrimarySwitchId();
377 } catch (InterruptedException | ExecutionException e) {
378 LOG.error("Error reading RouterToNaptSwitch model", e);
384 private static List<Uuid> getUpdatedSubnetIds(List<Uuid> updatedSubnetIds, List<Uuid> currentSubnetIds) {
385 if (updatedSubnetIds == null) {
388 List<Uuid> newSubnetIds = new ArrayList<>(updatedSubnetIds);
389 if (currentSubnetIds == null) {
392 List<Uuid> origSubnetIds = new ArrayList<>(currentSubnetIds);
393 for (Iterator<Uuid> iterator = newSubnetIds.iterator(); iterator.hasNext();) {
394 Uuid updatedSubnetId = iterator.next();
395 for (Uuid currentSubnetId : origSubnetIds) {
396 if (updatedSubnetId.getValue().equals(currentSubnetId.getValue())) {