2 * Copyright (c) 2015 - 2017 Ericsson India Global Services Pvt Ltd. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.netvirt.vpnmanager;
10 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
11 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
13 import com.google.common.base.Optional;
14 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
15 import java.util.ArrayList;
16 import java.util.Collection;
17 import java.util.Collections;
18 import java.util.Iterator;
19 import java.util.List;
20 import java.util.Objects;
22 import java.util.concurrent.ExecutionException;
23 import java.util.concurrent.Future;
24 import java.util.concurrent.locks.ReentrantLock;
25 import javax.annotation.PostConstruct;
26 import javax.inject.Inject;
27 import javax.inject.Singleton;
28 import org.eclipse.jdt.annotation.NonNull;
29 import org.eclipse.jdt.annotation.Nullable;
30 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
31 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
32 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
33 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
34 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
35 import org.opendaylight.genius.infra.Datastore.Configuration;
36 import org.opendaylight.genius.infra.Datastore.Operational;
37 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
38 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
39 import org.opendaylight.genius.infra.TypedReadTransaction;
40 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
41 import org.opendaylight.genius.infra.TypedWriteTransaction;
42 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
43 import org.opendaylight.genius.mdsalutil.FlowEntity;
44 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
45 import org.opendaylight.genius.mdsalutil.NwConstants;
46 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
47 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
48 import org.opendaylight.genius.utils.JvmGlobalLocks;
49 import org.opendaylight.infrautils.utils.concurrent.LoggingFutures;
50 import org.opendaylight.infrautils.utils.function.InterruptibleCheckedConsumer;
51 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
52 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput;
53 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput.ArpReponderInputBuilder;
54 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderUtil;
55 import org.opendaylight.netvirt.elanmanager.api.IElanService;
56 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
57 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
58 import org.opendaylight.netvirt.neutronvpn.api.enums.IpVersionChoice;
59 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
60 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
61 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
62 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
63 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
64 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
65 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
66 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
67 import org.opendaylight.serviceutils.upgrade.UpgradeState;
68 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
69 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.vpntargets.VpnTarget;
70 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
71 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
72 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
73 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInput;
75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInputBuilder;
76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolOutput;
77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.SubnetsAssociatedToRouteTargets;
83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.RouteTarget;
85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.RouteTargetKey;
86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.route.target.AssociatedSubnet;
87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.route.target.associated.subnet.AssociatedVpn;
88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
91 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
92 import org.opendaylight.yangtools.yang.common.RpcError;
93 import org.opendaylight.yangtools.yang.common.RpcResult;
94 import org.opendaylight.yangtools.yang.common.Uint32;
95 import org.opendaylight.yangtools.yang.common.Uint64;
96 import org.slf4j.Logger;
97 import org.slf4j.LoggerFactory;
100 public class VpnManagerImpl implements IVpnManager {
102 private static final Logger LOG = LoggerFactory.getLogger(VpnManagerImpl.class);
103 private final DataBroker dataBroker;
104 private final ManagedNewTransactionRunner txRunner;
105 private final IdManagerService idManager;
106 private final IMdsalApiManager mdsalManager;
107 private final IElanService elanService;
108 private final IInterfaceManager interfaceManager;
109 private final VpnSubnetRouteHandler vpnSubnetRouteHandler;
110 private final OdlInterfaceRpcService ifaceMgrRpcService;
111 private final IVpnLinkService ivpnLinkService;
112 private final IFibManager fibManager;
113 private final IBgpManager bgpManager;
114 private final InterVpnLinkCache interVpnLinkCache;
115 private final DataTreeEventCallbackRegistrar eventCallbacks;
116 private final UpgradeState upgradeState;
117 private final ItmRpcService itmRpcService;
118 private final VpnUtil vpnUtil;
121 public VpnManagerImpl(final DataBroker dataBroker,
122 final IdManagerService idManagerService,
123 final IMdsalApiManager mdsalManager,
124 final IElanService elanService,
125 final IInterfaceManager interfaceManager,
126 final VpnSubnetRouteHandler vpnSubnetRouteHandler,
127 final OdlInterfaceRpcService ifaceMgrRpcService,
128 final IVpnLinkService ivpnLinkService,
129 final IFibManager fibManager,
130 final IBgpManager bgpManager,
131 final InterVpnLinkCache interVpnLinkCache,
132 final DataTreeEventCallbackRegistrar dataTreeEventCallbackRegistrar,
133 final UpgradeState upgradeState,
134 final ItmRpcService itmRpcService,
135 final VpnUtil vpnUtil) {
136 this.dataBroker = dataBroker;
137 this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
138 this.idManager = idManagerService;
139 this.mdsalManager = mdsalManager;
140 this.elanService = elanService;
141 this.interfaceManager = interfaceManager;
142 this.vpnSubnetRouteHandler = vpnSubnetRouteHandler;
143 this.ifaceMgrRpcService = ifaceMgrRpcService;
144 this.ivpnLinkService = ivpnLinkService;
145 this.fibManager = fibManager;
146 this.bgpManager = bgpManager;
147 this.interVpnLinkCache = interVpnLinkCache;
148 this.eventCallbacks = dataTreeEventCallbackRegistrar;
149 this.upgradeState = upgradeState;
150 this.itmRpcService = itmRpcService;
151 this.vpnUtil = vpnUtil;
155 public void start() {
156 LOG.info("{} start", getClass().getSimpleName());
160 private void createIdPool() {
161 CreateIdPoolInput createPool = new CreateIdPoolInputBuilder()
162 .setPoolName(VpnConstants.VPN_IDPOOL_NAME)
163 .setLow(VpnConstants.VPN_IDPOOL_LOW)
164 .setHigh(VpnConstants.VPN_IDPOOL_HIGH)
167 Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPool);
168 if (result != null && result.get().isSuccessful()) {
169 LOG.info("Created IdPool for VPN Service");
171 LOG.error("createIdPool: Unable to create ID pool for VPNService");
173 } catch (InterruptedException | ExecutionException e) {
174 LOG.error("Failed to create idPool for VPN Service", e);
177 // Now an IdPool for InterVpnLink endpoint's pseudo ports
178 CreateIdPoolInput createPseudoLporTagPool =
179 new CreateIdPoolInputBuilder().setPoolName(VpnConstants.PSEUDO_LPORT_TAG_ID_POOL_NAME)
180 .setLow(VpnConstants.LOWER_PSEUDO_LPORT_TAG)
181 .setHigh(VpnConstants.UPPER_PSEUDO_LPORT_TAG)
184 Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPseudoLporTagPool);
185 if (result != null && result.get().isSuccessful()) {
186 LOG.debug("Created IdPool for Pseudo Port tags");
188 StringBuilder errMsg = new StringBuilder();
189 if (result != null && result.get() != null) {
190 Collection<RpcError> errors = result.get().getErrors();
191 for (RpcError err : errors) {
192 errMsg.append(err.getMessage()).append("\n");
195 LOG.error("IdPool creation for PseudoPort tags failed. Reasons: {}", errMsg);
197 } catch (InterruptedException | ExecutionException e) {
198 LOG.error("Failed to create idPool for Pseudo Port tags", e);
203 public void addExtraRoute(String vpnName, String destination, String nextHop, String rd, @Nullable String routerID,
204 Uint32 l3vni, RouteOrigin origin, @Nullable String intfName, @Nullable Adjacency operationalAdj,
205 VrfEntry.EncapType encapType, Set<String> prefixListForRefreshFib,
206 @NonNull TypedWriteTransaction<Configuration> confTx) {
207 //add extra route to vpn mapping; advertise with nexthop as tunnel ip
208 vpnUtil.syncUpdate(LogicalDatastoreType.OPERATIONAL,
209 VpnExtraRouteHelper.getVpnToExtrarouteVrfIdIdentifier(vpnName, rd != null ? rd : routerID,
211 VpnUtil.getVpnToExtraroute(destination, Collections.singletonList(nextHop)));
214 if (intfName != null && !intfName.isEmpty()) {
215 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
216 String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
217 if (nextHopIp == null || nextHopIp.isEmpty()) {
218 LOG.error("addExtraRoute: NextHop for interface {} is null / empty."
219 + " Failed advertising extra route for rd {} prefix {} dpn {}", intfName, rd, destination,
226 String primaryRd = vpnUtil.getPrimaryRd(vpnName);
228 // TODO: This is a limitation to be stated in docs. When configuring static route to go to
229 // another VPN, there can only be one nexthop or, at least, the nexthop to the interVpnLink should be in
231 Optional<InterVpnLinkDataComposite> optVpnLink = interVpnLinkCache.getInterVpnLinkByEndpoint(nextHop);
232 if (optVpnLink.isPresent() && optVpnLink.get().isActive()) {
233 InterVpnLinkDataComposite interVpnLink = optVpnLink.get();
234 // If the nexthop is the endpoint of Vpn2, then prefix must be advertised to Vpn1 in DC-GW, with nexthops
235 // pointing to the DPNs where Vpn1 is instantiated. LFIB in these DPNS must have a flow entry, with lower
236 // priority, where if Label matches then sets the lportTag of the Vpn2 endpoint and goes to LportDispatcher
237 // This is like leaking one of the Vpn2 routes towards Vpn1
238 String srcVpnUuid = interVpnLink.getVpnNameByIpAddress(nextHop);
239 String dstVpnUuid = interVpnLink.getOtherVpnNameByIpAddress(nextHop);
240 String dstVpnRd = vpnUtil.getVpnRd(dstVpnUuid);
241 Uint32 newLabel = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
242 VpnUtil.getNextHopLabelKey(dstVpnRd, destination));
243 if (newLabel.longValue() == VpnConstants.INVALID_LABEL) {
244 LOG.error("addExtraRoute: Unable to fetch label from Id Manager. Bailing out of adding intervpnlink"
245 + " route for destination {}", destination);
248 ivpnLinkService.leakRoute(interVpnLink, srcVpnUuid, dstVpnUuid, destination, newLabel, RouteOrigin.STATIC);
250 Optional<Routes> optVpnExtraRoutes = VpnExtraRouteHelper
251 .getVpnExtraroutes(dataBroker, vpnName, rd != null ? rd : routerID, destination);
252 if (optVpnExtraRoutes.isPresent()) {
253 List<String> nhList = optVpnExtraRoutes.get().getNexthopIpList();
254 if (nhList != null && nhList.size() > 1) {
255 // If nhList is greater than one for vpnextraroute, a call to populatefib doesn't update vrfentry.
256 prefixListForRefreshFib.add(destination);
258 L3vpnInput input = new L3vpnInput().setNextHop(operationalAdj).setNextHopIp(nextHop)
259 .setL3vni(l3vni.longValue())
260 .setPrimaryRd(primaryRd).setVpnName(vpnName).setDpnId(dpnId)
261 .setEncapType(encapType).setRd(rd).setRouteOrigin(origin);
262 L3vpnRegistry.getRegisteredPopulator(encapType).populateFib(input, confTx);
269 public void delExtraRoute(String vpnName, String destination, String nextHop, String rd, @Nullable String routerID,
270 @Nullable String intfName, @NonNull TypedWriteTransaction<Configuration> confTx,
271 @NonNull TypedWriteTransaction<Operational> operTx) {
273 String tunnelIp = nextHop;
274 if (intfName != null && !intfName.isEmpty()) {
275 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
276 String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
277 if (nextHopIp == null || nextHopIp.isEmpty()) {
278 LOG.error("delExtraRoute: NextHop for interface {} is null / empty."
279 + " Failed advertising extra route for rd {} prefix {} dpn {}", intfName, rd, destination,
282 tunnelIp = nextHopIp;
285 String primaryRd = vpnUtil.getVpnRd(vpnName);
286 removePrefixFromBGP(vpnName, primaryRd, rd, intfName, destination,
287 nextHop, tunnelIp, dpnId, confTx, operTx);
288 LOG.info("delExtraRoute: Removed extra route {} from interface {} for rd {}", destination, intfName, rd);
290 // add FIB route directly
291 fibManager.removeOrUpdateFibEntry(routerID, destination, tunnelIp, confTx);
292 LOG.info("delExtraRoute: Removed extra route {} from interface {} for rd {}", destination, intfName,
298 @SuppressWarnings("checkstyle:IllegalCatch")
299 public void removePrefixFromBGP(String vpnName, String primaryRd, String extraRouteRd, String vpnInterfaceName,
300 String prefix, String nextHop, String nextHopTunnelIp, Uint64 dpnId,
301 TypedWriteTransaction<Configuration> confTx,
302 TypedWriteTransaction<Operational> operTx) {
303 String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
304 // FIXME: separate out to somehow?
305 final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnNamePrefixKey);
306 LOG.info("removing prefix {} for nexthop {} in VPN {} rd {}", prefix, nextHop, vpnName, extraRouteRd);
309 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, extraRouteRd, vpnInterfaceName, prefix,
310 nextHop, nextHopTunnelIp, operTx)) {
313 fibManager.removeOrUpdateFibEntry(primaryRd, prefix, nextHopTunnelIp, confTx);
314 if (VpnUtil.isEligibleForBgp(extraRouteRd, vpnName, dpnId, null /*networkName*/)) {
315 // TODO: Might be needed to include nextHop here
316 bgpManager.withdrawPrefix(extraRouteRd, prefix);
318 LOG.info("removePrefixFromBGP: VPN WITHDRAW: Removed Fib Entry rd {} prefix {} nexthop {}", extraRouteRd,
320 } catch (RuntimeException e) {
321 LOG.error("removePrefixFromBGP: Delete prefix {} rd {} nextHop {} failed", prefix, extraRouteRd, nextHop);
328 public boolean isVPNConfigured() {
329 InstanceIdentifier<VpnInstances> vpnsIdentifier = InstanceIdentifier.builder(VpnInstances.class).build();
331 Optional<VpnInstances> optionalVpns =
332 SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
334 if (!optionalVpns.isPresent()
335 || optionalVpns.get().getVpnInstance() == null
336 || optionalVpns.get().getVpnInstance().isEmpty()) {
337 LOG.trace("isVPNConfigured: No VPNs configured.");
340 } catch (ReadFailedException e) {
341 throw new RuntimeException("Error reading VPN " + vpnsIdentifier, e);
343 LOG.trace("isVPNConfigured: VPNs are configured on the system.");
348 public void addSubnetMacIntoVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
349 Uint64 dpnId, TypedWriteTransaction<Configuration> confTx)
350 throws ExecutionException, InterruptedException {
351 setupSubnetMacInVpnInstance(vpnName, subnetVpnName, srcMacAddress, dpnId,
352 (vpnId, dpId, subnetVpnId) -> addGwMac(srcMacAddress, confTx, vpnId, dpId, subnetVpnId));
356 public void removeSubnetMacFromVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
357 Uint64 dpnId, TypedReadWriteTransaction<Configuration> confTx)
358 throws ExecutionException, InterruptedException {
359 setupSubnetMacInVpnInstance(vpnName, subnetVpnName, srcMacAddress, dpnId,
360 (vpnId, dpId, subnetVpnId) -> removeGwMac(srcMacAddress, confTx, vpnId, dpId, subnetVpnId));
364 private interface VpnInstanceSubnetMacSetupMethod {
365 void process(Uint32 vpnId, Uint64 dpId, Uint32 subnetVpnId) throws InterruptedException, ExecutionException;
368 private void setupSubnetMacInVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
369 Uint64 dpnId, VpnInstanceSubnetMacSetupMethod consumer)
370 throws ExecutionException, InterruptedException {
371 if (vpnName == null) {
372 LOG.warn("Cannot setup subnet MAC {} on DPN {}, null vpnName", srcMacAddress, dpnId);
376 Uint32 vpnId = vpnUtil.getVpnId(vpnName);
377 Uint32 subnetVpnId = vpnUtil.getVpnId(subnetVpnName);
378 if (dpnId.equals(Uint64.ZERO)) {
379 /* Apply the MAC on all DPNs in a VPN */
380 for (Uint64 dpId : vpnUtil.getDpnsOnVpn(vpnName)) {
381 consumer.process(vpnId, dpId, subnetVpnId);
384 consumer.process(vpnId, dpnId, subnetVpnId);
388 private void addGwMac(String srcMacAddress, TypedWriteTransaction<Configuration> tx, Uint32 vpnId, Uint64 dpId,
389 Uint32 subnetVpnId) {
390 FlowEntity flowEntity = vpnUtil.buildL3vpnGatewayFlow(dpId, srcMacAddress, vpnId, subnetVpnId);
391 mdsalManager.addFlow(tx, flowEntity);
394 // TODO skitt Fix the exception handling here
395 @SuppressWarnings("checkstyle:IllegalCatch")
396 @SuppressFBWarnings("REC_CATCH_EXCEPTION")
397 private void removeGwMac(String srcMacAddress, TypedReadWriteTransaction<Configuration> tx, Uint32 vpnId,
398 Uint64 dpId, Uint32 subnetVpnId) throws ExecutionException, InterruptedException {
399 mdsalManager.removeFlow(tx, dpId,
400 VpnUtil.getL3VpnGatewayFlowRef(NwConstants.L3_GW_MAC_TABLE, dpId, vpnId, srcMacAddress, subnetVpnId),
401 NwConstants.L3_GW_MAC_TABLE);
405 public void addRouterGwMacFlow(String routerName, String routerGwMac, Uint64 dpnId, Uuid extNetworkId,
406 String subnetVpnName, TypedWriteTransaction<Configuration> confTx)
407 throws ExecutionException, InterruptedException {
408 setupRouterGwMacFlow(routerName, routerGwMac, dpnId, extNetworkId,
409 vpnId -> addSubnetMacIntoVpnInstance(vpnId, subnetVpnName, routerGwMac, dpnId, confTx), "Installing");
413 public void removeRouterGwMacFlow(String routerName, String routerGwMac, Uint64 dpnId, Uuid extNetworkId,
414 String subnetVpnName, TypedReadWriteTransaction<Configuration> confTx)
415 throws ExecutionException, InterruptedException {
416 setupRouterGwMacFlow(routerName, routerGwMac, dpnId, extNetworkId,
417 vpnId -> removeSubnetMacFromVpnInstance(vpnId, subnetVpnName, routerGwMac, dpnId, confTx), "Removing");
420 private void setupRouterGwMacFlow(String routerName, String routerGwMac, Uint64 dpnId, Uuid extNetworkId,
421 InterruptibleCheckedConsumer<String, ExecutionException> consumer, String operation)
422 throws ExecutionException, InterruptedException {
423 if (routerGwMac == null) {
424 LOG.warn("Failed to handle router GW flow in GW-MAC table. MAC address is missing for router-id {}",
429 if (dpnId == null || Uint64.ZERO.equals(dpnId)) {
430 LOG.info("setupRouterGwMacFlow: DPN id is missing for router-id {}",
435 Uuid vpnId = vpnUtil.getExternalNetworkVpnId(extNetworkId);
437 LOG.warn("Network {} is not associated with VPN", extNetworkId.getValue());
441 LOG.info("{} router GW MAC flow for router-id {} on switch {}", operation, routerName, dpnId);
442 consumer.accept(vpnId.getValue());
446 public void addArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
447 Uint64 dpnId, Uuid extNetworkId) {
449 if (dpnId == null || Uint64.ZERO.equals(dpnId)) {
450 LOG.warn("Failed to install arp responder flows for router {}. DPN id is missing.", id);
454 String extInterfaceName = elanService.getExternalElanInterface(extNetworkId.getValue(), dpnId);
455 if (extInterfaceName != null) {
456 doAddArpResponderFlowsToExternalNetworkIps(
457 id, fixedIps, macAddress, dpnId, extInterfaceName);
461 LOG.warn("Failed to install responder flows for {}. No external interface found for DPN id {}", id, dpnId);
463 if (!upgradeState.isUpgradeInProgress()) {
467 // The following through the end of the function deals with an upgrade scenario where the neutron configuration
468 // is restored before the OVS switches reconnect. In such a case, the elan-dpn-interfaces entries will be
469 // missing from the operational data store. In order to mitigate this we use DataTreeEventCallbackRegistrar
470 // to wait for the exact operational md-sal object we need to contain the external interface we need.
472 LOG.info("Upgrade in process, waiting for an external interface to appear on dpn {} for elan {}",
473 dpnId, extNetworkId.getValue());
475 InstanceIdentifier<DpnInterfaces> dpnInterfacesIid =
476 elanService.getElanDpnInterfaceOperationalDataPath(extNetworkId.getValue(), dpnId);
478 eventCallbacks.onAddOrUpdate(LogicalDatastoreType.OPERATIONAL, dpnInterfacesIid, (unused, alsoUnused) -> {
479 LOG.info("Reattempting write of arp responder for external interfaces for external network {}",
481 DpnInterfaces dpnInterfaces = elanService.getElanInterfaceInfoByElanDpn(extNetworkId.getValue(), dpnId);
482 if (dpnInterfaces == null) {
483 LOG.error("Could not retrieve DpnInterfaces for {}, {}", extNetworkId.getValue(), dpnId);
484 return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
487 String extIfc = null;
488 @Nullable List<String> interfaces = dpnInterfaces.getInterfaces();
489 if (interfaces != null) {
490 for (String dpnInterface : interfaces) {
491 if (interfaceManager.isExternalInterface(dpnInterface)) {
492 extIfc = dpnInterface;
498 if (extIfc == null) {
499 if (upgradeState.isUpgradeInProgress()) {
500 LOG.info("External interface not found yet in elan {} on dpn {}, keep waiting",
501 extNetworkId.getValue(), dpnInterfaces);
502 return DataTreeEventCallbackRegistrar.NextAction.CALL_AGAIN;
504 return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
508 final String extIfcFinal = extIfc;
509 doAddArpResponderFlowsToExternalNetworkIps(id, fixedIps, macAddress, dpnId, extIfcFinal);
511 return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
517 public void addArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
518 Uint64 dpnId, String extInterfaceName, int lportTag) {
519 if (fixedIps == null || fixedIps.isEmpty()) {
520 LOG.debug("No external IPs defined for {}", id);
524 LOG.info("Installing ARP responder flows for {} fixed-ips {} on switch {}", id, fixedIps, dpnId);
526 for (String fixedIp : fixedIps) {
527 IpVersionChoice ipVersionChoice = VpnUtil.getIpVersionFromString(fixedIp);
528 if (ipVersionChoice == IpVersionChoice.IPV6) {
531 installArpResponderFlowsToExternalNetworkIp(macAddress, dpnId, extInterfaceName, lportTag,
536 private void doAddArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
537 Uint64 dpnId, String extInterfaceName) {
538 Interface extInterfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, extInterfaceName);
539 if (extInterfaceState == null) {
540 LOG.debug("No interface state found for interface {}. Delaying responder flows for {}", extInterfaceName,
545 Integer lportTag = extInterfaceState.getIfIndex();
546 if (lportTag == null) {
547 LOG.debug("No Lport tag found for interface {}. Delaying flows for router-id {}", extInterfaceName, id);
551 if (macAddress == null) {
553 LOG.debug("Failed to install arp responder flows for router-gateway-ip {} for router {}."
554 + "External Gw MacAddress is missing.", fixedIps, id);
558 addArpResponderFlowsToExternalNetworkIps(id, fixedIps, macAddress, dpnId, extInterfaceName, lportTag);
562 public void removeArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
563 Uint64 dpnId, Uuid extNetworkId) {
565 if (dpnId == null || Uint64.ZERO.equals(dpnId)) {
566 LOG.warn("Failed to remove arp responder flows for router {}. DPN id is missing.", id);
570 String extInterfaceName = elanService.getExternalElanInterface(extNetworkId.getValue(), dpnId);
571 if (extInterfaceName == null) {
572 LOG.warn("Failed to remove responder flows for {}. No external interface found for DPN id {}", id, dpnId);
576 Interface extInterfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, extInterfaceName);
577 if (extInterfaceState == null) {
578 LOG.debug("No interface state found for interface {}. Delaying responder flows for {}", extInterfaceName,
583 Integer lportTag = extInterfaceState.getIfIndex();
584 if (lportTag == null) {
585 LOG.debug("No Lport tag found for interface {}. Delaying flows for router-id {}", extInterfaceName, id);
589 if (macAddress == null) {
591 LOG.debug("Failed to remove arp responder flows for router-gateway-ip {} for router {}."
592 + "External Gw MacAddress is missing.", fixedIps, id);
596 removeArpResponderFlowsToExternalNetworkIps(id, fixedIps, dpnId,
597 extInterfaceName, lportTag);
601 public void removeArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps,
602 Uint64 dpnId, String extInterfaceName, int lportTag) {
603 if (fixedIps == null || fixedIps.isEmpty()) {
604 LOG.debug("No external IPs defined for {}", id);
608 LOG.info("Removing ARP responder flows for {} fixed-ips {} on switch {}", id, fixedIps, dpnId);
610 for (String fixedIp : fixedIps) {
611 removeArpResponderFlowsToExternalNetworkIp(dpnId, lportTag, fixedIp, extInterfaceName);
616 public String getPrimaryRdFromVpnInstance(VpnInstance vpnInstance) {
617 return VpnUtil.getPrimaryRd(vpnInstance);
620 private void installArpResponderFlowsToExternalNetworkIp(String macAddress, Uint64 dpnId,
621 String extInterfaceName, int lportTag, String fixedIp) {
622 // reset the split-horizon bit to allow traffic to be sent back to the
624 List<Instruction> instructions = new ArrayList<>();
626 new InstructionWriteMetadata(Uint64.ZERO,
627 MetaDataUtil.METADATA_MASK_SH_FLAG).buildInstruction(1));
629 ArpResponderUtil.getExtInterfaceInstructions(interfaceManager, itmRpcService, extInterfaceName,
630 fixedIp, macAddress));
631 ArpReponderInputBuilder builder = new ArpReponderInputBuilder().setDpId(dpnId.toJava())
632 .setInterfaceName(extInterfaceName).setSpa(fixedIp).setSha(macAddress).setLportTag(lportTag);
633 builder.setInstructions(instructions);
634 elanService.addArpResponderFlow(builder.buildForInstallFlow());
637 private void removeArpResponderFlowsToExternalNetworkIp(Uint64 dpnId, Integer lportTag, String fixedIp,
638 String extInterfaceName) {
639 ArpResponderInput arpInput = new ArpReponderInputBuilder()
640 .setDpId(dpnId.toJava()).setInterfaceName(extInterfaceName)
641 .setSpa(fixedIp).setLportTag(lportTag).buildForRemoveFlow();
642 elanService.removeArpResponderFlow(arpInput);
646 public void onSubnetAddedToVpn(Subnetmap subnetmap, boolean isBgpVpn, Long elanTag) {
647 vpnSubnetRouteHandler.onSubnetAddedToVpn(subnetmap, isBgpVpn, elanTag);
651 public void onSubnetDeletedFromVpn(Subnetmap subnetmap, boolean isBgpVpn) {
652 vpnSubnetRouteHandler.onSubnetDeletedFromVpn(subnetmap, isBgpVpn);
657 public VpnInstance getVpnInstance(DataBroker broker, String vpnInstanceName) {
658 return vpnUtil.getVpnInstance(vpnInstanceName);
662 public String getVpnRd(TypedReadTransaction<Configuration> confTx, String vpnName) {
663 return VpnUtil.getVpnRd(confTx, vpnName);
667 public String getVpnRd(DataBroker broker, String vpnName) {
668 return vpnUtil.getVpnRd(vpnName);
672 public VpnPortipToPort getNeutronPortFromVpnPortFixedIp(TypedReadTransaction<Configuration> confTx, String vpnName,
674 return VpnUtil.getNeutronPortFromVpnPortFixedIp(confTx, vpnName, fixedIp);
679 public VpnPortipToPort getNeutronPortFromVpnPortFixedIp(DataBroker broker, String vpnName, String fixedIp) {
680 return vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, fixedIp);
684 public Set<VpnTarget> getRtListForVpn(String vpnName) {
685 return vpnUtil.getRtListForVpn(vpnName);
689 public void updateRouteTargetsToSubnetAssociation(Set<VpnTarget> routeTargets, String cidr, String vpnName) {
690 LoggingFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
691 for (VpnTarget rt : routeTargets) {
692 String rtValue = rt.getVrfRTValue();
693 switch (rt.getVrfRTType()) {
694 case ImportExtcommunity:
695 addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr, vpnName,
696 tx, false/*isAssociationRemoved*/);
698 case ExportExtcommunity:
699 addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName,
700 tx, false/*isAssociationRemoved*/);
703 addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr, vpnName,
704 tx, false/*isAssociationRemoved*/);
705 addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName,
706 tx, false/*isAssociationRemoved*/);
709 LOG.error("updateRouteTargetsToSubnetAssociation: Invalid rt-type {} for vpn {}"
710 + " subnet-cidr {}", rt.getVrfRTType(), vpnName, cidr);
714 }), LOG, "updateRouteTargetsToSubnetAssociation: Failed for vpn {} subnet-cidr {}", vpnName, cidr);
715 LOG.info("updateRouteTargetsToSubnetAssociation: Updated RT to subnet association for vpn {} cidr {}",
720 public void removeRouteTargetsToSubnetAssociation(Set<VpnTarget> routeTargets, String cidr, String vpnName) {
721 LoggingFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
722 for (VpnTarget rt : routeTargets) {
723 String rtValue = rt.getVrfRTValue();
724 switch (rt.getVrfRTType()) {
725 case ImportExtcommunity:
726 addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr, vpnName, tx,
727 true/*isAssociationRemoved*/);
729 case ExportExtcommunity:
730 addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName, tx,
731 true/*isAssociationRemoved*/);
734 addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr, vpnName, tx,
735 true/*isAssociationRemoved*/);
736 addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName, tx,
737 true/*isAssociationRemoved*/);
740 LOG.error("removeRouteTargetsToSubnetAssociation: Invalid route-target type {} for vpn {}"
741 + " subnet-cidr {}", rt.getVrfRTType(), vpnName, cidr);
745 }), LOG, "removeRouteTargetsToSubnetAssociation: Failed for vpn {} subnet-cidr {}", vpnName, cidr);
746 LOG.info("removeRouteTargetsToSubnetAssociation: vpn {} cidr {}", vpnName, cidr);
749 private boolean doesExistingVpnsHaveConflictingSubnet(Set<VpnTarget> routeTargets, String subnetCidr) {
750 Set<RouteTarget> routeTargetSet = vpnUtil.getRouteTargetSet(routeTargets);
751 for (RouteTarget routerTarget : routeTargetSet) {
752 if (routerTarget.getAssociatedSubnet() != null) {
753 for (int i = 0; i < routerTarget.getAssociatedSubnet().size(); i++) {
754 AssociatedSubnet associatedSubnet = routerTarget.getAssociatedSubnet().get(i);
755 if (VpnUtil.areSubnetsOverlapping(associatedSubnet.getCidr(), subnetCidr)) {
758 if (routerTarget.getRtType() == RouteTarget.RtType.ERT) {
759 /* Check if there are multiple exports for the input iRT value (iRT in routeTargets)
760 * Example : (1) iRT=A eRT=B subnet-range=S1; OK
761 * (2) iRT=A eRT=B subnet-range=S1; OK
762 * (3) iRT=B eRT=A subnet-range=S2; NOK
763 * Check if (1) and (2) are importing the same subnet-range routes to (3) */
764 List<AssociatedVpn> multipleAssociatedVpn = associatedSubnet.getAssociatedVpn();
765 if (multipleAssociatedVpn != null && multipleAssociatedVpn.size() > 1) {
766 LOG.error("doesExistingVpnsHaveConflictingSubnet: There is an indirect complete overlap"
767 + " for subnet CIDR {} for rt {} rtType {}", subnetCidr, routerTarget.getRt(),
768 routerTarget.getRtType());
771 for (int j = i + 1; j < routerTarget.getAssociatedSubnet().size(); j++) {
772 if (VpnUtil.areSubnetsOverlapping(associatedSubnet.getCidr(),
773 routerTarget.getAssociatedSubnet().get(j).getCidr())) {
774 LOG.error("doesExistingVpnsHaveConflictingSubnet: There is an indirect paartial"
775 + " overlap for subnet CIDR {} for rt {} rtType {}", subnetCidr,
776 routerTarget.getRt(), routerTarget.getRtType());
782 /* Check if there are indirect EXPORTS for the eRT value (eRT in routeTargets)
783 * Example : (1) iRT=A eRT=B subnet-range=S1; OK
784 * (2) iRT=B eRT=A subnet-range=S2; OK
785 * (3) iRT=A eRT=B subnet-range=S1; NOK
786 * If associatedSubnet is non-null for a routeTarget in (2),
787 * it may have already imported routes from (1) */
788 if (routerTarget.getRtType() == RouteTarget.RtType.IRT) {
790 Optional<RouteTarget> indirectRts = SingleTransactionDataBroker.syncReadOptional(dataBroker,
791 LogicalDatastoreType.OPERATIONAL, VpnUtil.getRouteTargetsIdentifier(
792 routerTarget.getRt(), RouteTarget.RtType.ERT));
793 if (indirectRts.isPresent() && indirectRts.get().getAssociatedSubnet() != null
794 && routerTarget.getAssociatedSubnet() != null) {
795 for (AssociatedSubnet associatedSubnet : indirectRts.get().getAssociatedSubnet()) {
796 if (VpnUtil.areSubnetsOverlapping(associatedSubnet.getCidr(), subnetCidr)) {
797 LOG.error("doesExistingVpnsHaveConflictingSubnet: There is an indirect overlap for"
798 + " subnet CIDR {} for rt {} rtType {}", subnetCidr, routerTarget.getRt(),
799 routerTarget.getRtType());
804 } catch (ReadFailedException e) {
805 LOG.error("doesExistingVpnsHaveConflictingSubnet: Failed to read route targets to subnet"
806 + "association for rt {} type {} subnet-cidr {}", routerTarget.getRt(),
807 RouteTarget.RtType.ERT, subnetCidr);
808 return true; //Fail subnet association to avoid further damage to the data-stores
812 LOG.info("doesExistingVpnsHaveConflictingSubnet: No prior associated subnets for rtVal {} rtType {}",
813 routerTarget.getRt(), routerTarget.getRtType());
819 private void addSubnetAssociationOperationToTx(String rt, RouteTarget.RtType rtType, String cidr,
820 String vpnName, TypedReadWriteTransaction<Operational> tx,
821 boolean isAssociationRemoved)
822 throws InterruptedException, ExecutionException {
823 if (isAssociationRemoved) {
824 //Remove RT-Subnet-Vpn Association
825 Optional<AssociatedSubnet> associatedSubnet =
826 tx.read(VpnUtil.getAssociatedSubnetIdentifier(rt, rtType, cidr)).get();
827 boolean deleteParent = false;
828 if (associatedSubnet.isPresent()) {
829 List<AssociatedVpn> associatedVpns = new ArrayList<>(associatedSubnet.get().nonnullAssociatedVpn());
830 if (associatedVpns == null || associatedVpns.isEmpty()) {
833 for (Iterator<AssociatedVpn> iterator = associatedVpns.iterator(); iterator.hasNext();) {
834 AssociatedVpn associatedVpn = iterator.next();
835 if (Objects.equals(associatedVpn.getName(), vpnName)) {
840 if (associatedVpns.isEmpty()) {
846 deleteParentForSubnetToVpnAssociation(rt, rtType, cidr, tx);
848 //Some other VPNs are also part of this rtVal, rtType and subnetCidr combination.
849 //Delete only this AssociatedVpn Object
850 tx.delete(VpnUtil.getAssociatedSubnetAndVpnIdentifier(rt, rtType, cidr, vpnName));
851 LOG.debug("addSubnetAssocOperationToTx: Removed vpn {} from association rt {} rtType {} cidr {}",
852 vpnName, rt, rtType, cidr);
855 //Add RT-Subnet-Vpn Association
856 tx.put(VpnUtil.getAssociatedSubnetAndVpnIdentifier(rt, rtType, cidr, vpnName),
857 VpnUtil.buildAssociatedSubnetAndVpn(vpnName), CREATE_MISSING_PARENTS);
861 private void deleteParentForSubnetToVpnAssociation(String rt, RouteTarget.RtType rtType,
862 String cidr, TypedReadWriteTransaction<Operational> tx)
863 throws InterruptedException, ExecutionException {
864 //Check if you need to delete rtVal+rtType or just the subnetCidr
865 InstanceIdentifier<RouteTarget> rtIdentifier = InstanceIdentifier
866 .builder(SubnetsAssociatedToRouteTargets.class).child(RouteTarget.class,
867 new RouteTargetKey(rt, rtType)).build();
868 Optional<RouteTarget> rtToSubnetsAssociation = tx.read(rtIdentifier).get();
869 if (rtToSubnetsAssociation.isPresent()) {
870 List<AssociatedSubnet> associatedSubnets = new ArrayList<>(rtToSubnetsAssociation.get()
871 .nonnullAssociatedSubnet());
872 if (associatedSubnets != null && !associatedSubnets.isEmpty()) {
873 for (Iterator<AssociatedSubnet> iterator = associatedSubnets.iterator(); iterator.hasNext(); ) {
874 if (Objects.equals(iterator.next().getCidr(), cidr)) {
879 if (associatedSubnets.isEmpty()) {
880 //The entire rt to subnet association is empty
881 //Delete the RouteTarget object
882 tx.delete(rtIdentifier);
883 LOG.debug("deleteParentForSubnetToVpnAssociation: Removed rt {} rtType {} from association,",
886 //Some other VPNs are also part of this rtVal, rtType combination
887 //Delete only this AssociatedSubnet
888 tx.delete(VpnUtil.getAssociatedSubnetIdentifier(rt, rtType, cidr));
889 LOG.debug("deleteParentForSubnetToVpnAssociation: Removed cidr {} from association rt {}"
890 + " rtType {}", cidr, rt, rtType);
897 public boolean checkForOverlappingSubnets(Uuid network, List<Subnetmap> subnetmapList, Uuid vpn,
898 Set<VpnTarget> routeTargets, List<String> failedNwList) {
899 for (Subnetmap subnetmap : subnetmapList) {
900 //Check if any other subnet that is already part of a different vpn with same rt, has overlapping CIDR
901 if (checkExistingSubnetWithSameRoutTargets(routeTargets, vpn, subnetmap, failedNwList)) {
908 private boolean checkExistingSubnetWithSameRoutTargets(Set<VpnTarget> routeTargets, Uuid vpn, Subnetmap subnetmap,
909 List<String> failedNwList) {
910 String cidr = String.valueOf(subnetmap.getSubnetIp());
911 boolean subnetExistsWithSameRt = doesExistingVpnsHaveConflictingSubnet(routeTargets, cidr);
912 if (subnetExistsWithSameRt) {
913 failedNwList.add(String.format("Another subnet with the same/overlapping cidr %s as subnet %s"
914 + " is already associated to a vpn with routeTargets %s. Ignoring subnet addition to vpn"
915 + " %s", cidr, subnetmap.getId().getValue(), routeTargets.toString(), vpn.getValue()));
917 return subnetExistsWithSameRt;