2 * Copyright © 2017 Ericsson India Global Services Pvt Ltd. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.netvirt.fibmanager;
10 import static java.util.Objects.requireNonNull;
11 import static java.util.stream.Collectors.toList;
13 import java.util.ArrayList;
14 import java.util.List;
16 import java.util.Optional;
17 import java.util.concurrent.BlockingQueue;
18 import java.util.concurrent.LinkedBlockingQueue;
19 import java.util.function.Consumer;
20 import javax.annotation.PostConstruct;
21 import javax.inject.Inject;
22 import javax.inject.Singleton;
23 import org.eclipse.jdt.annotation.Nullable;
24 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
25 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
26 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
27 import org.opendaylight.genius.mdsalutil.ActionInfo;
28 import org.opendaylight.genius.mdsalutil.InstructionInfo;
29 import org.opendaylight.genius.mdsalutil.NwConstants;
30 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
31 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
32 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
33 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
34 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
35 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldMplsLabel;
36 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
37 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
38 import org.opendaylight.genius.utils.batching.ActionableResource;
39 import org.opendaylight.genius.utils.batching.ActionableResources;
40 import org.opendaylight.genius.utils.batching.ResourceBatchingManager;
41 import org.opendaylight.genius.utils.batching.ResourceHandler;
42 import org.opendaylight.genius.utils.batching.SubTransaction;
43 import org.opendaylight.infrautils.utils.concurrent.LoggingFutures;
44 import org.opendaylight.mdsal.binding.api.DataBroker;
45 import org.opendaylight.mdsal.binding.api.WriteTransaction;
46 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
47 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
48 import org.opendaylight.serviceutils.upgrade.UpgradeState;
49 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
50 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
51 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
52 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
54 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
55 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
56 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
57 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
58 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
59 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
60 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnListKey;
61 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
62 import org.opendaylight.yangtools.concepts.Identifier;
63 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
64 import org.opendaylight.yangtools.yang.common.Uint32;
65 import org.opendaylight.yangtools.yang.common.Uint64;
66 import org.slf4j.Logger;
67 import org.slf4j.LoggerFactory;
71 public class BgpRouteVrfEntryHandler extends BaseVrfEntryHandler implements ResourceHandler {
72 private static final class ActionableResourceIdentifier implements Identifier {
73 private static final long serialVersionUID = 1L;
75 private final String routeDistinguisher;
76 private final String destPrefix;
78 ActionableResourceIdentifier(final String routeDistinguisher, final String destPrefix) {
79 this.routeDistinguisher = requireNonNull(routeDistinguisher);
80 this.destPrefix = requireNonNull(destPrefix);
84 public int hashCode() {
85 return routeDistinguisher.hashCode() * 31 + destPrefix.hashCode();
89 public boolean equals(final Object obj) {
93 if (!(obj instanceof ActionableResourceIdentifier)) {
96 final ActionableResourceIdentifier other = (ActionableResourceIdentifier) obj;
97 return routeDistinguisher.equals(other.routeDistinguisher) && destPrefix.equals(other.destPrefix);
101 public String toString() {
102 return routeDistinguisher + destPrefix;
106 private static final Logger LOG = LoggerFactory.getLogger(BgpRouteVrfEntryHandler.class);
107 private static final int BATCH_INTERVAL = 500;
108 private static final int BATCH_SIZE = 1000;
110 private final DataBroker dataBroker;
111 private final ManagedNewTransactionRunner txRunner;
112 private final BlockingQueue<ActionableResource<?>> vrfEntryBufferQ = new LinkedBlockingQueue<>();
113 private final ResourceBatchingManager resourceBatchingManager;
114 private final NexthopManager nexthopManager;
117 public BgpRouteVrfEntryHandler(final DataBroker dataBroker,
118 final NexthopManager nexthopManager,
119 final FibUtil fibUtil,
120 final UpgradeState upgradeState,
121 final DataTreeEventCallbackRegistrar eventCallbacks) {
122 super(dataBroker, nexthopManager, null, fibUtil, upgradeState, eventCallbacks);
123 this.dataBroker = dataBroker;
124 this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
125 this.nexthopManager = nexthopManager;
127 resourceBatchingManager = ResourceBatchingManager.getInstance();
128 resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY", vrfEntryBufferQ, this);
133 LOG.info("{} start", getClass().getSimpleName());
137 public void close() {
138 LOG.info("{} close", getClass().getSimpleName());
142 public DataBroker getResourceBroker() {
147 public int getBatchSize() {
152 public int getBatchInterval() {
153 return BATCH_INTERVAL;
157 public LogicalDatastoreType getDatastoreType() {
158 return LogicalDatastoreType.CONFIGURATION;
162 public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
163 Object original, Object update, List<SubTransaction> subTxns) {
164 if (original instanceof VrfEntry && update instanceof VrfEntry) {
165 createFibEntries(tx, identifier, (VrfEntry) update, subTxns);
170 public void updateContainer(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
171 Object original, Object update, List<SubTransaction> transactionObjects) {
175 public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
176 Object vrfEntry, List<SubTransaction> subTxns) {
177 if (vrfEntry instanceof VrfEntry) {
178 createFibEntries(tx, identifier, (VrfEntry) vrfEntry, subTxns);
183 public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
184 Object vrfEntry, List<SubTransaction> subTxns) {
185 if (vrfEntry instanceof VrfEntry) {
186 deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry, subTxns);
190 void createFlows(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
191 vrfEntryBufferQ.add(ActionableResources.create(new ActionableResourceIdentifier(rd, vrfEntry.getDestPrefix()),
192 identifier, vrfEntry));
195 void removeFlows(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
196 vrfEntryBufferQ.add(ActionableResources.delete(new ActionableResourceIdentifier(rd, vrfEntry.getDestPrefix()),
197 identifier, vrfEntry));
200 void updateFlows(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update, String rd) {
201 vrfEntryBufferQ.add(ActionableResources.update(new ActionableResourceIdentifier(rd, update.getDestPrefix()),
202 identifier, update, original));
206 Please note that the following createFibEntries will be invoked only for BGP Imported Routes.
207 The invocation of the following method is via create() callback from the MDSAL Batching Infrastructure
208 provided by ResourceBatchingManager
210 private void createFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> vrfEntryIid,
211 final VrfEntry vrfEntry, List<SubTransaction> subTxns) {
212 final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
213 LOG.trace("Creating fib entry for vrfEntry with destPrefix{}, rd {}",
214 vrfEntry.getDestPrefix(), vrfTableKey.getRouteDistinguisher());
215 final VpnInstanceOpDataEntry vpnInstance =
216 getFibUtil().getVpnInstance(vrfTableKey.getRouteDistinguisher());
217 if (vpnInstance == null || vpnInstance.getVpnId() == null) {
218 LOG.error("Vpn Instance not availabe {}", vrfTableKey.getRouteDistinguisher());
221 final Map<VpnToDpnListKey, VpnToDpnList> keyVpnToDpnListMap = vpnInstance.nonnullVpnToDpnList();
222 if (keyVpnToDpnListMap != null) {
223 for (VpnToDpnList vpnDpn : keyVpnToDpnListMap.values()) {
224 LOG.trace("Dpnstate is {} for dpn {} in vpn {}", vpnDpn.getDpnState(), vpnDpn.getDpnId(),
225 vpnInstance.getVpnId());
226 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
227 createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(),
228 vrfTableKey.getRouteDistinguisher(), vrfEntry, writeTx, subTxns);
232 LOG.trace("Created fib entry for vrfEntry with destPrefix{}, rd {}",
233 vrfEntry.getDestPrefix(), vrfTableKey.getRouteDistinguisher());
237 Please note that the following deleteFibEntries will be invoked only for BGP Imported Routes.
238 The invocation of the following method is via delete() callback from the MDSAL Batching Infrastructure
239 provided by ResourceBatchingManager
241 private void deleteFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> identifier,
242 final VrfEntry vrfEntry, List<SubTransaction> subTxns) {
243 final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
244 String rd = vrfTableKey.getRouteDistinguisher();
245 final VpnInstanceOpDataEntry vpnInstance =
246 getFibUtil().getVpnInstance(vrfTableKey.getRouteDistinguisher());
247 if (vpnInstance == null) {
248 LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
251 String vpnName = getFibUtil().getVpnNameFromId(vpnInstance.getVpnId());
252 final Map<VpnToDpnListKey, VpnToDpnList> keyVpnToDpnListMap = vpnInstance.nonnullVpnToDpnList();
253 if (keyVpnToDpnListMap != null) {
254 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
255 vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
256 Optional<Routes> extraRouteOptional;
257 //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
258 if (usedRds != null && !usedRds.isEmpty()) {
259 if (usedRds.size() > 1) {
260 LOG.error("The extra route prefix is still present in some DPNs");
263 extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
264 usedRds.get(0), vrfEntry.getDestPrefix());
267 extraRouteOptional = Optional.empty();
269 for (VpnToDpnList curDpn : keyVpnToDpnListMap.values()) {
270 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
271 deleteRemoteRoute(Uint64.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(),
272 vrfTableKey, vrfEntry, extraRouteOptional, writeTx, subTxns);
278 public void programRemoteFibForBgpRoutes(final Uint64 remoteDpnId,
280 final VrfEntry vrfEntry,
283 List<NexthopManager.AdjacencyResult> adjacencyResults,
284 List<SubTransaction> subTxns) {
285 if (vrfEntry.nonnullRoutePaths().size() > 2) {
286 LOG.error("DC-GW can advertise only 2 bestPaths for prefix {}", vrfEntry.getDestPrefix());
289 LOG.trace("Start programming remote fib for destPrefix {}, vpnId {}, dpnId {}",
290 vrfEntry.getDestPrefix(), vpnId, remoteDpnId);
291 if (adjacencyResults.size() == 1) {
292 programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, subTxns);
295 // ECMP Use case, point to LB group. Move the mpls label accordingly.
296 List<String> tunnelList =
297 adjacencyResults.stream()
298 .map(NexthopManager.AdjacencyResult::getNextHopIp)
299 .sorted().collect(toList());
300 String lbGroupKey = FibUtil.getGreLbGroupKey(tunnelList);
301 long groupId = nexthopManager.createNextHopPointer(lbGroupKey);
303 List<ActionInfo> actionInfos = new ArrayList<>();
304 for (NexthopManager.AdjacencyResult adjResult : adjacencyResults) {
305 String nextHopIp = adjResult.getNextHopIp();
306 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
307 if (!optionalLabel.isPresent()) {
308 LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
311 long label = optionalLabel.get().toJava();
313 actionInfos.add(new ActionRegLoad(index, FibConstants.NXM_REG_MAPPING.get(index++), 0,
316 List<InstructionInfo> instructions = new ArrayList<>();
317 actionInfos.add(new ActionGroup(index, groupId));
318 instructions.add(new InstructionApplyActions(actionInfos));
319 makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx, subTxns);
320 LOG.trace("End programming remote fib for destPrefix {}, vpnId {}, dpnId {}",
321 vrfEntry.getDestPrefix(), vpnId, remoteDpnId);
324 // Allow deprecated TransactionRunner calls for now
325 @SuppressWarnings("ForbidCertainMethod")
326 public void createRemoteFibEntry(final Uint64 remoteDpnId,
329 final VrfEntry vrfEntry,
331 List<SubTransaction> subTxns) {
333 LoggingFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
334 newTx -> createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx, subTxns)), LOG,
335 "Error creating remote FIB entry");
339 LOG.debug("createRemoteFibEntry: adding route {} for rd {} on remoteDpnId {}",
340 vrfEntry.getDestPrefix(), rd, remoteDpnId);
342 List<NexthopManager.AdjacencyResult> adjacencyResults =
343 resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
344 if (adjacencyResults.isEmpty()) {
345 LOG.error("Could not get interface for route-paths: {} in vpn {}", vrfEntry.getRoutePaths(), rd);
346 LOG.warn("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
350 programRemoteFibForBgpRoutes(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, subTxns);
352 LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
355 private void deleteFibEntryForBgpRoutes(Uint64 remoteDpnId, Uint32 vpnId, VrfEntry vrfEntry,
356 String rd, WriteTransaction tx, List<SubTransaction> subTxns) {
357 // When the tunnel is removed the fib entries should be reprogrammed/deleted depending on
358 // the adjacencyResults.
359 List<NexthopManager.AdjacencyResult> adjacencyResults = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
360 if (!adjacencyResults.isEmpty()) {
361 programRemoteFibForBgpRoutes(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, subTxns);
365 // Allow deprecated TransactionRunner calls for now
366 @SuppressWarnings("ForbidCertainMethod")
367 public void deleteRemoteRoute(@Nullable final Uint64 localDpnId, final Uint64 remoteDpnId,
368 final Uint32 vpnId, final VrfTablesKey vrfTableKey,
369 final VrfEntry vrfEntry, Optional<Routes> extraRouteOptional,
370 @Nullable WriteTransaction tx, List<SubTransaction> subTxns) {
372 LoggingFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
373 newTx -> deleteRemoteRoute(localDpnId, remoteDpnId, vpnId, vrfTableKey, vrfEntry,
374 extraRouteOptional, newTx)), LOG, "Error deleting remote route");
378 LOG.debug("deleting remote route: prefix={}, vpnId={} localDpnId {} remoteDpnId {}",
379 vrfEntry.getDestPrefix(), vpnId, localDpnId, remoteDpnId);
380 String rd = vrfTableKey.getRouteDistinguisher();
382 if (localDpnId != null && !Uint64.ZERO.equals(localDpnId)) {
383 // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
384 if (extraRouteOptional.isPresent()) {
385 nexthopManager.deleteLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
387 deleteFibEntryForBgpRoutes(remoteDpnId, vpnId, vrfEntry, rd, tx, subTxns);
391 // below two reads are kept as is, until best way is found to identify dpnID
392 VpnNexthop localNextHopInfo = nexthopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
393 if (extraRouteOptional.isPresent()) {
394 nexthopManager.deleteLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
396 checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx, subTxns);
400 public Consumer<? super VrfEntry> getConsumerForCreatingRemoteFib(
401 final Uint64 dpnId, final Uint32 vpnId, final String rd,
402 final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
403 WriteTransaction writeCfgTxn, List<SubTransaction> subTxns) {
404 return vrfEntry -> vrfEntry.nonnullRoutePaths().values().stream()
405 .filter(routes -> !routes.getNexthopAddress().isEmpty()
406 && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
408 .ifPresent(routes -> {
409 LOG.trace("creating remote FIB entry for prefix {} rd {} on Dpn {}",
410 vrfEntry.getDestPrefix(), rd, dpnId);
411 createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
412 vrfEntry, writeCfgTxn, subTxns);
416 public Consumer<? super VrfEntry> getConsumerForDeletingRemoteFib(
417 final Uint64 dpnId, final Uint32 vpnId,
418 final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
419 WriteTransaction writeCfgTxn, List<SubTransaction> subTxns) {
420 return vrfEntry -> vrfEntry.nonnullRoutePaths().values().stream()
421 .filter(routes -> !routes.getNexthopAddress().isEmpty()
422 && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
424 .ifPresent(routes -> {
425 LOG.trace(" deleting remote FIB entry {}", vrfEntry);
426 deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().key(), vrfEntry,
427 Optional.empty(), writeCfgTxn, subTxns);
432 protected void addTunnelInterfaceActions(NexthopManager.AdjacencyResult adjacencyResult, Uint32 vpnId,
433 VrfEntry vrfEntry, List<ActionInfo> actionInfos, String rd) {
434 Class<? extends TunnelTypeBase> tunnelType = VpnExtraRouteHelper
435 .getTunnelType(getNextHopManager().getItmManager(), adjacencyResult.getInterfaceName());
436 if (tunnelType == null) {
437 LOG.debug("Tunnel type not found for vrfEntry {}", vrfEntry);
440 String nextHopIp = adjacencyResult.getNextHopIp();
441 if (tunnelType.equals(TunnelTypeMplsOverGre.class)) {
442 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
443 if (!optionalLabel.isPresent()) {
444 LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
447 long label = optionalLabel.get().toJava();
448 LOG.debug("addTunnelInterfaceActions: Push label action for prefix {} rd {} l3vni {} nextHop {}",
449 vrfEntry.getDestPrefix(), rd, vrfEntry.getL3vni(), nextHopIp);
450 actionInfos.add(new ActionPushMpls());
451 actionInfos.add(new ActionSetFieldMplsLabel(label));
452 actionInfos.add(new ActionNxLoadInPort(Uint64.ZERO));
453 } else if (tunnelType.equals(TunnelTypeVxlan.class)) {
454 actionInfos.add(new ActionSetFieldTunnelId(Uint64.valueOf(vrfEntry.getL3vni().longValue())));
455 LOG.debug("addTunnelInterfaceActions: adding set tunnel id action for prefix {} rd {} l3vni {}"
456 + " nextHop {} ", vrfEntry.getDestPrefix(), rd, vrfEntry.getL3vni(), nextHopIp);
457 addRewriteDstMacAction(vpnId, vrfEntry, null /*prefixInfo*/, actionInfos);
462 protected void addRewriteDstMacAction(Uint32 vpnId, VrfEntry vrfEntry, @Nullable Prefixes prefixInfo,
463 List<ActionInfo> actionInfos) {
464 if (vrfEntry.getGatewayMacAddress() != null) {
465 actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
466 new MacAddress(vrfEntry.getGatewayMacAddress())));