2 * Copyright © 2017 Ericsson India Global Services Pvt Ltd. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.netvirt.fibmanager;
10 import static java.util.stream.Collectors.toList;
12 import java.util.ArrayList;
13 import java.util.List;
15 import java.util.Optional;
16 import java.util.concurrent.BlockingQueue;
17 import java.util.concurrent.LinkedBlockingQueue;
18 import java.util.function.Consumer;
19 import javax.annotation.PostConstruct;
20 import javax.inject.Inject;
21 import javax.inject.Singleton;
22 import org.eclipse.jdt.annotation.Nullable;
23 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
24 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
25 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
26 import org.opendaylight.genius.mdsalutil.ActionInfo;
27 import org.opendaylight.genius.mdsalutil.InstructionInfo;
28 import org.opendaylight.genius.mdsalutil.NwConstants;
29 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
30 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
31 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
32 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
33 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
34 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldMplsLabel;
35 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
36 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
37 import org.opendaylight.genius.utils.batching.ActionableResource;
38 import org.opendaylight.genius.utils.batching.ActionableResourceImpl;
39 import org.opendaylight.genius.utils.batching.ResourceBatchingManager;
40 import org.opendaylight.genius.utils.batching.ResourceHandler;
41 import org.opendaylight.genius.utils.batching.SubTransaction;
42 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
43 import org.opendaylight.mdsal.binding.api.DataBroker;
44 import org.opendaylight.mdsal.binding.api.WriteTransaction;
45 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
46 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
47 import org.opendaylight.serviceutils.upgrade.UpgradeState;
48 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
49 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
50 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
51 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
54 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
55 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
56 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
57 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
58 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
59 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnListKey;
60 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
61 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
62 import org.opendaylight.yangtools.yang.common.Uint32;
63 import org.opendaylight.yangtools.yang.common.Uint64;
64 import org.slf4j.Logger;
65 import org.slf4j.LoggerFactory;
69 public class BgpRouteVrfEntryHandler extends BaseVrfEntryHandler implements ResourceHandler {
71 private static final Logger LOG = LoggerFactory.getLogger(BgpRouteVrfEntryHandler.class);
72 private static final int BATCH_INTERVAL = 500;
73 private static final int BATCH_SIZE = 1000;
75 private final DataBroker dataBroker;
76 private final ManagedNewTransactionRunner txRunner;
77 private final BlockingQueue<ActionableResource> vrfEntryBufferQ = new LinkedBlockingQueue<>();
78 private final ResourceBatchingManager resourceBatchingManager;
79 private final NexthopManager nexthopManager;
82 public BgpRouteVrfEntryHandler(final DataBroker dataBroker,
83 final NexthopManager nexthopManager,
84 final FibUtil fibUtil,
85 final UpgradeState upgradeState,
86 final DataTreeEventCallbackRegistrar eventCallbacks) {
87 super(dataBroker, nexthopManager, null, fibUtil, upgradeState, eventCallbacks);
88 this.dataBroker = dataBroker;
89 this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
90 this.nexthopManager = nexthopManager;
92 resourceBatchingManager = ResourceBatchingManager.getInstance();
93 resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY", vrfEntryBufferQ, this);
98 LOG.info("{} start", getClass().getSimpleName());
102 public void close() {
103 LOG.info("{} close", getClass().getSimpleName());
107 public DataBroker getResourceBroker() {
112 public int getBatchSize() {
117 public int getBatchInterval() {
118 return BATCH_INTERVAL;
122 public LogicalDatastoreType getDatastoreType() {
123 return LogicalDatastoreType.CONFIGURATION;
127 public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
128 Object original, Object update, List<SubTransaction> subTxns) {
129 if (original instanceof VrfEntry && update instanceof VrfEntry) {
130 createFibEntries(tx, identifier, (VrfEntry) update, subTxns);
135 public void updateContainer(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
136 Object original, Object update, List<SubTransaction> transactionObjects) {
140 public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
141 Object vrfEntry, List<SubTransaction> subTxns) {
142 if (vrfEntry instanceof VrfEntry) {
143 createFibEntries(tx, identifier, (VrfEntry) vrfEntry, subTxns);
148 public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
149 Object vrfEntry, List<SubTransaction> subTxns) {
150 if (vrfEntry instanceof VrfEntry) {
151 deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry, subTxns);
155 void createFlows(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
156 ActionableResourceImpl actResource = new ActionableResourceImpl(rd + vrfEntry.getDestPrefix());
157 actResource.setAction(ActionableResource.CREATE);
158 actResource.setInstanceIdentifier(identifier);
159 actResource.setInstance(vrfEntry);
160 vrfEntryBufferQ.add(actResource);
163 void removeFlows(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
164 ActionableResourceImpl actResource = new ActionableResourceImpl(rd + vrfEntry.getDestPrefix());
165 actResource.setAction(ActionableResource.DELETE);
166 actResource.setInstanceIdentifier(identifier);
167 actResource.setInstance(vrfEntry);
168 vrfEntryBufferQ.add(actResource);
171 void updateFlows(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update, String rd) {
172 ActionableResourceImpl actResource = new ActionableResourceImpl(rd + update.getDestPrefix());
173 actResource.setAction(ActionableResource.UPDATE);
174 actResource.setInstanceIdentifier(identifier);
175 actResource.setInstance(update);
176 actResource.setOldInstance(original);
177 vrfEntryBufferQ.add(actResource);
181 Please note that the following createFibEntries will be invoked only for BGP Imported Routes.
182 The invocation of the following method is via create() callback from the MDSAL Batching Infrastructure
183 provided by ResourceBatchingManager
185 private void createFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> vrfEntryIid,
186 final VrfEntry vrfEntry, List<SubTransaction> subTxns) {
187 final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
188 LOG.trace("Creating fib entry for vrfEntry with destPrefix{}, rd {}",
189 vrfEntry.getDestPrefix(), vrfTableKey.getRouteDistinguisher());
190 final VpnInstanceOpDataEntry vpnInstance =
191 getFibUtil().getVpnInstance(vrfTableKey.getRouteDistinguisher());
192 if (vpnInstance == null || vpnInstance.getVpnId() == null) {
193 LOG.error("Vpn Instance not availabe {}", vrfTableKey.getRouteDistinguisher());
196 final Map<VpnToDpnListKey, VpnToDpnList> keyVpnToDpnListMap = vpnInstance.getVpnToDpnList();
197 if (keyVpnToDpnListMap != null) {
198 for (VpnToDpnList vpnDpn : keyVpnToDpnListMap.values()) {
199 LOG.trace("Dpnstate is {} for dpn {} in vpn {}", vpnDpn.getDpnState(), vpnDpn.getDpnId(),
200 vpnInstance.getVpnId());
201 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
202 createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(),
203 vrfTableKey.getRouteDistinguisher(), vrfEntry, writeTx, subTxns);
207 LOG.trace("Created fib entry for vrfEntry with destPrefix{}, rd {}",
208 vrfEntry.getDestPrefix(), vrfTableKey.getRouteDistinguisher());
212 Please note that the following deleteFibEntries will be invoked only for BGP Imported Routes.
213 The invocation of the following method is via delete() callback from the MDSAL Batching Infrastructure
214 provided by ResourceBatchingManager
216 private void deleteFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> identifier,
217 final VrfEntry vrfEntry, List<SubTransaction> subTxns) {
218 final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
219 String rd = vrfTableKey.getRouteDistinguisher();
220 final VpnInstanceOpDataEntry vpnInstance =
221 getFibUtil().getVpnInstance(vrfTableKey.getRouteDistinguisher());
222 if (vpnInstance == null) {
223 LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
226 String vpnName = getFibUtil().getVpnNameFromId(vpnInstance.getVpnId());
227 final Map<VpnToDpnListKey, VpnToDpnList> keyVpnToDpnListMap = vpnInstance.getVpnToDpnList();
228 if (keyVpnToDpnListMap != null) {
229 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
230 vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
231 Optional<Routes> extraRouteOptional;
232 //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
233 if (usedRds != null && !usedRds.isEmpty()) {
234 if (usedRds.size() > 1) {
235 LOG.error("The extra route prefix is still present in some DPNs");
238 extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
239 usedRds.get(0), vrfEntry.getDestPrefix());
242 extraRouteOptional = Optional.empty();
244 for (VpnToDpnList curDpn : keyVpnToDpnListMap.values()) {
245 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
246 deleteRemoteRoute(Uint64.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(),
247 vrfTableKey, vrfEntry, extraRouteOptional, writeTx, subTxns);
253 public void programRemoteFibForBgpRoutes(final Uint64 remoteDpnId,
255 final VrfEntry vrfEntry,
258 List<NexthopManager.AdjacencyResult> adjacencyResults,
259 List<SubTransaction> subTxns) {
260 if (vrfEntry.nonnullRoutePaths().size() > 2) {
261 LOG.error("DC-GW can advertise only 2 bestPaths for prefix {}", vrfEntry.getDestPrefix());
264 LOG.trace("Start programming remote fib for destPrefix {}, vpnId {}, dpnId {}",
265 vrfEntry.getDestPrefix(), vpnId, remoteDpnId);
266 if (adjacencyResults.size() == 1) {
267 programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, subTxns);
270 // ECMP Use case, point to LB group. Move the mpls label accordingly.
271 List<String> tunnelList =
272 adjacencyResults.stream()
273 .map(NexthopManager.AdjacencyResult::getNextHopIp)
274 .sorted().collect(toList());
275 String lbGroupKey = FibUtil.getGreLbGroupKey(tunnelList);
276 long groupId = nexthopManager.createNextHopPointer(lbGroupKey);
278 List<ActionInfo> actionInfos = new ArrayList<>();
279 for (NexthopManager.AdjacencyResult adjResult : adjacencyResults) {
280 String nextHopIp = adjResult.getNextHopIp();
281 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
282 if (!optionalLabel.isPresent()) {
283 LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
286 long label = optionalLabel.get().toJava();
288 actionInfos.add(new ActionRegLoad(index, FibConstants.NXM_REG_MAPPING.get(index++), 0,
291 List<InstructionInfo> instructions = new ArrayList<>();
292 actionInfos.add(new ActionGroup(index, groupId));
293 instructions.add(new InstructionApplyActions(actionInfos));
294 makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx, subTxns);
295 LOG.trace("End programming remote fib for destPrefix {}, vpnId {}, dpnId {}",
296 vrfEntry.getDestPrefix(), vpnId, remoteDpnId);
299 // Allow deprecated TransactionRunner calls for now
300 @SuppressWarnings("ForbidCertainMethod")
301 public void createRemoteFibEntry(final Uint64 remoteDpnId,
304 final VrfEntry vrfEntry,
306 List<SubTransaction> subTxns) {
308 ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
309 newTx -> createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx, subTxns)), LOG,
310 "Error creating remote FIB entry");
314 LOG.debug("createRemoteFibEntry: adding route {} for rd {} on remoteDpnId {}",
315 vrfEntry.getDestPrefix(), rd, remoteDpnId);
317 List<NexthopManager.AdjacencyResult> adjacencyResults =
318 resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
319 if (adjacencyResults.isEmpty()) {
320 LOG.error("Could not get interface for route-paths: {} in vpn {}", vrfEntry.getRoutePaths(), rd);
321 LOG.warn("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
325 programRemoteFibForBgpRoutes(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, subTxns);
327 LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
330 private void deleteFibEntryForBgpRoutes(Uint64 remoteDpnId, Uint32 vpnId, VrfEntry vrfEntry,
331 String rd, WriteTransaction tx, List<SubTransaction> subTxns) {
332 // When the tunnel is removed the fib entries should be reprogrammed/deleted depending on
333 // the adjacencyResults.
334 List<NexthopManager.AdjacencyResult> adjacencyResults = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
335 if (!adjacencyResults.isEmpty()) {
336 programRemoteFibForBgpRoutes(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, subTxns);
340 // Allow deprecated TransactionRunner calls for now
341 @SuppressWarnings("ForbidCertainMethod")
342 public void deleteRemoteRoute(@Nullable final Uint64 localDpnId, final Uint64 remoteDpnId,
343 final Uint32 vpnId, final VrfTablesKey vrfTableKey,
344 final VrfEntry vrfEntry, Optional<Routes> extraRouteOptional,
345 @Nullable WriteTransaction tx, List<SubTransaction> subTxns) {
347 ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
348 newTx -> deleteRemoteRoute(localDpnId, remoteDpnId, vpnId, vrfTableKey, vrfEntry,
349 extraRouteOptional, newTx)), LOG, "Error deleting remote route");
353 LOG.debug("deleting remote route: prefix={}, vpnId={} localDpnId {} remoteDpnId {}",
354 vrfEntry.getDestPrefix(), vpnId, localDpnId, remoteDpnId);
355 String rd = vrfTableKey.getRouteDistinguisher();
357 if (localDpnId != null && !Uint64.ZERO.equals(localDpnId)) {
358 // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
359 if (extraRouteOptional.isPresent()) {
360 nexthopManager.deleteLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
362 deleteFibEntryForBgpRoutes(remoteDpnId, vpnId, vrfEntry, rd, tx, subTxns);
366 // below two reads are kept as is, until best way is found to identify dpnID
367 VpnNexthop localNextHopInfo = nexthopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
368 if (extraRouteOptional.isPresent()) {
369 nexthopManager.deleteLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
371 checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx, subTxns);
375 public Consumer<? super VrfEntry> getConsumerForCreatingRemoteFib(
376 final Uint64 dpnId, final Uint32 vpnId, final String rd,
377 final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
378 WriteTransaction writeCfgTxn, List<SubTransaction> subTxns) {
379 return vrfEntry -> vrfEntry.nonnullRoutePaths().values().stream()
380 .filter(routes -> !routes.getNexthopAddress().isEmpty()
381 && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
383 .ifPresent(routes -> {
384 LOG.trace("creating remote FIB entry for prefix {} rd {} on Dpn {}",
385 vrfEntry.getDestPrefix(), rd, dpnId);
386 createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
387 vrfEntry, writeCfgTxn, subTxns);
391 public Consumer<? super VrfEntry> getConsumerForDeletingRemoteFib(
392 final Uint64 dpnId, final Uint32 vpnId,
393 final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
394 WriteTransaction writeCfgTxn, List<SubTransaction> subTxns) {
395 return vrfEntry -> vrfEntry.nonnullRoutePaths().values().stream()
396 .filter(routes -> !routes.getNexthopAddress().isEmpty()
397 && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
399 .ifPresent(routes -> {
400 LOG.trace(" deleting remote FIB entry {}", vrfEntry);
401 deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().key(), vrfEntry,
402 Optional.empty(), writeCfgTxn, subTxns);
407 protected void addTunnelInterfaceActions(NexthopManager.AdjacencyResult adjacencyResult, Uint32 vpnId,
408 VrfEntry vrfEntry, List<ActionInfo> actionInfos, String rd) {
409 Class<? extends TunnelTypeBase> tunnelType = VpnExtraRouteHelper
410 .getTunnelType(getNextHopManager().getItmManager(), adjacencyResult.getInterfaceName());
411 if (tunnelType == null) {
412 LOG.debug("Tunnel type not found for vrfEntry {}", vrfEntry);
415 String nextHopIp = adjacencyResult.getNextHopIp();
416 if (tunnelType.equals(TunnelTypeMplsOverGre.class)) {
417 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
418 if (!optionalLabel.isPresent()) {
419 LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
422 long label = optionalLabel.get().toJava();
423 LOG.debug("addTunnelInterfaceActions: Push label action for prefix {} rd {} l3vni {} nextHop {}",
424 vrfEntry.getDestPrefix(), rd, vrfEntry.getL3vni(), nextHopIp);
425 actionInfos.add(new ActionPushMpls());
426 actionInfos.add(new ActionSetFieldMplsLabel(label));
427 actionInfos.add(new ActionNxLoadInPort(Uint64.ZERO));
428 } else if (tunnelType.equals(TunnelTypeVxlan.class)) {
429 actionInfos.add(new ActionSetFieldTunnelId(Uint64.valueOf(vrfEntry.getL3vni().longValue())));
430 LOG.debug("addTunnelInterfaceActions: adding set tunnel id action for prefix {} rd {} l3vni {}"
431 + " nextHop {} ", vrfEntry.getDestPrefix(), rd, vrfEntry.getL3vni(), nextHopIp);
432 addRewriteDstMacAction(vpnId, vrfEntry, null /*prefixInfo*/, actionInfos);
437 protected void addRewriteDstMacAction(Uint32 vpnId, VrfEntry vrfEntry, @Nullable Prefixes prefixInfo,
438 List<ActionInfo> actionInfos) {
439 if (vrfEntry.getGatewayMacAddress() != null) {
440 actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
441 new MacAddress(vrfEntry.getGatewayMacAddress())));