2 * Copyright © 2017 Ericsson India Global Services Pvt Ltd. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.netvirt.fibmanager;
10 import static java.util.stream.Collectors.toList;
12 import java.util.ArrayList;
13 import java.util.Collection;
14 import java.util.List;
15 import java.util.Optional;
16 import java.util.concurrent.BlockingQueue;
17 import java.util.concurrent.LinkedBlockingQueue;
18 import java.util.function.Consumer;
19 import javax.annotation.PostConstruct;
20 import javax.inject.Inject;
21 import javax.inject.Singleton;
22 import org.eclipse.jdt.annotation.Nullable;
23 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
24 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
25 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
26 import org.opendaylight.genius.mdsalutil.ActionInfo;
27 import org.opendaylight.genius.mdsalutil.InstructionInfo;
28 import org.opendaylight.genius.mdsalutil.NwConstants;
29 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
30 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
31 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
32 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
33 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
34 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldMplsLabel;
35 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
36 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
37 import org.opendaylight.genius.utils.batching.ActionableResource;
38 import org.opendaylight.genius.utils.batching.ActionableResourceImpl;
39 import org.opendaylight.genius.utils.batching.ResourceBatchingManager;
40 import org.opendaylight.genius.utils.batching.ResourceHandler;
41 import org.opendaylight.genius.utils.batching.SubTransaction;
42 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
43 import org.opendaylight.mdsal.binding.api.DataBroker;
44 import org.opendaylight.mdsal.binding.api.WriteTransaction;
45 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
46 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
47 import org.opendaylight.serviceutils.upgrade.UpgradeState;
48 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
49 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
50 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
51 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
54 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
55 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
56 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
57 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
58 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
59 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
60 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
61 import org.opendaylight.yangtools.yang.common.Uint32;
62 import org.opendaylight.yangtools.yang.common.Uint64;
63 import org.slf4j.Logger;
64 import org.slf4j.LoggerFactory;
68 public class BgpRouteVrfEntryHandler extends BaseVrfEntryHandler implements ResourceHandler {
70 private static final Logger LOG = LoggerFactory.getLogger(BgpRouteVrfEntryHandler.class);
71 private static final int BATCH_INTERVAL = 500;
72 private static final int BATCH_SIZE = 1000;
74 private final DataBroker dataBroker;
75 private final ManagedNewTransactionRunner txRunner;
76 private final BlockingQueue<ActionableResource> vrfEntryBufferQ = new LinkedBlockingQueue<>();
77 private final ResourceBatchingManager resourceBatchingManager;
78 private final NexthopManager nexthopManager;
81 public BgpRouteVrfEntryHandler(final DataBroker dataBroker,
82 final NexthopManager nexthopManager,
83 final FibUtil fibUtil,
84 final UpgradeState upgradeState,
85 final DataTreeEventCallbackRegistrar eventCallbacks) {
86 super(dataBroker, nexthopManager, null, fibUtil, upgradeState, eventCallbacks);
87 this.dataBroker = dataBroker;
88 this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
89 this.nexthopManager = nexthopManager;
91 resourceBatchingManager = ResourceBatchingManager.getInstance();
92 resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY", vrfEntryBufferQ, this);
97 LOG.info("{} start", getClass().getSimpleName());
101 public void close() {
102 LOG.info("{} close", getClass().getSimpleName());
106 public DataBroker getResourceBroker() {
111 public int getBatchSize() {
116 public int getBatchInterval() {
117 return BATCH_INTERVAL;
121 public LogicalDatastoreType getDatastoreType() {
122 return LogicalDatastoreType.CONFIGURATION;
126 public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
127 Object original, Object update, List<SubTransaction> subTxns) {
128 if (original instanceof VrfEntry && update instanceof VrfEntry) {
129 createFibEntries(tx, identifier, (VrfEntry) update, subTxns);
134 public void updateContainer(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
135 Object original, Object update, List<SubTransaction> transactionObjects) {
139 public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
140 Object vrfEntry, List<SubTransaction> subTxns) {
141 if (vrfEntry instanceof VrfEntry) {
142 createFibEntries(tx, identifier, (VrfEntry) vrfEntry, subTxns);
147 public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
148 Object vrfEntry, List<SubTransaction> subTxns) {
149 if (vrfEntry instanceof VrfEntry) {
150 deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry, subTxns);
154 void createFlows(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
155 ActionableResourceImpl actResource = new ActionableResourceImpl(rd + vrfEntry.getDestPrefix());
156 actResource.setAction(ActionableResource.CREATE);
157 actResource.setInstanceIdentifier(identifier);
158 actResource.setInstance(vrfEntry);
159 vrfEntryBufferQ.add(actResource);
162 void removeFlows(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
163 ActionableResourceImpl actResource = new ActionableResourceImpl(rd + vrfEntry.getDestPrefix());
164 actResource.setAction(ActionableResource.DELETE);
165 actResource.setInstanceIdentifier(identifier);
166 actResource.setInstance(vrfEntry);
167 vrfEntryBufferQ.add(actResource);
170 void updateFlows(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update, String rd) {
171 ActionableResourceImpl actResource = new ActionableResourceImpl(rd + update.getDestPrefix());
172 actResource.setAction(ActionableResource.UPDATE);
173 actResource.setInstanceIdentifier(identifier);
174 actResource.setInstance(update);
175 actResource.setOldInstance(original);
176 vrfEntryBufferQ.add(actResource);
180 Please note that the following createFibEntries will be invoked only for BGP Imported Routes.
181 The invocation of the following method is via create() callback from the MDSAL Batching Infrastructure
182 provided by ResourceBatchingManager
184 private void createFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> vrfEntryIid,
185 final VrfEntry vrfEntry, List<SubTransaction> subTxns) {
186 final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
187 LOG.trace("Creating fib entry for vrfEntry with destPrefix{}, rd {}",
188 vrfEntry.getDestPrefix(), vrfTableKey.getRouteDistinguisher());
189 final VpnInstanceOpDataEntry vpnInstance =
190 getFibUtil().getVpnInstance(vrfTableKey.getRouteDistinguisher());
191 if (vpnInstance == null || vpnInstance.getVpnId() == null) {
192 LOG.error("Vpn Instance not availabe {}", vrfTableKey.getRouteDistinguisher());
195 final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
196 if (vpnToDpnList != null) {
197 for (VpnToDpnList vpnDpn : vpnToDpnList) {
198 LOG.trace("Dpnstate is {} for dpn {} in vpn {}", vpnDpn.getDpnState(), vpnDpn.getDpnId(),
199 vpnInstance.getVpnId());
200 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
201 createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(),
202 vrfTableKey.getRouteDistinguisher(), vrfEntry, writeTx, subTxns);
206 LOG.trace("Created fib entry for vrfEntry with destPrefix{}, rd {}",
207 vrfEntry.getDestPrefix(), vrfTableKey.getRouteDistinguisher());
211 Please note that the following deleteFibEntries will be invoked only for BGP Imported Routes.
212 The invocation of the following method is via delete() callback from the MDSAL Batching Infrastructure
213 provided by ResourceBatchingManager
215 private void deleteFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> identifier,
216 final VrfEntry vrfEntry, List<SubTransaction> subTxns) {
217 final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
218 String rd = vrfTableKey.getRouteDistinguisher();
219 final VpnInstanceOpDataEntry vpnInstance =
220 getFibUtil().getVpnInstance(vrfTableKey.getRouteDistinguisher());
221 if (vpnInstance == null) {
222 LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
225 String vpnName = getFibUtil().getVpnNameFromId(vpnInstance.getVpnId());
226 final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
227 if (vpnToDpnList != null) {
228 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
229 vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
230 Optional<Routes> extraRouteOptional;
231 //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
232 if (usedRds != null && !usedRds.isEmpty()) {
233 if (usedRds.size() > 1) {
234 LOG.error("The extra route prefix is still present in some DPNs");
237 extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
238 usedRds.get(0), vrfEntry.getDestPrefix());
241 extraRouteOptional = Optional.empty();
243 for (VpnToDpnList curDpn : vpnToDpnList) {
244 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
245 deleteRemoteRoute(Uint64.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(),
246 vrfTableKey, vrfEntry, extraRouteOptional, writeTx, subTxns);
252 public void programRemoteFibForBgpRoutes(final Uint64 remoteDpnId,
254 final VrfEntry vrfEntry,
257 List<NexthopManager.AdjacencyResult> adjacencyResults,
258 List<SubTransaction> subTxns) {
259 if (vrfEntry.nonnullRoutePaths().size() > 2) {
260 LOG.error("DC-GW can advertise only 2 bestPaths for prefix {}", vrfEntry.getDestPrefix());
263 LOG.trace("Start programming remote fib for destPrefix {}, vpnId {}, dpnId {}",
264 vrfEntry.getDestPrefix(), vpnId, remoteDpnId);
265 if (adjacencyResults.size() == 1) {
266 programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, subTxns);
269 // ECMP Use case, point to LB group. Move the mpls label accordingly.
270 List<String> tunnelList =
271 adjacencyResults.stream()
272 .map(NexthopManager.AdjacencyResult::getNextHopIp)
273 .sorted().collect(toList());
274 String lbGroupKey = FibUtil.getGreLbGroupKey(tunnelList);
275 long groupId = nexthopManager.createNextHopPointer(lbGroupKey);
277 List<ActionInfo> actionInfos = new ArrayList<>();
278 for (NexthopManager.AdjacencyResult adjResult : adjacencyResults) {
279 String nextHopIp = adjResult.getNextHopIp();
280 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
281 if (!optionalLabel.isPresent()) {
282 LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
285 long label = optionalLabel.get().toJava();
287 actionInfos.add(new ActionRegLoad(index, FibConstants.NXM_REG_MAPPING.get(index++), 0,
290 List<InstructionInfo> instructions = new ArrayList<>();
291 actionInfos.add(new ActionGroup(index, groupId));
292 instructions.add(new InstructionApplyActions(actionInfos));
293 makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx, subTxns);
294 LOG.trace("End programming remote fib for destPrefix {}, vpnId {}, dpnId {}",
295 vrfEntry.getDestPrefix(), vpnId, remoteDpnId);
298 // Allow deprecated TransactionRunner calls for now
299 @SuppressWarnings("ForbidCertainMethod")
300 public void createRemoteFibEntry(final Uint64 remoteDpnId,
303 final VrfEntry vrfEntry,
305 List<SubTransaction> subTxns) {
307 ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
308 newTx -> createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx, subTxns)), LOG,
309 "Error creating remote FIB entry");
313 LOG.debug("createRemoteFibEntry: adding route {} for rd {} on remoteDpnId {}",
314 vrfEntry.getDestPrefix(), rd, remoteDpnId);
316 List<NexthopManager.AdjacencyResult> adjacencyResults =
317 resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
318 if (adjacencyResults.isEmpty()) {
319 LOG.error("Could not get interface for route-paths: {} in vpn {}", vrfEntry.getRoutePaths(), rd);
320 LOG.warn("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
324 programRemoteFibForBgpRoutes(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, subTxns);
326 LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
329 private void deleteFibEntryForBgpRoutes(Uint64 remoteDpnId, Uint32 vpnId, VrfEntry vrfEntry,
330 String rd, WriteTransaction tx, List<SubTransaction> subTxns) {
331 // When the tunnel is removed the fib entries should be reprogrammed/deleted depending on
332 // the adjacencyResults.
333 List<NexthopManager.AdjacencyResult> adjacencyResults = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
334 if (!adjacencyResults.isEmpty()) {
335 programRemoteFibForBgpRoutes(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, subTxns);
339 // Allow deprecated TransactionRunner calls for now
340 @SuppressWarnings("ForbidCertainMethod")
341 public void deleteRemoteRoute(@Nullable final Uint64 localDpnId, final Uint64 remoteDpnId,
342 final Uint32 vpnId, final VrfTablesKey vrfTableKey,
343 final VrfEntry vrfEntry, Optional<Routes> extraRouteOptional,
344 @Nullable WriteTransaction tx, List<SubTransaction> subTxns) {
346 ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
347 newTx -> deleteRemoteRoute(localDpnId, remoteDpnId, vpnId, vrfTableKey, vrfEntry,
348 extraRouteOptional, newTx)), LOG, "Error deleting remote route");
352 LOG.debug("deleting remote route: prefix={}, vpnId={} localDpnId {} remoteDpnId {}",
353 vrfEntry.getDestPrefix(), vpnId, localDpnId, remoteDpnId);
354 String rd = vrfTableKey.getRouteDistinguisher();
356 if (localDpnId != null && !Uint64.ZERO.equals(localDpnId)) {
357 // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
358 if (extraRouteOptional.isPresent()) {
359 nexthopManager.deleteLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
361 deleteFibEntryForBgpRoutes(remoteDpnId, vpnId, vrfEntry, rd, tx, subTxns);
365 // below two reads are kept as is, until best way is found to identify dpnID
366 VpnNexthop localNextHopInfo = nexthopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
367 if (extraRouteOptional.isPresent()) {
368 nexthopManager.deleteLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
370 checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx, subTxns);
374 public Consumer<? super VrfEntry> getConsumerForCreatingRemoteFib(
375 final Uint64 dpnId, final Uint32 vpnId, final String rd,
376 final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
377 WriteTransaction writeCfgTxn, List<SubTransaction> subTxns) {
378 return vrfEntry -> vrfEntry.nonnullRoutePaths().stream()
379 .filter(routes -> !routes.getNexthopAddress().isEmpty()
380 && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
382 .ifPresent(routes -> {
383 LOG.trace("creating remote FIB entry for prefix {} rd {} on Dpn {}",
384 vrfEntry.getDestPrefix(), rd, dpnId);
385 createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
386 vrfEntry, writeCfgTxn, subTxns);
390 public Consumer<? super VrfEntry> getConsumerForDeletingRemoteFib(
391 final Uint64 dpnId, final Uint32 vpnId,
392 final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
393 WriteTransaction writeCfgTxn, List<SubTransaction> subTxns) {
394 return vrfEntry -> vrfEntry.nonnullRoutePaths().stream()
395 .filter(routes -> !routes.getNexthopAddress().isEmpty()
396 && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
398 .ifPresent(routes -> {
399 LOG.trace(" deleting remote FIB entry {}", vrfEntry);
400 deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().key(), vrfEntry,
401 Optional.empty(), writeCfgTxn, subTxns);
406 protected void addTunnelInterfaceActions(NexthopManager.AdjacencyResult adjacencyResult, Uint32 vpnId,
407 VrfEntry vrfEntry, List<ActionInfo> actionInfos, String rd) {
408 Class<? extends TunnelTypeBase> tunnelType = VpnExtraRouteHelper
409 .getTunnelType(getNextHopManager().getItmManager(), adjacencyResult.getInterfaceName());
410 if (tunnelType == null) {
411 LOG.debug("Tunnel type not found for vrfEntry {}", vrfEntry);
414 String nextHopIp = adjacencyResult.getNextHopIp();
415 if (tunnelType.equals(TunnelTypeMplsOverGre.class)) {
416 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
417 if (!optionalLabel.isPresent()) {
418 LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
421 long label = optionalLabel.get().toJava();
422 LOG.debug("addTunnelInterfaceActions: Push label action for prefix {} rd {} l3vni {} nextHop {}",
423 vrfEntry.getDestPrefix(), rd, vrfEntry.getL3vni(), nextHopIp);
424 actionInfos.add(new ActionPushMpls());
425 actionInfos.add(new ActionSetFieldMplsLabel(label));
426 actionInfos.add(new ActionNxLoadInPort(Uint64.ZERO));
427 } else if (tunnelType.equals(TunnelTypeVxlan.class)) {
428 actionInfos.add(new ActionSetFieldTunnelId(Uint64.valueOf(vrfEntry.getL3vni().longValue())));
429 LOG.debug("addTunnelInterfaceActions: adding set tunnel id action for prefix {} rd {} l3vni {}"
430 + " nextHop {} ", vrfEntry.getDestPrefix(), rd, vrfEntry.getL3vni(), nextHopIp);
431 addRewriteDstMacAction(vpnId, vrfEntry, null /*prefixInfo*/, actionInfos);
436 protected void addRewriteDstMacAction(Uint32 vpnId, VrfEntry vrfEntry, @Nullable Prefixes prefixInfo,
437 List<ActionInfo> actionInfos) {
438 if (vrfEntry.getGatewayMacAddress() != null) {
439 actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
440 new MacAddress(vrfEntry.getGatewayMacAddress())));