2 * Copyright © 2017 Ericsson India Global Services Pvt Ltd. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.netvirt.fibmanager;
10 import static java.util.stream.Collectors.toList;
12 import com.google.common.base.Optional;
13 import java.util.ArrayList;
14 import java.util.Collection;
15 import java.util.List;
16 import java.util.concurrent.BlockingQueue;
17 import java.util.concurrent.LinkedBlockingQueue;
18 import java.util.function.Consumer;
19 import javax.annotation.PostConstruct;
20 import javax.inject.Inject;
21 import javax.inject.Singleton;
22 import org.eclipse.jdt.annotation.Nullable;
23 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
24 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
25 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
26 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
27 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
28 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
29 import org.opendaylight.genius.mdsalutil.ActionInfo;
30 import org.opendaylight.genius.mdsalutil.InstructionInfo;
31 import org.opendaylight.genius.mdsalutil.NwConstants;
32 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
33 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
34 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
35 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
36 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
37 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldMplsLabel;
38 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
39 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
40 import org.opendaylight.genius.utils.batching.ActionableResource;
41 import org.opendaylight.genius.utils.batching.ActionableResourceImpl;
42 import org.opendaylight.genius.utils.batching.ResourceBatchingManager;
43 import org.opendaylight.genius.utils.batching.ResourceHandler;
44 import org.opendaylight.genius.utils.batching.SubTransaction;
45 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
46 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
47 import org.opendaylight.serviceutils.upgrade.UpgradeState;
48 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
49 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
50 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
51 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
54 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
55 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
56 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
57 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
58 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
59 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
60 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
61 import org.opendaylight.yangtools.yang.common.Uint32;
62 import org.opendaylight.yangtools.yang.common.Uint64;
63 import org.slf4j.Logger;
64 import org.slf4j.LoggerFactory;
68 public class BgpRouteVrfEntryHandler extends BaseVrfEntryHandler implements ResourceHandler {
70 private static final Logger LOG = LoggerFactory.getLogger(BgpRouteVrfEntryHandler.class);
71 private static final int BATCH_INTERVAL = 500;
72 private static final int BATCH_SIZE = 1000;
74 private final DataBroker dataBroker;
75 private final ManagedNewTransactionRunner txRunner;
76 private final BlockingQueue<ActionableResource> vrfEntryBufferQ = new LinkedBlockingQueue<>();
77 private final ResourceBatchingManager resourceBatchingManager;
78 private final NexthopManager nexthopManager;
81 public BgpRouteVrfEntryHandler(final DataBroker dataBroker,
82 final NexthopManager nexthopManager,
83 final FibUtil fibUtil,
84 final UpgradeState upgradeState,
85 final DataTreeEventCallbackRegistrar eventCallbacks) {
86 super(dataBroker, nexthopManager, null, fibUtil, upgradeState, eventCallbacks);
87 this.dataBroker = dataBroker;
88 this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
89 this.nexthopManager = nexthopManager;
91 resourceBatchingManager = ResourceBatchingManager.getInstance();
92 resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY", vrfEntryBufferQ, this);
97 LOG.info("{} start", getClass().getSimpleName());
101 public void close() {
102 LOG.info("{} close", getClass().getSimpleName());
106 public DataBroker getResourceBroker() {
111 public int getBatchSize() {
116 public int getBatchInterval() {
117 return BATCH_INTERVAL;
121 public LogicalDatastoreType getDatastoreType() {
122 return LogicalDatastoreType.CONFIGURATION;
126 public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
127 Object original, Object update, List<SubTransaction> subTxns) {
128 if (original instanceof VrfEntry && update instanceof VrfEntry) {
129 createFibEntries(tx, identifier, (VrfEntry) update, subTxns);
134 public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
135 Object vrfEntry, List<SubTransaction> subTxns) {
136 if (vrfEntry instanceof VrfEntry) {
137 createFibEntries(tx, identifier, (VrfEntry) vrfEntry, subTxns);
142 public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
143 Object vrfEntry, List<SubTransaction> subTxns) {
144 if (vrfEntry instanceof VrfEntry) {
145 deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry, subTxns);
149 void createFlows(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
150 ActionableResourceImpl actResource = new ActionableResourceImpl(rd + vrfEntry.getDestPrefix());
151 actResource.setAction(ActionableResource.CREATE);
152 actResource.setInstanceIdentifier(identifier);
153 actResource.setInstance(vrfEntry);
154 vrfEntryBufferQ.add(actResource);
157 void removeFlows(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
158 ActionableResourceImpl actResource = new ActionableResourceImpl(rd + vrfEntry.getDestPrefix());
159 actResource.setAction(ActionableResource.DELETE);
160 actResource.setInstanceIdentifier(identifier);
161 actResource.setInstance(vrfEntry);
162 vrfEntryBufferQ.add(actResource);
165 void updateFlows(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update, String rd) {
166 ActionableResourceImpl actResource = new ActionableResourceImpl(rd + update.getDestPrefix());
167 actResource.setAction(ActionableResource.UPDATE);
168 actResource.setInstanceIdentifier(identifier);
169 actResource.setInstance(update);
170 actResource.setOldInstance(original);
171 vrfEntryBufferQ.add(actResource);
175 Please note that the following createFibEntries will be invoked only for BGP Imported Routes.
176 The invocation of the following method is via create() callback from the MDSAL Batching Infrastructure
177 provided by ResourceBatchingManager
179 private void createFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> vrfEntryIid,
180 final VrfEntry vrfEntry, List<SubTransaction> subTxns) {
181 final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
182 LOG.trace("Creating fib entry for vrfEntry with destPrefix{}, rd {}",
183 vrfEntry.getDestPrefix(), vrfTableKey.getRouteDistinguisher());
184 final VpnInstanceOpDataEntry vpnInstance =
185 getFibUtil().getVpnInstance(vrfTableKey.getRouteDistinguisher());
186 if (vpnInstance == null || vpnInstance.getVpnId() == null) {
187 LOG.error("Vpn Instance not availabe {}", vrfTableKey.getRouteDistinguisher());
190 final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
191 if (vpnToDpnList != null) {
192 for (VpnToDpnList vpnDpn : vpnToDpnList) {
193 LOG.trace("Dpnstate is {} for dpn {} in vpn {}", vpnDpn.getDpnState(), vpnDpn.getDpnId(),
194 vpnInstance.getVpnId());
195 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
196 createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(),
197 vrfTableKey.getRouteDistinguisher(), vrfEntry, writeTx, subTxns);
201 LOG.trace("Created fib entry for vrfEntry with destPrefix{}, rd {}",
202 vrfEntry.getDestPrefix(), vrfTableKey.getRouteDistinguisher());
206 Please note that the following deleteFibEntries will be invoked only for BGP Imported Routes.
207 The invocation of the following method is via delete() callback from the MDSAL Batching Infrastructure
208 provided by ResourceBatchingManager
210 private void deleteFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> identifier,
211 final VrfEntry vrfEntry, List<SubTransaction> subTxns) {
212 final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
213 String rd = vrfTableKey.getRouteDistinguisher();
214 final VpnInstanceOpDataEntry vpnInstance =
215 getFibUtil().getVpnInstance(vrfTableKey.getRouteDistinguisher());
216 if (vpnInstance == null) {
217 LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
220 String vpnName = getFibUtil().getVpnNameFromId(vpnInstance.getVpnId());
221 final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
222 if (vpnToDpnList != null) {
223 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
224 vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
225 Optional<Routes> extraRouteOptional;
226 //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
227 if (usedRds != null && !usedRds.isEmpty()) {
228 if (usedRds.size() > 1) {
229 LOG.error("The extra route prefix is still present in some DPNs");
232 extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
233 usedRds.get(0), vrfEntry.getDestPrefix());
236 extraRouteOptional = Optional.absent();
238 for (VpnToDpnList curDpn : vpnToDpnList) {
239 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
240 deleteRemoteRoute(Uint64.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(),
241 vrfTableKey, vrfEntry, extraRouteOptional, writeTx, subTxns);
247 public void programRemoteFibForBgpRoutes(final Uint64 remoteDpnId,
249 final VrfEntry vrfEntry,
252 List<NexthopManager.AdjacencyResult> adjacencyResults,
253 List<SubTransaction> subTxns) {
254 if (vrfEntry.nonnullRoutePaths().size() > 2) {
255 LOG.error("DC-GW can advertise only 2 bestPaths for prefix {}", vrfEntry.getDestPrefix());
258 LOG.trace("Start programming remote fib for destPrefix {}, vpnId {}, dpnId {}",
259 vrfEntry.getDestPrefix(), vpnId, remoteDpnId);
260 if (adjacencyResults.size() == 1) {
261 programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, subTxns);
264 // ECMP Use case, point to LB group. Move the mpls label accordingly.
265 List<String> tunnelList =
266 adjacencyResults.stream()
267 .map(NexthopManager.AdjacencyResult::getNextHopIp)
268 .sorted().collect(toList());
269 String lbGroupKey = FibUtil.getGreLbGroupKey(tunnelList);
270 long groupId = nexthopManager.createNextHopPointer(lbGroupKey);
272 List<ActionInfo> actionInfos = new ArrayList<>();
273 for (NexthopManager.AdjacencyResult adjResult : adjacencyResults) {
274 String nextHopIp = adjResult.getNextHopIp();
275 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
276 if (!optionalLabel.isPresent()) {
277 LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
280 long label = optionalLabel.get().toJava();
282 actionInfos.add(new ActionRegLoad(index, FibConstants.NXM_REG_MAPPING.get(index++), 0,
285 List<InstructionInfo> instructions = new ArrayList<>();
286 actionInfos.add(new ActionGroup(index, groupId));
287 instructions.add(new InstructionApplyActions(actionInfos));
288 makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx, subTxns);
289 LOG.trace("End programming remote fib for destPrefix {}, vpnId {}, dpnId {}",
290 vrfEntry.getDestPrefix(), vpnId, remoteDpnId);
293 // Allow deprecated TransactionRunner calls for now
294 @SuppressWarnings("ForbidCertainMethod")
295 public void createRemoteFibEntry(final Uint64 remoteDpnId,
298 final VrfEntry vrfEntry,
300 List<SubTransaction> subTxns) {
302 ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
303 newTx -> createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx, subTxns)), LOG,
304 "Error creating remote FIB entry");
308 LOG.debug("createRemoteFibEntry: adding route {} for rd {} on remoteDpnId {}",
309 vrfEntry.getDestPrefix(), rd, remoteDpnId);
311 List<NexthopManager.AdjacencyResult> adjacencyResults =
312 resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
313 if (adjacencyResults.isEmpty()) {
314 LOG.error("Could not get interface for route-paths: {} in vpn {}", vrfEntry.getRoutePaths(), rd);
315 LOG.warn("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
319 programRemoteFibForBgpRoutes(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, subTxns);
321 LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
324 private void deleteFibEntryForBgpRoutes(Uint64 remoteDpnId, Uint32 vpnId, VrfEntry vrfEntry,
325 String rd, WriteTransaction tx, List<SubTransaction> subTxns) {
326 // When the tunnel is removed the fib entries should be reprogrammed/deleted depending on
327 // the adjacencyResults.
328 List<NexthopManager.AdjacencyResult> adjacencyResults = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
329 if (!adjacencyResults.isEmpty()) {
330 programRemoteFibForBgpRoutes(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, subTxns);
334 // Allow deprecated TransactionRunner calls for now
335 @SuppressWarnings("ForbidCertainMethod")
336 public void deleteRemoteRoute(@Nullable final Uint64 localDpnId, final Uint64 remoteDpnId,
337 final Uint32 vpnId, final VrfTablesKey vrfTableKey,
338 final VrfEntry vrfEntry, Optional<Routes> extraRouteOptional,
339 @Nullable WriteTransaction tx, List<SubTransaction> subTxns) {
341 ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
342 newTx -> deleteRemoteRoute(localDpnId, remoteDpnId, vpnId, vrfTableKey, vrfEntry,
343 extraRouteOptional, newTx)), LOG, "Error deleting remote route");
347 LOG.debug("deleting remote route: prefix={}, vpnId={} localDpnId {} remoteDpnId {}",
348 vrfEntry.getDestPrefix(), vpnId, localDpnId, remoteDpnId);
349 String rd = vrfTableKey.getRouteDistinguisher();
351 if (localDpnId != null && !Uint64.ZERO.equals(localDpnId)) {
352 // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
353 if (extraRouteOptional.isPresent()) {
354 nexthopManager.deleteLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
356 deleteFibEntryForBgpRoutes(remoteDpnId, vpnId, vrfEntry, rd, tx, subTxns);
360 // below two reads are kept as is, until best way is found to identify dpnID
361 VpnNexthop localNextHopInfo = nexthopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
362 if (extraRouteOptional.isPresent()) {
363 nexthopManager.deleteLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
365 checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx, subTxns);
369 public Consumer<? super VrfEntry> getConsumerForCreatingRemoteFib(
370 final Uint64 dpnId, final Uint32 vpnId, final String rd,
371 final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
372 WriteTransaction writeCfgTxn, List<SubTransaction> subTxns) {
373 return vrfEntry -> vrfEntry.nonnullRoutePaths().stream()
374 .filter(routes -> !routes.getNexthopAddress().isEmpty()
375 && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
377 .ifPresent(routes -> {
378 LOG.trace("creating remote FIB entry for prefix {} rd {} on Dpn {}",
379 vrfEntry.getDestPrefix(), rd, dpnId);
380 createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
381 vrfEntry, writeCfgTxn, subTxns);
385 public Consumer<? super VrfEntry> getConsumerForDeletingRemoteFib(
386 final Uint64 dpnId, final Uint32 vpnId,
387 final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
388 WriteTransaction writeCfgTxn, List<SubTransaction> subTxns) {
389 return vrfEntry -> vrfEntry.nonnullRoutePaths().stream()
390 .filter(routes -> !routes.getNexthopAddress().isEmpty()
391 && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
393 .ifPresent(routes -> {
394 LOG.trace(" deleting remote FIB entry {}", vrfEntry);
395 deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().key(), vrfEntry,
396 Optional.absent(), writeCfgTxn, subTxns);
401 protected void addTunnelInterfaceActions(NexthopManager.AdjacencyResult adjacencyResult, Uint32 vpnId,
402 VrfEntry vrfEntry, List<ActionInfo> actionInfos, String rd) {
403 Class<? extends TunnelTypeBase> tunnelType = VpnExtraRouteHelper
404 .getTunnelType(getNextHopManager().getItmManager(), adjacencyResult.getInterfaceName());
405 if (tunnelType == null) {
406 LOG.debug("Tunnel type not found for vrfEntry {}", vrfEntry);
409 String nextHopIp = adjacencyResult.getNextHopIp();
410 if (tunnelType.equals(TunnelTypeMplsOverGre.class)) {
411 java.util.Optional<Uint32> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
412 if (!optionalLabel.isPresent()) {
413 LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
416 long label = optionalLabel.get().toJava();
417 LOG.debug("addTunnelInterfaceActions: Push label action for prefix {} rd {} l3vni {} nextHop {}",
418 vrfEntry.getDestPrefix(), rd, vrfEntry.getL3vni(), nextHopIp);
419 actionInfos.add(new ActionPushMpls());
420 actionInfos.add(new ActionSetFieldMplsLabel(label));
421 actionInfos.add(new ActionNxLoadInPort(Uint64.ZERO));
422 } else if (tunnelType.equals(TunnelTypeVxlan.class)) {
423 actionInfos.add(new ActionSetFieldTunnelId(Uint64.valueOf(vrfEntry.getL3vni().longValue())));
424 LOG.debug("addTunnelInterfaceActions: adding set tunnel id action for prefix {} rd {} l3vni {}"
425 + " nextHop {} ", vrfEntry.getDestPrefix(), rd, vrfEntry.getL3vni(), nextHopIp);
426 addRewriteDstMacAction(vpnId, vrfEntry, null /*prefixInfo*/, actionInfos);
431 protected void addRewriteDstMacAction(Uint32 vpnId, VrfEntry vrfEntry, @Nullable Prefixes prefixInfo,
432 List<ActionInfo> actionInfos) {
433 if (vrfEntry.getGatewayMacAddress() != null) {
434 actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
435 new MacAddress(vrfEntry.getGatewayMacAddress())));