2 * Copyright © 2017 Ericsson India Global Services Pvt Ltd. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.netvirt.fibmanager;
10 import static java.util.stream.Collectors.toList;
12 import com.google.common.base.Optional;
13 import com.google.common.base.Preconditions;
14 import java.math.BigInteger;
15 import java.util.ArrayList;
16 import java.util.Collection;
17 import java.util.Collections;
18 import java.util.List;
19 import java.util.concurrent.BlockingQueue;
20 import java.util.concurrent.LinkedBlockingQueue;
21 import java.util.function.Consumer;
22 import javax.annotation.PostConstruct;
23 import javax.inject.Inject;
24 import javax.inject.Singleton;
25 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
26 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
27 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
28 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
29 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
30 import org.opendaylight.genius.mdsalutil.ActionInfo;
31 import org.opendaylight.genius.mdsalutil.InstructionInfo;
32 import org.opendaylight.genius.mdsalutil.NwConstants;
33 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
34 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
35 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
36 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
37 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
38 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldMplsLabel;
39 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
40 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
41 import org.opendaylight.genius.utils.batching.ActionableResource;
42 import org.opendaylight.genius.utils.batching.ActionableResourceImpl;
43 import org.opendaylight.genius.utils.batching.ResourceBatchingManager;
44 import org.opendaylight.genius.utils.batching.ResourceHandler;
45 import org.opendaylight.genius.utils.batching.SubTransaction;
46 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
47 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
48 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
49 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
50 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
51 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
54 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
55 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
56 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
57 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
58 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
59 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
60 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
61 import org.slf4j.Logger;
62 import org.slf4j.LoggerFactory;
66 public class BgpRouteVrfEntryHandler extends BaseVrfEntryHandler
67 implements ResourceHandler, IVrfEntryHandler {
69 private static final Logger LOG = LoggerFactory.getLogger(BgpRouteVrfEntryHandler.class);
70 private static final int BATCH_INTERVAL = 500;
71 private static final int BATCH_SIZE = 1000;
73 private final DataBroker dataBroker;
74 private final ManagedNewTransactionRunner txRunner;
75 private final BlockingQueue<ActionableResource> vrfEntryBufferQ = new LinkedBlockingQueue<>();
76 private final ResourceBatchingManager resourceBatchingManager;
77 private final NexthopManager nexthopManager;
80 public BgpRouteVrfEntryHandler(final DataBroker dataBroker,
81 final NexthopManager nexthopManager,
82 final FibUtil fibUtil) {
83 super(dataBroker, nexthopManager, null, fibUtil);
84 this.dataBroker = dataBroker;
85 this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
86 this.nexthopManager = nexthopManager;
88 resourceBatchingManager = ResourceBatchingManager.getInstance();
89 resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY", vrfEntryBufferQ, this);
94 LOG.info("{} start", getClass().getSimpleName());
99 LOG.info("{} close", getClass().getSimpleName());
103 public DataBroker getResourceBroker() {
108 public int getBatchSize() {
113 public int getBatchInterval() {
114 return BATCH_INTERVAL;
118 public LogicalDatastoreType getDatastoreType() {
119 return LogicalDatastoreType.CONFIGURATION;
123 public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
124 Object original, Object update, List<SubTransaction> subTxns) {
125 if (original instanceof VrfEntry && update instanceof VrfEntry) {
126 createFibEntries(tx, identifier, (VrfEntry) update, subTxns);
131 public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
132 Object vrfEntry, List<SubTransaction> subTxns) {
133 if (vrfEntry instanceof VrfEntry) {
134 createFibEntries(tx, identifier, (VrfEntry) vrfEntry, subTxns);
139 public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
140 Object vrfEntry, List<SubTransaction> subTxns) {
141 if (vrfEntry instanceof VrfEntry) {
142 deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry, subTxns);
147 public void createFlows(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
148 ActionableResource actResource = new ActionableResourceImpl(rd + vrfEntry.getDestPrefix());
149 actResource.setAction(ActionableResource.CREATE);
150 actResource.setInstanceIdentifier(identifier);
151 actResource.setInstance(vrfEntry);
152 vrfEntryBufferQ.add(actResource);
156 public void removeFlows(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
157 ActionableResource actResource = new ActionableResourceImpl(rd + vrfEntry.getDestPrefix());
158 actResource.setAction(ActionableResource.DELETE);
159 actResource.setInstanceIdentifier(identifier);
160 actResource.setInstance(vrfEntry);
161 vrfEntryBufferQ.add(actResource);
165 public void updateFlows(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update, String rd) {
166 ActionableResource actResource = new ActionableResourceImpl(rd + update.getDestPrefix());
167 actResource.setAction(ActionableResource.UPDATE);
168 actResource.setInstanceIdentifier(identifier);
169 actResource.setInstance(update);
170 actResource.setOldInstance(original);
171 vrfEntryBufferQ.add(actResource);
175 Please note that the following createFibEntries will be invoked only for BGP Imported Routes.
176 The invocation of the following method is via create() callback from the MDSAL Batching Infrastructure
177 provided by ResourceBatchingManager
179 private void createFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> vrfEntryIid,
180 final VrfEntry vrfEntry, List<SubTransaction> subTxns) {
181 final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
183 final VpnInstanceOpDataEntry vpnInstance =
184 getFibUtil().getVpnInstance(vrfTableKey.getRouteDistinguisher());
185 Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
186 Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId()
187 + " has null vpnId!");
189 final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
190 if (vpnToDpnList != null) {
191 for (VpnToDpnList vpnDpn : vpnToDpnList) {
192 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
193 createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(),
194 vrfEntry, writeTx, subTxns);
201 Please note that the following deleteFibEntries will be invoked only for BGP Imported Routes.
202 The invocation of the following method is via delete() callback from the MDSAL Batching Infrastructure
203 provided by ResourceBatchingManager
205 private void deleteFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> identifier,
206 final VrfEntry vrfEntry, List<SubTransaction> subTxns) {
207 final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
208 String rd = vrfTableKey.getRouteDistinguisher();
209 final VpnInstanceOpDataEntry vpnInstance =
210 getFibUtil().getVpnInstance(vrfTableKey.getRouteDistinguisher());
211 if (vpnInstance == null) {
212 LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
215 String vpnName = getFibUtil().getVpnNameFromId(vpnInstance.getVpnId());
216 final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
217 if (vpnToDpnList != null) {
218 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
219 vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
220 Optional<Routes> extraRouteOptional;
221 //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
222 if (usedRds != null && !usedRds.isEmpty()) {
223 if (usedRds.size() > 1) {
224 LOG.error("The extra route prefix is still present in some DPNs");
227 extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
228 usedRds.get(0), vrfEntry.getDestPrefix());
231 extraRouteOptional = Optional.absent();
233 for (VpnToDpnList curDpn : vpnToDpnList) {
234 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
235 deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(),
236 vrfTableKey, vrfEntry, extraRouteOptional, writeTx, subTxns);
242 public void programRemoteFibForBgpRoutes(final BigInteger remoteDpnId,
244 final VrfEntry vrfEntry,
247 List<NexthopManager.AdjacencyResult> adjacencyResults,
248 List<SubTransaction> subTxns) {
249 Preconditions.checkArgument(vrfEntry.getRoutePaths().size() <= 2);
251 if (adjacencyResults.size() == 1) {
252 programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, subTxns);
255 // ECMP Use case, point to LB group. Move the mpls label accordingly.
256 List<String> tunnelList =
257 adjacencyResults.stream()
258 .map(NexthopManager.AdjacencyResult::getNextHopIp)
259 .sorted().collect(toList());
260 String lbGroupKey = FibUtil.getGreLbGroupKey(tunnelList);
261 long groupId = nexthopManager.createNextHopPointer(lbGroupKey);
263 List<ActionInfo> actionInfos = new ArrayList<>();
264 for (NexthopManager.AdjacencyResult adjResult : adjacencyResults) {
265 String nextHopIp = adjResult.getNextHopIp();
266 java.util.Optional<Long> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
267 if (!optionalLabel.isPresent()) {
268 LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
271 long label = optionalLabel.get();
273 actionInfos.add(new ActionRegLoad(index, FibConstants.NXM_REG_MAPPING.get(index++), 0,
276 List<InstructionInfo> instructions = new ArrayList<>();
277 actionInfos.add(new ActionGroup(index, groupId));
278 instructions.add(new InstructionApplyActions(actionInfos));
279 makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx, subTxns);
282 public void createRemoteFibEntry(final BigInteger remoteDpnId,
285 final VrfEntry vrfEntry,
287 List<SubTransaction> subTxns) {
289 ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
290 newTx -> createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx, subTxns)), LOG,
291 "Error creating remote FIB entry");
295 LOG.debug("createRemoteFibEntry: adding route {} for rd {} on remoteDpnId {}",
296 vrfEntry.getDestPrefix(), rd, remoteDpnId);
298 List<NexthopManager.AdjacencyResult> adjacencyResults =
299 resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
300 if (adjacencyResults.isEmpty()) {
301 LOG.error("Could not get interface for route-paths: {} in vpn {}", vrfEntry.getRoutePaths(), rd);
302 LOG.warn("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
306 programRemoteFibForBgpRoutes(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, subTxns);
308 LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
311 private void deleteFibEntryForBgpRoutes(BigInteger remoteDpnId, long vpnId, VrfEntry vrfEntry,
312 String rd, WriteTransaction tx, List<SubTransaction> subTxns) {
313 // When the tunnel is removed the fib entries should be reprogrammed/deleted depending on
314 // the adjacencyResults.
315 List<NexthopManager.AdjacencyResult> adjacencyResults = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
316 if (!adjacencyResults.isEmpty()) {
317 programRemoteFibForBgpRoutes(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, subTxns);
321 public void deleteRemoteRoute(final BigInteger localDpnId, final BigInteger remoteDpnId,
322 final long vpnId, final VrfTablesKey vrfTableKey,
323 final VrfEntry vrfEntry, Optional<Routes> extraRouteOptional,
324 WriteTransaction tx, List<SubTransaction> subTxns) {
326 ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
327 newTx -> deleteRemoteRoute(localDpnId, remoteDpnId, vpnId, vrfTableKey, vrfEntry,
328 extraRouteOptional, newTx)), LOG, "Error deleting remote route");
332 LOG.debug("deleting remote route: prefix={}, vpnId={} localDpnId {} remoteDpnId {}",
333 vrfEntry.getDestPrefix(), vpnId, localDpnId, remoteDpnId);
334 String rd = vrfTableKey.getRouteDistinguisher();
336 if (localDpnId != null && localDpnId != BigInteger.ZERO) {
337 // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
338 if (extraRouteOptional.isPresent()) {
339 nexthopManager.setupLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix(),
340 Collections.emptyList() /*listBucketInfo*/ , false);
342 deleteFibEntryForBgpRoutes(remoteDpnId, vpnId, vrfEntry, rd, tx, subTxns);
346 // below two reads are kept as is, until best way is found to identify dpnID
347 VpnNexthop localNextHopInfo = nexthopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
348 if (extraRouteOptional.isPresent()) {
349 nexthopManager.setupLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix(),
350 Collections.emptyList() /*listBucketInfo*/ , false);
352 checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx, subTxns);
356 public Consumer<? super VrfEntry> getConsumerForCreatingRemoteFib(
357 final BigInteger dpnId, final long vpnId, final String rd,
358 final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
359 WriteTransaction writeCfgTxn, List<SubTransaction> subTxns) {
360 return vrfEntry -> vrfEntry.getRoutePaths().stream()
361 .filter(routes -> !routes.getNexthopAddress().isEmpty()
362 && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
364 .ifPresent(routes -> {
365 LOG.trace("creating remote FIB entry for prefix {} rd {} on Dpn {}",
366 vrfEntry.getDestPrefix(), rd, dpnId);
367 createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
368 vrfEntry, writeCfgTxn, subTxns);
372 public Consumer<? super VrfEntry> getConsumerForDeletingRemoteFib(
373 final BigInteger dpnId, final long vpnId,
374 final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
375 WriteTransaction writeCfgTxn, List<SubTransaction> subTxns) {
376 return vrfEntry -> vrfEntry.getRoutePaths().stream()
377 .filter(routes -> !routes.getNexthopAddress().isEmpty()
378 && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
380 .ifPresent(routes -> {
381 LOG.trace(" deleting remote FIB entry {}", vrfEntry);
382 deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().key(), vrfEntry,
383 Optional.absent(), writeCfgTxn, subTxns);
388 protected void addTunnelInterfaceActions(NexthopManager.AdjacencyResult adjacencyResult, long vpnId,
389 VrfEntry vrfEntry, List<ActionInfo> actionInfos, String rd) {
390 Class<? extends TunnelTypeBase> tunnelType = VpnExtraRouteHelper
391 .getTunnelType(getNextHopManager().getItmManager(), adjacencyResult.getInterfaceName());
392 if (tunnelType == null) {
393 LOG.debug("Tunnel type not found for vrfEntry {}", vrfEntry);
396 String nextHopIp = adjacencyResult.getNextHopIp();
397 if (tunnelType.equals(TunnelTypeMplsOverGre.class)) {
398 java.util.Optional<Long> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
399 if (!optionalLabel.isPresent()) {
400 LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
403 long label = optionalLabel.get();
404 LOG.debug("addTunnelInterfaceActions: Push label action for prefix {} rd {} l3vni {} nextHop {}",
405 vrfEntry.getDestPrefix(), rd, vrfEntry.getL3vni(), nextHopIp);
406 actionInfos.add(new ActionPushMpls());
407 actionInfos.add(new ActionSetFieldMplsLabel(label));
408 actionInfos.add(new ActionNxLoadInPort(BigInteger.ZERO));
409 } else if (tunnelType.equals(TunnelTypeVxlan.class)) {
410 actionInfos.add(new ActionSetFieldTunnelId(BigInteger.valueOf(vrfEntry.getL3vni())));
411 LOG.debug("addTunnelInterfaceActions: adding set tunnel id action for prefix {} rd {} l3vni {}"
412 + " nextHop {} ", vrfEntry.getDestPrefix(), rd, vrfEntry.getL3vni(), nextHopIp);
413 addRewriteDstMacAction(vpnId, vrfEntry, null /*prefixInfo*/, actionInfos);
418 protected void addRewriteDstMacAction(long vpnId, VrfEntry vrfEntry, Prefixes prefixInfo,
419 List<ActionInfo> actionInfos) {
420 if (vrfEntry.getGatewayMacAddress() != null) {
421 actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
422 new MacAddress(vrfEntry.getGatewayMacAddress())));