2 * Copyright © 2017 Ericsson India Global Services Pvt Ltd. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.netvirt.fibmanager;
10 import static java.util.stream.Collectors.toList;
12 import com.google.common.base.Optional;
13 import java.math.BigInteger;
14 import java.util.ArrayList;
15 import java.util.Collection;
16 import java.util.List;
17 import java.util.concurrent.BlockingQueue;
18 import java.util.concurrent.LinkedBlockingQueue;
19 import java.util.function.Consumer;
20 import javax.annotation.Nullable;
21 import javax.annotation.PostConstruct;
22 import javax.inject.Inject;
23 import javax.inject.Singleton;
24 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
25 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
26 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
27 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
28 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
29 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
30 import org.opendaylight.genius.mdsalutil.ActionInfo;
31 import org.opendaylight.genius.mdsalutil.InstructionInfo;
32 import org.opendaylight.genius.mdsalutil.NwConstants;
33 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
34 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
35 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
36 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
37 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
38 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldMplsLabel;
39 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
40 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
41 import org.opendaylight.genius.utils.batching.ActionableResource;
42 import org.opendaylight.genius.utils.batching.ActionableResourceImpl;
43 import org.opendaylight.genius.utils.batching.ResourceBatchingManager;
44 import org.opendaylight.genius.utils.batching.ResourceHandler;
45 import org.opendaylight.genius.utils.batching.SubTransaction;
46 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
47 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
48 import org.opendaylight.serviceutils.upgrade.UpgradeState;
49 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
50 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
51 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
52 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
54 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
55 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
56 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
57 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
58 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
59 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
60 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
61 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
62 import org.slf4j.Logger;
63 import org.slf4j.LoggerFactory;
67 public class BgpRouteVrfEntryHandler extends BaseVrfEntryHandler
68 implements ResourceHandler, IVrfEntryHandler {
70 private static final Logger LOG = LoggerFactory.getLogger(BgpRouteVrfEntryHandler.class);
71 private static final int BATCH_INTERVAL = 500;
72 private static final int BATCH_SIZE = 1000;
74 private final DataBroker dataBroker;
75 private final ManagedNewTransactionRunner txRunner;
76 private final BlockingQueue<ActionableResource> vrfEntryBufferQ = new LinkedBlockingQueue<>();
77 private final ResourceBatchingManager resourceBatchingManager;
78 private final NexthopManager nexthopManager;
81 public BgpRouteVrfEntryHandler(final DataBroker dataBroker,
82 final NexthopManager nexthopManager,
83 final FibUtil fibUtil,
84 final UpgradeState upgradeState,
85 final DataTreeEventCallbackRegistrar eventCallbacks) {
86 super(dataBroker, nexthopManager, null, fibUtil, upgradeState, eventCallbacks);
87 this.dataBroker = dataBroker;
88 this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
89 this.nexthopManager = nexthopManager;
91 resourceBatchingManager = ResourceBatchingManager.getInstance();
92 resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY", vrfEntryBufferQ, this);
97 LOG.info("{} start", getClass().getSimpleName());
101 public void close() {
102 LOG.info("{} close", getClass().getSimpleName());
106 public DataBroker getResourceBroker() {
111 public int getBatchSize() {
116 public int getBatchInterval() {
117 return BATCH_INTERVAL;
121 public LogicalDatastoreType getDatastoreType() {
122 return LogicalDatastoreType.CONFIGURATION;
126 public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
127 Object original, Object update, List<SubTransaction> subTxns) {
128 if (original instanceof VrfEntry && update instanceof VrfEntry) {
129 createFibEntries(tx, identifier, (VrfEntry) update, subTxns);
134 public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
135 Object vrfEntry, List<SubTransaction> subTxns) {
136 if (vrfEntry instanceof VrfEntry) {
137 createFibEntries(tx, identifier, (VrfEntry) vrfEntry, subTxns);
142 public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
143 Object vrfEntry, List<SubTransaction> subTxns) {
144 if (vrfEntry instanceof VrfEntry) {
145 deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry, subTxns);
150 public void createFlows(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
151 ActionableResource actResource = new ActionableResourceImpl(rd + vrfEntry.getDestPrefix());
152 actResource.setAction(ActionableResource.CREATE);
153 actResource.setInstanceIdentifier(identifier);
154 actResource.setInstance(vrfEntry);
155 vrfEntryBufferQ.add(actResource);
159 public void removeFlows(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
160 ActionableResource actResource = new ActionableResourceImpl(rd + vrfEntry.getDestPrefix());
161 actResource.setAction(ActionableResource.DELETE);
162 actResource.setInstanceIdentifier(identifier);
163 actResource.setInstance(vrfEntry);
164 vrfEntryBufferQ.add(actResource);
168 public void updateFlows(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update, String rd) {
169 ActionableResource actResource = new ActionableResourceImpl(rd + update.getDestPrefix());
170 actResource.setAction(ActionableResource.UPDATE);
171 actResource.setInstanceIdentifier(identifier);
172 actResource.setInstance(update);
173 actResource.setOldInstance(original);
174 vrfEntryBufferQ.add(actResource);
178 Please note that the following createFibEntries will be invoked only for BGP Imported Routes.
179 The invocation of the following method is via create() callback from the MDSAL Batching Infrastructure
180 provided by ResourceBatchingManager
182 private void createFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> vrfEntryIid,
183 final VrfEntry vrfEntry, List<SubTransaction> subTxns) {
184 final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
185 LOG.trace("Creating fib entry for vrfEntry with destPrefix{}, rd {}",
186 vrfEntry.getDestPrefix(), vrfTableKey.getRouteDistinguisher());
187 final VpnInstanceOpDataEntry vpnInstance =
188 getFibUtil().getVpnInstance(vrfTableKey.getRouteDistinguisher());
189 if (vpnInstance == null || vpnInstance.getVpnId() == null) {
190 LOG.error("Vpn Instance not availabe {}", vrfTableKey.getRouteDistinguisher());
193 final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
194 if (vpnToDpnList != null) {
195 for (VpnToDpnList vpnDpn : vpnToDpnList) {
196 LOG.trace("Dpnstate is {} for dpn {} in vpn {}", vpnDpn.getDpnState(), vpnDpn.getDpnId(),
197 vpnInstance.getVpnId());
198 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
199 createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(),
200 vrfEntry, writeTx, subTxns);
204 LOG.trace("Created fib entry for vrfEntry with destPrefix{}, rd {}",
205 vrfEntry.getDestPrefix(), vrfTableKey.getRouteDistinguisher());
209 Please note that the following deleteFibEntries will be invoked only for BGP Imported Routes.
210 The invocation of the following method is via delete() callback from the MDSAL Batching Infrastructure
211 provided by ResourceBatchingManager
213 private void deleteFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> identifier,
214 final VrfEntry vrfEntry, List<SubTransaction> subTxns) {
215 final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
216 String rd = vrfTableKey.getRouteDistinguisher();
217 final VpnInstanceOpDataEntry vpnInstance =
218 getFibUtil().getVpnInstance(vrfTableKey.getRouteDistinguisher());
219 if (vpnInstance == null) {
220 LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
223 String vpnName = getFibUtil().getVpnNameFromId(vpnInstance.getVpnId());
224 final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
225 if (vpnToDpnList != null) {
226 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
227 vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
228 Optional<Routes> extraRouteOptional;
229 //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
230 if (usedRds != null && !usedRds.isEmpty()) {
231 if (usedRds.size() > 1) {
232 LOG.error("The extra route prefix is still present in some DPNs");
235 extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
236 usedRds.get(0), vrfEntry.getDestPrefix());
239 extraRouteOptional = Optional.absent();
241 for (VpnToDpnList curDpn : vpnToDpnList) {
242 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
243 deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(),
244 vrfTableKey, vrfEntry, extraRouteOptional, writeTx, subTxns);
250 public void programRemoteFibForBgpRoutes(final BigInteger remoteDpnId,
252 final VrfEntry vrfEntry,
255 List<NexthopManager.AdjacencyResult> adjacencyResults,
256 List<SubTransaction> subTxns) {
257 if (vrfEntry.nonnullRoutePaths().size() > 2) {
258 LOG.error("DC-GW can advertise only 2 bestPaths for prefix {}", vrfEntry.getDestPrefix());
261 LOG.trace("Start programming remote fib for destPrefix {}, vpnId {}, dpnId {}",
262 vrfEntry.getDestPrefix(), vpnId, remoteDpnId);
263 if (adjacencyResults.size() == 1) {
264 programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, subTxns);
267 // ECMP Use case, point to LB group. Move the mpls label accordingly.
268 List<String> tunnelList =
269 adjacencyResults.stream()
270 .map(NexthopManager.AdjacencyResult::getNextHopIp)
271 .sorted().collect(toList());
272 String lbGroupKey = FibUtil.getGreLbGroupKey(tunnelList);
273 long groupId = nexthopManager.createNextHopPointer(lbGroupKey);
275 List<ActionInfo> actionInfos = new ArrayList<>();
276 for (NexthopManager.AdjacencyResult adjResult : adjacencyResults) {
277 String nextHopIp = adjResult.getNextHopIp();
278 java.util.Optional<Long> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
279 if (!optionalLabel.isPresent()) {
280 LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
283 long label = optionalLabel.get();
285 actionInfos.add(new ActionRegLoad(index, FibConstants.NXM_REG_MAPPING.get(index++), 0,
288 List<InstructionInfo> instructions = new ArrayList<>();
289 actionInfos.add(new ActionGroup(index, groupId));
290 instructions.add(new InstructionApplyActions(actionInfos));
291 makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx, subTxns);
292 LOG.trace("End programming remote fib for destPrefix {}, vpnId {}, dpnId {}",
293 vrfEntry.getDestPrefix(), vpnId, remoteDpnId);
296 public void createRemoteFibEntry(final BigInteger remoteDpnId,
299 final VrfEntry vrfEntry,
301 List<SubTransaction> subTxns) {
303 ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
304 newTx -> createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx, subTxns)), LOG,
305 "Error creating remote FIB entry");
309 LOG.debug("createRemoteFibEntry: adding route {} for rd {} on remoteDpnId {}",
310 vrfEntry.getDestPrefix(), rd, remoteDpnId);
312 List<NexthopManager.AdjacencyResult> adjacencyResults =
313 resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
314 if (adjacencyResults.isEmpty()) {
315 LOG.error("Could not get interface for route-paths: {} in vpn {}", vrfEntry.getRoutePaths(), rd);
316 LOG.warn("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
320 programRemoteFibForBgpRoutes(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, subTxns);
322 LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
325 private void deleteFibEntryForBgpRoutes(BigInteger remoteDpnId, long vpnId, VrfEntry vrfEntry,
326 String rd, WriteTransaction tx, List<SubTransaction> subTxns) {
327 // When the tunnel is removed the fib entries should be reprogrammed/deleted depending on
328 // the adjacencyResults.
329 List<NexthopManager.AdjacencyResult> adjacencyResults = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
330 if (!adjacencyResults.isEmpty()) {
331 programRemoteFibForBgpRoutes(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, subTxns);
335 public void deleteRemoteRoute(@Nullable final BigInteger localDpnId, final BigInteger remoteDpnId,
336 final long vpnId, final VrfTablesKey vrfTableKey,
337 final VrfEntry vrfEntry, Optional<Routes> extraRouteOptional,
338 @Nullable WriteTransaction tx, List<SubTransaction> subTxns) {
340 ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
341 newTx -> deleteRemoteRoute(localDpnId, remoteDpnId, vpnId, vrfTableKey, vrfEntry,
342 extraRouteOptional, newTx)), LOG, "Error deleting remote route");
346 LOG.debug("deleting remote route: prefix={}, vpnId={} localDpnId {} remoteDpnId {}",
347 vrfEntry.getDestPrefix(), vpnId, localDpnId, remoteDpnId);
348 String rd = vrfTableKey.getRouteDistinguisher();
350 if (localDpnId != null && localDpnId != BigInteger.ZERO) {
351 // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
352 if (extraRouteOptional.isPresent()) {
353 nexthopManager.deleteLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
355 deleteFibEntryForBgpRoutes(remoteDpnId, vpnId, vrfEntry, rd, tx, subTxns);
359 // below two reads are kept as is, until best way is found to identify dpnID
360 VpnNexthop localNextHopInfo = nexthopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
361 if (extraRouteOptional.isPresent()) {
362 nexthopManager.deleteLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
364 checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx, subTxns);
368 public Consumer<? super VrfEntry> getConsumerForCreatingRemoteFib(
369 final BigInteger dpnId, final long vpnId, final String rd,
370 final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
371 WriteTransaction writeCfgTxn, List<SubTransaction> subTxns) {
372 return vrfEntry -> vrfEntry.nonnullRoutePaths().stream()
373 .filter(routes -> !routes.getNexthopAddress().isEmpty()
374 && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
376 .ifPresent(routes -> {
377 LOG.trace("creating remote FIB entry for prefix {} rd {} on Dpn {}",
378 vrfEntry.getDestPrefix(), rd, dpnId);
379 createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
380 vrfEntry, writeCfgTxn, subTxns);
384 public Consumer<? super VrfEntry> getConsumerForDeletingRemoteFib(
385 final BigInteger dpnId, final long vpnId,
386 final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
387 WriteTransaction writeCfgTxn, List<SubTransaction> subTxns) {
388 return vrfEntry -> vrfEntry.nonnullRoutePaths().stream()
389 .filter(routes -> !routes.getNexthopAddress().isEmpty()
390 && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
392 .ifPresent(routes -> {
393 LOG.trace(" deleting remote FIB entry {}", vrfEntry);
394 deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().key(), vrfEntry,
395 Optional.absent(), writeCfgTxn, subTxns);
400 protected void addTunnelInterfaceActions(NexthopManager.AdjacencyResult adjacencyResult, long vpnId,
401 VrfEntry vrfEntry, List<ActionInfo> actionInfos, String rd) {
402 Class<? extends TunnelTypeBase> tunnelType = VpnExtraRouteHelper
403 .getTunnelType(getNextHopManager().getItmManager(), adjacencyResult.getInterfaceName());
404 if (tunnelType == null) {
405 LOG.debug("Tunnel type not found for vrfEntry {}", vrfEntry);
408 String nextHopIp = adjacencyResult.getNextHopIp();
409 if (tunnelType.equals(TunnelTypeMplsOverGre.class)) {
410 java.util.Optional<Long> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
411 if (!optionalLabel.isPresent()) {
412 LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
415 long label = optionalLabel.get();
416 LOG.debug("addTunnelInterfaceActions: Push label action for prefix {} rd {} l3vni {} nextHop {}",
417 vrfEntry.getDestPrefix(), rd, vrfEntry.getL3vni(), nextHopIp);
418 actionInfos.add(new ActionPushMpls());
419 actionInfos.add(new ActionSetFieldMplsLabel(label));
420 actionInfos.add(new ActionNxLoadInPort(BigInteger.ZERO));
421 } else if (tunnelType.equals(TunnelTypeVxlan.class)) {
422 actionInfos.add(new ActionSetFieldTunnelId(BigInteger.valueOf(vrfEntry.getL3vni())));
423 LOG.debug("addTunnelInterfaceActions: adding set tunnel id action for prefix {} rd {} l3vni {}"
424 + " nextHop {} ", vrfEntry.getDestPrefix(), rd, vrfEntry.getL3vni(), nextHopIp);
425 addRewriteDstMacAction(vpnId, vrfEntry, null /*prefixInfo*/, actionInfos);
430 protected void addRewriteDstMacAction(long vpnId, VrfEntry vrfEntry, @Nullable Prefixes prefixInfo,
431 List<ActionInfo> actionInfos) {
432 if (vrfEntry.getGatewayMacAddress() != null) {
433 actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
434 new MacAddress(vrfEntry.getGatewayMacAddress())));