2 * Copyright © 2017 Ericsson India Global Services Pvt Ltd. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.netvirt.fibmanager;
10 import static java.util.stream.Collectors.toList;
12 import com.google.common.base.Optional;
13 import java.math.BigInteger;
14 import java.util.ArrayList;
15 import java.util.Collection;
16 import java.util.List;
17 import java.util.concurrent.BlockingQueue;
18 import java.util.concurrent.LinkedBlockingQueue;
19 import java.util.function.Consumer;
20 import javax.annotation.PostConstruct;
21 import javax.inject.Inject;
22 import javax.inject.Singleton;
23 import org.eclipse.jdt.annotation.Nullable;
24 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
25 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
26 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
27 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
28 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
29 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
30 import org.opendaylight.genius.mdsalutil.ActionInfo;
31 import org.opendaylight.genius.mdsalutil.InstructionInfo;
32 import org.opendaylight.genius.mdsalutil.NwConstants;
33 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
34 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
35 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
36 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
37 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
38 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldMplsLabel;
39 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
40 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
41 import org.opendaylight.genius.utils.batching.ActionableResource;
42 import org.opendaylight.genius.utils.batching.ActionableResourceImpl;
43 import org.opendaylight.genius.utils.batching.ResourceBatchingManager;
44 import org.opendaylight.genius.utils.batching.ResourceHandler;
45 import org.opendaylight.genius.utils.batching.SubTransaction;
46 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
47 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
48 import org.opendaylight.serviceutils.upgrade.UpgradeState;
49 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
50 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
51 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
52 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
54 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
55 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
56 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
57 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
58 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
59 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
60 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
61 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
62 import org.slf4j.Logger;
63 import org.slf4j.LoggerFactory;
67 public class BgpRouteVrfEntryHandler extends BaseVrfEntryHandler implements ResourceHandler {
69 private static final Logger LOG = LoggerFactory.getLogger(BgpRouteVrfEntryHandler.class);
70 private static final int BATCH_INTERVAL = 500;
71 private static final int BATCH_SIZE = 1000;
73 private final DataBroker dataBroker;
74 private final ManagedNewTransactionRunner txRunner;
75 private final BlockingQueue<ActionableResource> vrfEntryBufferQ = new LinkedBlockingQueue<>();
76 private final ResourceBatchingManager resourceBatchingManager;
77 private final NexthopManager nexthopManager;
80 public BgpRouteVrfEntryHandler(final DataBroker dataBroker,
81 final NexthopManager nexthopManager,
82 final FibUtil fibUtil,
83 final UpgradeState upgradeState,
84 final DataTreeEventCallbackRegistrar eventCallbacks) {
85 super(dataBroker, nexthopManager, null, fibUtil, upgradeState, eventCallbacks);
86 this.dataBroker = dataBroker;
87 this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
88 this.nexthopManager = nexthopManager;
90 resourceBatchingManager = ResourceBatchingManager.getInstance();
91 resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY", vrfEntryBufferQ, this);
96 LOG.info("{} start", getClass().getSimpleName());
100 public void close() {
101 LOG.info("{} close", getClass().getSimpleName());
105 public DataBroker getResourceBroker() {
110 public int getBatchSize() {
115 public int getBatchInterval() {
116 return BATCH_INTERVAL;
120 public LogicalDatastoreType getDatastoreType() {
121 return LogicalDatastoreType.CONFIGURATION;
125 public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
126 Object original, Object update, List<SubTransaction> subTxns) {
127 if (original instanceof VrfEntry && update instanceof VrfEntry) {
128 createFibEntries(tx, identifier, (VrfEntry) update, subTxns);
133 public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
134 Object vrfEntry, List<SubTransaction> subTxns) {
135 if (vrfEntry instanceof VrfEntry) {
136 createFibEntries(tx, identifier, (VrfEntry) vrfEntry, subTxns);
141 public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
142 Object vrfEntry, List<SubTransaction> subTxns) {
143 if (vrfEntry instanceof VrfEntry) {
144 deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry, subTxns);
148 void createFlows(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
149 ActionableResource actResource = new ActionableResourceImpl(rd + vrfEntry.getDestPrefix());
150 actResource.setAction(ActionableResource.CREATE);
151 actResource.setInstanceIdentifier(identifier);
152 actResource.setInstance(vrfEntry);
153 vrfEntryBufferQ.add(actResource);
156 void removeFlows(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
157 ActionableResource actResource = new ActionableResourceImpl(rd + vrfEntry.getDestPrefix());
158 actResource.setAction(ActionableResource.DELETE);
159 actResource.setInstanceIdentifier(identifier);
160 actResource.setInstance(vrfEntry);
161 vrfEntryBufferQ.add(actResource);
164 void updateFlows(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update, String rd) {
165 ActionableResource actResource = new ActionableResourceImpl(rd + update.getDestPrefix());
166 actResource.setAction(ActionableResource.UPDATE);
167 actResource.setInstanceIdentifier(identifier);
168 actResource.setInstance(update);
169 actResource.setOldInstance(original);
170 vrfEntryBufferQ.add(actResource);
174 Please note that the following createFibEntries will be invoked only for BGP Imported Routes.
175 The invocation of the following method is via create() callback from the MDSAL Batching Infrastructure
176 provided by ResourceBatchingManager
178 private void createFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> vrfEntryIid,
179 final VrfEntry vrfEntry, List<SubTransaction> subTxns) {
180 final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
181 LOG.trace("Creating fib entry for vrfEntry with destPrefix{}, rd {}",
182 vrfEntry.getDestPrefix(), vrfTableKey.getRouteDistinguisher());
183 final VpnInstanceOpDataEntry vpnInstance =
184 getFibUtil().getVpnInstance(vrfTableKey.getRouteDistinguisher());
185 if (vpnInstance == null || vpnInstance.getVpnId() == null) {
186 LOG.error("Vpn Instance not availabe {}", vrfTableKey.getRouteDistinguisher());
189 final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
190 if (vpnToDpnList != null) {
191 for (VpnToDpnList vpnDpn : vpnToDpnList) {
192 LOG.trace("Dpnstate is {} for dpn {} in vpn {}", vpnDpn.getDpnState(), vpnDpn.getDpnId(),
193 vpnInstance.getVpnId());
194 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
195 createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(),
196 vrfEntry, writeTx, subTxns);
200 LOG.trace("Created fib entry for vrfEntry with destPrefix{}, rd {}",
201 vrfEntry.getDestPrefix(), vrfTableKey.getRouteDistinguisher());
205 Please note that the following deleteFibEntries will be invoked only for BGP Imported Routes.
206 The invocation of the following method is via delete() callback from the MDSAL Batching Infrastructure
207 provided by ResourceBatchingManager
209 private void deleteFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> identifier,
210 final VrfEntry vrfEntry, List<SubTransaction> subTxns) {
211 final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
212 String rd = vrfTableKey.getRouteDistinguisher();
213 final VpnInstanceOpDataEntry vpnInstance =
214 getFibUtil().getVpnInstance(vrfTableKey.getRouteDistinguisher());
215 if (vpnInstance == null) {
216 LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
219 String vpnName = getFibUtil().getVpnNameFromId(vpnInstance.getVpnId());
220 final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
221 if (vpnToDpnList != null) {
222 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
223 vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
224 Optional<Routes> extraRouteOptional;
225 //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
226 if (usedRds != null && !usedRds.isEmpty()) {
227 if (usedRds.size() > 1) {
228 LOG.error("The extra route prefix is still present in some DPNs");
231 extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
232 usedRds.get(0), vrfEntry.getDestPrefix());
235 extraRouteOptional = Optional.absent();
237 for (VpnToDpnList curDpn : vpnToDpnList) {
238 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
239 deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(),
240 vrfTableKey, vrfEntry, extraRouteOptional, writeTx, subTxns);
246 public void programRemoteFibForBgpRoutes(final BigInteger remoteDpnId,
248 final VrfEntry vrfEntry,
251 List<NexthopManager.AdjacencyResult> adjacencyResults,
252 List<SubTransaction> subTxns) {
253 if (vrfEntry.nonnullRoutePaths().size() > 2) {
254 LOG.error("DC-GW can advertise only 2 bestPaths for prefix {}", vrfEntry.getDestPrefix());
257 LOG.trace("Start programming remote fib for destPrefix {}, vpnId {}, dpnId {}",
258 vrfEntry.getDestPrefix(), vpnId, remoteDpnId);
259 if (adjacencyResults.size() == 1) {
260 programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, subTxns);
263 // ECMP Use case, point to LB group. Move the mpls label accordingly.
264 List<String> tunnelList =
265 adjacencyResults.stream()
266 .map(NexthopManager.AdjacencyResult::getNextHopIp)
267 .sorted().collect(toList());
268 String lbGroupKey = FibUtil.getGreLbGroupKey(tunnelList);
269 long groupId = nexthopManager.createNextHopPointer(lbGroupKey);
271 List<ActionInfo> actionInfos = new ArrayList<>();
272 for (NexthopManager.AdjacencyResult adjResult : adjacencyResults) {
273 String nextHopIp = adjResult.getNextHopIp();
274 java.util.Optional<Long> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
275 if (!optionalLabel.isPresent()) {
276 LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
279 long label = optionalLabel.get();
281 actionInfos.add(new ActionRegLoad(index, FibConstants.NXM_REG_MAPPING.get(index++), 0,
284 List<InstructionInfo> instructions = new ArrayList<>();
285 actionInfos.add(new ActionGroup(index, groupId));
286 instructions.add(new InstructionApplyActions(actionInfos));
287 makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx, subTxns);
288 LOG.trace("End programming remote fib for destPrefix {}, vpnId {}, dpnId {}",
289 vrfEntry.getDestPrefix(), vpnId, remoteDpnId);
292 public void createRemoteFibEntry(final BigInteger remoteDpnId,
295 final VrfEntry vrfEntry,
297 List<SubTransaction> subTxns) {
299 ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
300 newTx -> createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx, subTxns)), LOG,
301 "Error creating remote FIB entry");
305 LOG.debug("createRemoteFibEntry: adding route {} for rd {} on remoteDpnId {}",
306 vrfEntry.getDestPrefix(), rd, remoteDpnId);
308 List<NexthopManager.AdjacencyResult> adjacencyResults =
309 resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
310 if (adjacencyResults.isEmpty()) {
311 LOG.error("Could not get interface for route-paths: {} in vpn {}", vrfEntry.getRoutePaths(), rd);
312 LOG.warn("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
316 programRemoteFibForBgpRoutes(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, subTxns);
318 LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
321 private void deleteFibEntryForBgpRoutes(BigInteger remoteDpnId, long vpnId, VrfEntry vrfEntry,
322 String rd, WriteTransaction tx, List<SubTransaction> subTxns) {
323 // When the tunnel is removed the fib entries should be reprogrammed/deleted depending on
324 // the adjacencyResults.
325 List<NexthopManager.AdjacencyResult> adjacencyResults = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
326 if (!adjacencyResults.isEmpty()) {
327 programRemoteFibForBgpRoutes(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, subTxns);
331 public void deleteRemoteRoute(@Nullable final BigInteger localDpnId, final BigInteger remoteDpnId,
332 final long vpnId, final VrfTablesKey vrfTableKey,
333 final VrfEntry vrfEntry, Optional<Routes> extraRouteOptional,
334 @Nullable WriteTransaction tx, List<SubTransaction> subTxns) {
336 ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
337 newTx -> deleteRemoteRoute(localDpnId, remoteDpnId, vpnId, vrfTableKey, vrfEntry,
338 extraRouteOptional, newTx)), LOG, "Error deleting remote route");
342 LOG.debug("deleting remote route: prefix={}, vpnId={} localDpnId {} remoteDpnId {}",
343 vrfEntry.getDestPrefix(), vpnId, localDpnId, remoteDpnId);
344 String rd = vrfTableKey.getRouteDistinguisher();
346 if (localDpnId != null && !BigInteger.ZERO.equals(localDpnId)) {
347 // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
348 if (extraRouteOptional.isPresent()) {
349 nexthopManager.deleteLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
351 deleteFibEntryForBgpRoutes(remoteDpnId, vpnId, vrfEntry, rd, tx, subTxns);
355 // below two reads are kept as is, until best way is found to identify dpnID
356 VpnNexthop localNextHopInfo = nexthopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
357 if (extraRouteOptional.isPresent()) {
358 nexthopManager.deleteLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
360 checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx, subTxns);
364 public Consumer<? super VrfEntry> getConsumerForCreatingRemoteFib(
365 final BigInteger dpnId, final long vpnId, final String rd,
366 final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
367 WriteTransaction writeCfgTxn, List<SubTransaction> subTxns) {
368 return vrfEntry -> vrfEntry.nonnullRoutePaths().stream()
369 .filter(routes -> !routes.getNexthopAddress().isEmpty()
370 && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
372 .ifPresent(routes -> {
373 LOG.trace("creating remote FIB entry for prefix {} rd {} on Dpn {}",
374 vrfEntry.getDestPrefix(), rd, dpnId);
375 createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
376 vrfEntry, writeCfgTxn, subTxns);
380 public Consumer<? super VrfEntry> getConsumerForDeletingRemoteFib(
381 final BigInteger dpnId, final long vpnId,
382 final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
383 WriteTransaction writeCfgTxn, List<SubTransaction> subTxns) {
384 return vrfEntry -> vrfEntry.nonnullRoutePaths().stream()
385 .filter(routes -> !routes.getNexthopAddress().isEmpty()
386 && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
388 .ifPresent(routes -> {
389 LOG.trace(" deleting remote FIB entry {}", vrfEntry);
390 deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().key(), vrfEntry,
391 Optional.absent(), writeCfgTxn, subTxns);
396 protected void addTunnelInterfaceActions(NexthopManager.AdjacencyResult adjacencyResult, long vpnId,
397 VrfEntry vrfEntry, List<ActionInfo> actionInfos, String rd) {
398 Class<? extends TunnelTypeBase> tunnelType = VpnExtraRouteHelper
399 .getTunnelType(getNextHopManager().getItmManager(), adjacencyResult.getInterfaceName());
400 if (tunnelType == null) {
401 LOG.debug("Tunnel type not found for vrfEntry {}", vrfEntry);
404 String nextHopIp = adjacencyResult.getNextHopIp();
405 if (tunnelType.equals(TunnelTypeMplsOverGre.class)) {
406 java.util.Optional<Long> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
407 if (!optionalLabel.isPresent()) {
408 LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
411 long label = optionalLabel.get();
412 LOG.debug("addTunnelInterfaceActions: Push label action for prefix {} rd {} l3vni {} nextHop {}",
413 vrfEntry.getDestPrefix(), rd, vrfEntry.getL3vni(), nextHopIp);
414 actionInfos.add(new ActionPushMpls());
415 actionInfos.add(new ActionSetFieldMplsLabel(label));
416 actionInfos.add(new ActionNxLoadInPort(BigInteger.ZERO));
417 } else if (tunnelType.equals(TunnelTypeVxlan.class)) {
418 actionInfos.add(new ActionSetFieldTunnelId(BigInteger.valueOf(vrfEntry.getL3vni())));
419 LOG.debug("addTunnelInterfaceActions: adding set tunnel id action for prefix {} rd {} l3vni {}"
420 + " nextHop {} ", vrfEntry.getDestPrefix(), rd, vrfEntry.getL3vni(), nextHopIp);
421 addRewriteDstMacAction(vpnId, vrfEntry, null /*prefixInfo*/, actionInfos);
426 protected void addRewriteDstMacAction(long vpnId, VrfEntry vrfEntry, @Nullable Prefixes prefixInfo,
427 List<ActionInfo> actionInfos) {
428 if (vrfEntry.getGatewayMacAddress() != null) {
429 actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
430 new MacAddress(vrfEntry.getGatewayMacAddress())));