2 * Copyright © 2017 Ericsson India Global Services Pvt Ltd. and others. All rights reserved.
4 * This program and the accompanying materials are made available under the
5 * terms of the Eclipse Public License v1.0 which accompanies this distribution,
6 * and is available at http://www.eclipse.org/legal/epl-v10.html
8 package org.opendaylight.netvirt.fibmanager;
10 import static java.util.stream.Collectors.toList;
12 import com.google.common.base.Optional;
13 import java.math.BigInteger;
14 import java.util.ArrayList;
15 import java.util.Collection;
16 import java.util.List;
17 import java.util.concurrent.BlockingQueue;
18 import java.util.concurrent.LinkedBlockingQueue;
19 import java.util.function.Consumer;
20 import javax.annotation.PostConstruct;
21 import javax.inject.Inject;
22 import javax.inject.Singleton;
23 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
24 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
25 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
26 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
27 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
28 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
29 import org.opendaylight.genius.mdsalutil.ActionInfo;
30 import org.opendaylight.genius.mdsalutil.InstructionInfo;
31 import org.opendaylight.genius.mdsalutil.NwConstants;
32 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
33 import org.opendaylight.genius.mdsalutil.actions.ActionNxLoadInPort;
34 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
35 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
36 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
37 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldMplsLabel;
38 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
39 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
40 import org.opendaylight.genius.utils.batching.ActionableResource;
41 import org.opendaylight.genius.utils.batching.ActionableResourceImpl;
42 import org.opendaylight.genius.utils.batching.ResourceBatchingManager;
43 import org.opendaylight.genius.utils.batching.ResourceHandler;
44 import org.opendaylight.genius.utils.batching.SubTransaction;
45 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
46 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
47 import org.opendaylight.serviceutils.upgrade.UpgradeState;
48 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
49 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
50 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
51 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
52 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
54 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
55 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
56 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
57 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
58 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
59 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
60 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
61 import org.slf4j.Logger;
62 import org.slf4j.LoggerFactory;
66 public class BgpRouteVrfEntryHandler extends BaseVrfEntryHandler
67 implements ResourceHandler, IVrfEntryHandler {
69 private static final Logger LOG = LoggerFactory.getLogger(BgpRouteVrfEntryHandler.class);
70 private static final int BATCH_INTERVAL = 500;
71 private static final int BATCH_SIZE = 1000;
73 private final DataBroker dataBroker;
74 private final ManagedNewTransactionRunner txRunner;
75 private final BlockingQueue<ActionableResource> vrfEntryBufferQ = new LinkedBlockingQueue<>();
76 private final ResourceBatchingManager resourceBatchingManager;
77 private final NexthopManager nexthopManager;
80 public BgpRouteVrfEntryHandler(final DataBroker dataBroker,
81 final NexthopManager nexthopManager,
82 final FibUtil fibUtil,
83 final UpgradeState upgradeState,
84 final DataTreeEventCallbackRegistrar eventCallbacks) {
85 super(dataBroker, nexthopManager, null, fibUtil, upgradeState, eventCallbacks);
86 this.dataBroker = dataBroker;
87 this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
88 this.nexthopManager = nexthopManager;
90 resourceBatchingManager = ResourceBatchingManager.getInstance();
91 resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY", vrfEntryBufferQ, this);
96 LOG.info("{} start", getClass().getSimpleName());
100 public void close() {
101 LOG.info("{} close", getClass().getSimpleName());
105 public DataBroker getResourceBroker() {
110 public int getBatchSize() {
115 public int getBatchInterval() {
116 return BATCH_INTERVAL;
120 public LogicalDatastoreType getDatastoreType() {
121 return LogicalDatastoreType.CONFIGURATION;
125 public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
126 Object original, Object update, List<SubTransaction> subTxns) {
127 if (original instanceof VrfEntry && update instanceof VrfEntry) {
128 createFibEntries(tx, identifier, (VrfEntry) update, subTxns);
133 public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
134 Object vrfEntry, List<SubTransaction> subTxns) {
135 if (vrfEntry instanceof VrfEntry) {
136 createFibEntries(tx, identifier, (VrfEntry) vrfEntry, subTxns);
141 public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier,
142 Object vrfEntry, List<SubTransaction> subTxns) {
143 if (vrfEntry instanceof VrfEntry) {
144 deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry, subTxns);
149 public void createFlows(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
150 ActionableResource actResource = new ActionableResourceImpl(rd + vrfEntry.getDestPrefix());
151 actResource.setAction(ActionableResource.CREATE);
152 actResource.setInstanceIdentifier(identifier);
153 actResource.setInstance(vrfEntry);
154 vrfEntryBufferQ.add(actResource);
158 public void removeFlows(InstanceIdentifier<VrfEntry> identifier, VrfEntry vrfEntry, String rd) {
159 ActionableResource actResource = new ActionableResourceImpl(rd + vrfEntry.getDestPrefix());
160 actResource.setAction(ActionableResource.DELETE);
161 actResource.setInstanceIdentifier(identifier);
162 actResource.setInstance(vrfEntry);
163 vrfEntryBufferQ.add(actResource);
167 public void updateFlows(InstanceIdentifier<VrfEntry> identifier, VrfEntry original, VrfEntry update, String rd) {
168 ActionableResource actResource = new ActionableResourceImpl(rd + update.getDestPrefix());
169 actResource.setAction(ActionableResource.UPDATE);
170 actResource.setInstanceIdentifier(identifier);
171 actResource.setInstance(update);
172 actResource.setOldInstance(original);
173 vrfEntryBufferQ.add(actResource);
177 Please note that the following createFibEntries will be invoked only for BGP Imported Routes.
178 The invocation of the following method is via create() callback from the MDSAL Batching Infrastructure
179 provided by ResourceBatchingManager
181 private void createFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> vrfEntryIid,
182 final VrfEntry vrfEntry, List<SubTransaction> subTxns) {
183 final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
184 LOG.trace("Creating fib entry for vrfEntry with destPrefix{}, rd {}",
185 vrfEntry.getDestPrefix(), vrfTableKey.getRouteDistinguisher());
186 final VpnInstanceOpDataEntry vpnInstance =
187 getFibUtil().getVpnInstance(vrfTableKey.getRouteDistinguisher());
188 if (vpnInstance == null || vpnInstance.getVpnId() == null) {
189 LOG.error("Vpn Instance not availabe {}", vrfTableKey.getRouteDistinguisher());
192 final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
193 if (vpnToDpnList != null) {
194 for (VpnToDpnList vpnDpn : vpnToDpnList) {
195 LOG.trace("Dpnstate is {} for dpn {} in vpn {}", vpnDpn.getDpnState(), vpnDpn.getDpnId(),
196 vpnInstance.getVpnId());
197 if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
198 createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(),
199 vrfEntry, writeTx, subTxns);
203 LOG.trace("Created fib entry for vrfEntry with destPrefix{}, rd {}",
204 vrfEntry.getDestPrefix(), vrfTableKey.getRouteDistinguisher());
208 Please note that the following deleteFibEntries will be invoked only for BGP Imported Routes.
209 The invocation of the following method is via delete() callback from the MDSAL Batching Infrastructure
210 provided by ResourceBatchingManager
212 private void deleteFibEntries(WriteTransaction writeTx, final InstanceIdentifier<VrfEntry> identifier,
213 final VrfEntry vrfEntry, List<SubTransaction> subTxns) {
214 final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
215 String rd = vrfTableKey.getRouteDistinguisher();
216 final VpnInstanceOpDataEntry vpnInstance =
217 getFibUtil().getVpnInstance(vrfTableKey.getRouteDistinguisher());
218 if (vpnInstance == null) {
219 LOG.debug("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
222 String vpnName = getFibUtil().getVpnNameFromId(vpnInstance.getVpnId());
223 final Collection<VpnToDpnList> vpnToDpnList = vpnInstance.getVpnToDpnList();
224 if (vpnToDpnList != null) {
225 List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker,
226 vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
227 Optional<Routes> extraRouteOptional;
228 //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
229 if (usedRds != null && !usedRds.isEmpty()) {
230 if (usedRds.size() > 1) {
231 LOG.error("The extra route prefix is still present in some DPNs");
234 extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName,
235 usedRds.get(0), vrfEntry.getDestPrefix());
238 extraRouteOptional = Optional.absent();
240 for (VpnToDpnList curDpn : vpnToDpnList) {
241 if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
242 deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(),
243 vrfTableKey, vrfEntry, extraRouteOptional, writeTx, subTxns);
249 public void programRemoteFibForBgpRoutes(final BigInteger remoteDpnId,
251 final VrfEntry vrfEntry,
254 List<NexthopManager.AdjacencyResult> adjacencyResults,
255 List<SubTransaction> subTxns) {
256 if (vrfEntry.getRoutePaths().size() > 2) {
257 LOG.error("DC-GW can advertise only 2 bestPaths for prefix {}", vrfEntry.getDestPrefix());
260 LOG.trace("Start programming remote fib for destPrefix {}, vpnId {}, dpnId {}",
261 vrfEntry.getDestPrefix(), vpnId, remoteDpnId);
262 if (adjacencyResults.size() == 1) {
263 programRemoteFib(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, subTxns);
266 // ECMP Use case, point to LB group. Move the mpls label accordingly.
267 List<String> tunnelList =
268 adjacencyResults.stream()
269 .map(NexthopManager.AdjacencyResult::getNextHopIp)
270 .sorted().collect(toList());
271 String lbGroupKey = FibUtil.getGreLbGroupKey(tunnelList);
272 long groupId = nexthopManager.createNextHopPointer(lbGroupKey);
274 List<ActionInfo> actionInfos = new ArrayList<>();
275 for (NexthopManager.AdjacencyResult adjResult : adjacencyResults) {
276 String nextHopIp = adjResult.getNextHopIp();
277 java.util.Optional<Long> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
278 if (!optionalLabel.isPresent()) {
279 LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
282 long label = optionalLabel.get();
284 actionInfos.add(new ActionRegLoad(index, FibConstants.NXM_REG_MAPPING.get(index++), 0,
287 List<InstructionInfo> instructions = new ArrayList<>();
288 actionInfos.add(new ActionGroup(index, groupId));
289 instructions.add(new InstructionApplyActions(actionInfos));
290 makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx, subTxns);
291 LOG.trace("End programming remote fib for destPrefix {}, vpnId {}, dpnId {}",
292 vrfEntry.getDestPrefix(), vpnId, remoteDpnId);
295 public void createRemoteFibEntry(final BigInteger remoteDpnId,
298 final VrfEntry vrfEntry,
300 List<SubTransaction> subTxns) {
302 ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
303 newTx -> createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx, subTxns)), LOG,
304 "Error creating remote FIB entry");
308 LOG.debug("createRemoteFibEntry: adding route {} for rd {} on remoteDpnId {}",
309 vrfEntry.getDestPrefix(), rd, remoteDpnId);
311 List<NexthopManager.AdjacencyResult> adjacencyResults =
312 resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
313 if (adjacencyResults.isEmpty()) {
314 LOG.error("Could not get interface for route-paths: {} in vpn {}", vrfEntry.getRoutePaths(), rd);
315 LOG.warn("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd);
319 programRemoteFibForBgpRoutes(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, subTxns);
321 LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId);
324 private void deleteFibEntryForBgpRoutes(BigInteger remoteDpnId, long vpnId, VrfEntry vrfEntry,
325 String rd, WriteTransaction tx, List<SubTransaction> subTxns) {
326 // When the tunnel is removed the fib entries should be reprogrammed/deleted depending on
327 // the adjacencyResults.
328 List<NexthopManager.AdjacencyResult> adjacencyResults = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd);
329 if (!adjacencyResults.isEmpty()) {
330 programRemoteFibForBgpRoutes(remoteDpnId, vpnId, vrfEntry, tx, rd, adjacencyResults, subTxns);
334 public void deleteRemoteRoute(final BigInteger localDpnId, final BigInteger remoteDpnId,
335 final long vpnId, final VrfTablesKey vrfTableKey,
336 final VrfEntry vrfEntry, Optional<Routes> extraRouteOptional,
337 WriteTransaction tx, List<SubTransaction> subTxns) {
339 ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
340 newTx -> deleteRemoteRoute(localDpnId, remoteDpnId, vpnId, vrfTableKey, vrfEntry,
341 extraRouteOptional, newTx)), LOG, "Error deleting remote route");
345 LOG.debug("deleting remote route: prefix={}, vpnId={} localDpnId {} remoteDpnId {}",
346 vrfEntry.getDestPrefix(), vpnId, localDpnId, remoteDpnId);
347 String rd = vrfTableKey.getRouteDistinguisher();
349 if (localDpnId != null && localDpnId != BigInteger.ZERO) {
350 // localDpnId is not known when clean up happens for last vm for a vpn on a dpn
351 if (extraRouteOptional.isPresent()) {
352 nexthopManager.deleteLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
354 deleteFibEntryForBgpRoutes(remoteDpnId, vpnId, vrfEntry, rd, tx, subTxns);
358 // below two reads are kept as is, until best way is found to identify dpnID
359 VpnNexthop localNextHopInfo = nexthopManager.getVpnNexthop(vpnId, vrfEntry.getDestPrefix());
360 if (extraRouteOptional.isPresent()) {
361 nexthopManager.deleteLoadBalancingNextHop(vpnId, remoteDpnId, vrfEntry.getDestPrefix());
363 checkDpnDeleteFibEntry(localNextHopInfo, remoteDpnId, vpnId, vrfEntry, rd, tx, subTxns);
367 public Consumer<? super VrfEntry> getConsumerForCreatingRemoteFib(
368 final BigInteger dpnId, final long vpnId, final String rd,
369 final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
370 WriteTransaction writeCfgTxn, List<SubTransaction> subTxns) {
371 return vrfEntry -> vrfEntry.getRoutePaths().stream()
372 .filter(routes -> !routes.getNexthopAddress().isEmpty()
373 && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
375 .ifPresent(routes -> {
376 LOG.trace("creating remote FIB entry for prefix {} rd {} on Dpn {}",
377 vrfEntry.getDestPrefix(), rd, dpnId);
378 createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getRouteDistinguisher(),
379 vrfEntry, writeCfgTxn, subTxns);
383 public Consumer<? super VrfEntry> getConsumerForDeletingRemoteFib(
384 final BigInteger dpnId, final long vpnId,
385 final String remoteNextHopIp, final Optional<VrfTables> vrfTable,
386 WriteTransaction writeCfgTxn, List<SubTransaction> subTxns) {
387 return vrfEntry -> vrfEntry.getRoutePaths().stream()
388 .filter(routes -> !routes.getNexthopAddress().isEmpty()
389 && remoteNextHopIp.trim().equals(routes.getNexthopAddress().trim()))
391 .ifPresent(routes -> {
392 LOG.trace(" deleting remote FIB entry {}", vrfEntry);
393 deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().key(), vrfEntry,
394 Optional.absent(), writeCfgTxn, subTxns);
399 protected void addTunnelInterfaceActions(NexthopManager.AdjacencyResult adjacencyResult, long vpnId,
400 VrfEntry vrfEntry, List<ActionInfo> actionInfos, String rd) {
401 Class<? extends TunnelTypeBase> tunnelType = VpnExtraRouteHelper
402 .getTunnelType(getNextHopManager().getItmManager(), adjacencyResult.getInterfaceName());
403 if (tunnelType == null) {
404 LOG.debug("Tunnel type not found for vrfEntry {}", vrfEntry);
407 String nextHopIp = adjacencyResult.getNextHopIp();
408 if (tunnelType.equals(TunnelTypeMplsOverGre.class)) {
409 java.util.Optional<Long> optionalLabel = FibUtil.getLabelForNextHop(vrfEntry, nextHopIp);
410 if (!optionalLabel.isPresent()) {
411 LOG.warn("NextHopIp {} not found in vrfEntry {}", nextHopIp, vrfEntry);
414 long label = optionalLabel.get();
415 LOG.debug("addTunnelInterfaceActions: Push label action for prefix {} rd {} l3vni {} nextHop {}",
416 vrfEntry.getDestPrefix(), rd, vrfEntry.getL3vni(), nextHopIp);
417 actionInfos.add(new ActionPushMpls());
418 actionInfos.add(new ActionSetFieldMplsLabel(label));
419 actionInfos.add(new ActionNxLoadInPort(BigInteger.ZERO));
420 } else if (tunnelType.equals(TunnelTypeVxlan.class)) {
421 actionInfos.add(new ActionSetFieldTunnelId(BigInteger.valueOf(vrfEntry.getL3vni())));
422 LOG.debug("addTunnelInterfaceActions: adding set tunnel id action for prefix {} rd {} l3vni {}"
423 + " nextHop {} ", vrfEntry.getDestPrefix(), rd, vrfEntry.getL3vni(), nextHopIp);
424 addRewriteDstMacAction(vpnId, vrfEntry, null /*prefixInfo*/, actionInfos);
429 protected void addRewriteDstMacAction(long vpnId, VrfEntry vrfEntry, Prefixes prefixInfo,
430 List<ActionInfo> actionInfos) {
431 if (vrfEntry.getGatewayMacAddress() != null) {
432 actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
433 new MacAddress(vrfEntry.getGatewayMacAddress())));