X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=vpnservice%2Ffibmanager%2Ffibmanager-impl%2Fsrc%2Fmain%2Fjava%2Forg%2Fopendaylight%2Fnetvirt%2Ffibmanager%2FVrfEntryListener.java;h=362100b612eb1b92f5619de8592a1cae9352d8bb;hb=6f6c4ace0ff338698cc5d8261c8f18015701b0da;hp=d38fc2a9f29d292269221498d8dd78b984333833;hpb=03b266a67d94f5d2eec39946a793ac7227c29598;p=netvirt.git diff --git a/vpnservice/fibmanager/fibmanager-impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java b/vpnservice/fibmanager/fibmanager-impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java old mode 100644 new mode 100755 index d38fc2a9f2..362100b612 --- a/vpnservice/fibmanager/fibmanager-impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java +++ b/vpnservice/fibmanager/fibmanager-impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java @@ -9,11 +9,11 @@ package org.opendaylight.netvirt.fibmanager; import com.google.common.base.Optional; import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.CheckedFuture; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import java.math.BigInteger; +import java.net.Inet4Address; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; @@ -26,14 +26,11 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; import org.opendaylight.controller.md.sal.binding.api.DataBroker; -import org.opendaylight.controller.md.sal.binding.api.DataChangeListener; import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction; import org.opendaylight.controller.md.sal.binding.api.WriteTransaction; -import org.opendaylight.controller.md.sal.common.api.data.AsyncDataBroker.DataChangeScope; import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType; -import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException; +import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase; import org.opendaylight.genius.datastoreutils.DataStoreJobCoordinator; -import org.opendaylight.genius.mdsalutil.AbstractDataChangeListener; import org.opendaylight.genius.mdsalutil.ActionInfo; import org.opendaylight.genius.mdsalutil.ActionType; import org.opendaylight.genius.mdsalutil.FlowEntity; @@ -45,14 +42,23 @@ import org.opendaylight.genius.mdsalutil.MatchInfo; import org.opendaylight.genius.mdsalutil.MetaDataUtil; import org.opendaylight.genius.mdsalutil.NwConstants; import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager; +import org.opendaylight.genius.mdsalutil.packet.IPProtocols; import org.opendaylight.genius.utils.ServiceIndex; import org.opendaylight.genius.utils.batching.ActionableResource; import org.opendaylight.genius.utils.batching.ActionableResourceImpl; import org.opendaylight.genius.utils.batching.ResourceBatchingManager; import org.opendaylight.genius.utils.batching.ResourceHandler; +import org.opendaylight.genius.utils.batching.SubTransaction; +import org.opendaylight.genius.utils.batching.SubTransactionImpl; +import org.opendaylight.netvirt.elanmanager.api.IElanService; +import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult; import org.opendaylight.netvirt.fibmanager.api.RouteOrigin; import org.opendaylight.netvirt.vpnmanager.api.IVpnManager; +import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache; +import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite; import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress; +import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev140508.Tunnel; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.Table; @@ -75,6 +81,7 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.N import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.NodeKey; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey; @@ -106,9 +113,9 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.Extraroute; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroute.vpn.ExtrarouteKey; import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState; +import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State; import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.links.InterVpnLink; import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.overlay.rev150105.TunnelTypeVxlan; -import org.opendaylight.yangtools.concepts.ListenerRegistration; import org.opendaylight.yangtools.yang.binding.DataObject; import org.opendaylight.yangtools.yang.binding.InstanceIdentifier; import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder; @@ -116,39 +123,40 @@ import org.opendaylight.yangtools.yang.common.RpcResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class VrfEntryListener extends AbstractDataChangeListener implements AutoCloseable, ResourceHandler { +public class VrfEntryListener extends AsyncDataTreeChangeListenerBase implements AutoCloseable, ResourceHandler { private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class); private static final String FLOWID_PREFIX = "L3."; - private ListenerRegistration listenerRegistration; private final DataBroker dataBroker; private final IMdsalApiManager mdsalManager; private IVpnManager vpnmanager; - private NexthopManager nextHopManager; + private final NexthopManager nextHopManager; private ItmRpcService itmManager; - private OdlInterfaceRpcService interfaceManager; - private IdManagerService idManager; + private final OdlInterfaceRpcService interfaceManager; + private final IdManagerService idManager; private static final BigInteger COOKIE_VM_FIB_TABLE = new BigInteger("8000003", 16); private static final int DEFAULT_FIB_FLOW_PRIORITY = 10; private static final int LFIB_INTERVPN_PRIORITY = 1; private static final BigInteger METADATA_MASK_CLEAR = new BigInteger("000000FFFFFFFFFF", 16); private static final BigInteger CLEAR_METADATA = BigInteger.valueOf(0); public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16); + List transactionObjects; private static final int PERIODICITY = 500; private static Integer batchSize; private static Integer batchInterval; private static final int BATCH_SIZE = 1000; private static BlockingQueue vrfEntryBufferQ = new LinkedBlockingQueue<>(); - private ResourceBatchingManager resourceBatchingManager; + private final ResourceBatchingManager resourceBatchingManager; public VrfEntryListener(final DataBroker dataBroker, final IMdsalApiManager mdsalApiManager, final NexthopManager nexthopManager, final OdlInterfaceRpcService interfaceManager, final IdManagerService idManager) { - super(VrfEntry.class); + super(VrfEntry.class, VrfEntryListener.class); this.dataBroker = dataBroker; this.mdsalManager = mdsalApiManager; this.nextHopManager = nexthopManager; this.interfaceManager = interfaceManager; this.idManager = idManager; + batchSize = Integer.getInteger("batch.size"); if (batchSize == null) { batchSize = BATCH_SIZE; @@ -159,25 +167,20 @@ public class VrfEntryListener extends AbstractDataChangeListener imple } resourceBatchingManager = ResourceBatchingManager.getInstance(); resourceBatchingManager.registerBatchableResource("FIB-VRFENTRY",vrfEntryBufferQ, this); + transactionObjects = new ArrayList<>(); } public void start() { LOG.info("{} start", getClass().getSimpleName()); - listenerRegistration = dataBroker.registerDataChangeListener(LogicalDatastoreType.CONFIGURATION, - getWildCardPath(), this, DataChangeScope.SUBTREE); + registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker); } - private InstanceIdentifier getWildCardPath() { - return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class); - } + @Override + protected VrfEntryListener getDataTreeChangeListener() { return VrfEntryListener.this; } @Override - public void close() throws Exception { - if (listenerRegistration != null) { - listenerRegistration.close(); - listenerRegistration = null; - } - LOG.info("{} close", getClass().getSimpleName()); + protected InstanceIdentifier getWildCardPath() { + return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class); } @Override @@ -246,14 +249,16 @@ public class VrfEntryListener extends AbstractDataChangeListener imple } @Override - public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object vrfEntry) { + public void create(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object vrfEntry, List transactionObjects) { + this.transactionObjects = transactionObjects; if (vrfEntry instanceof VrfEntry) { createFibEntries(tx, identifier, (VrfEntry)vrfEntry); } } @Override - public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object vrfEntry) { + public void delete(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object vrfEntry, List transactionObjects) { + this.transactionObjects = transactionObjects; if (vrfEntry instanceof VrfEntry) { deleteFibEntries(tx, identifier, (VrfEntry) vrfEntry); } @@ -261,7 +266,8 @@ public class VrfEntryListener extends AbstractDataChangeListener imple @Override public void update(WriteTransaction tx, LogicalDatastoreType datastoreType, InstanceIdentifier identifier, Object original, - Object update) { + Object update, List transactionObjects) { + this.transactionObjects = transactionObjects; if ((original instanceof VrfEntry) && (update instanceof VrfEntry)) { createFibEntries(tx, identifier, (VrfEntry)update); } @@ -317,6 +323,10 @@ public class VrfEntryListener extends AbstractDataChangeListener imple } return; } + // ping responder for router interfaces + if (installRouterFibEntries(vrfEntry, vpnToDpnList, vpnId, NwConstants.ADD_FLOW)) { + return; + } if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.INTERVPN) { // When it is a leaked route, the LFIB and FIB goes a bit different. @@ -349,14 +359,17 @@ public class VrfEntryListener extends AbstractDataChangeListener imple Optional optVpnUuid = FibUtil.getVpnNameFromRd(dataBroker, rd); if ( optVpnUuid.isPresent() ) { - Optional interVpnLink = FibUtil.getInterVpnLinkByVpnUuid(dataBroker, optVpnUuid.get()); - if ( interVpnLink.isPresent() ) { + Optional optInterVpnLink = InterVpnLinkCache.getInterVpnLinkByVpnId(optVpnUuid.get()); + LOG.debug("InterVpnLink {} found in Cache: {}", optVpnUuid.get(), optInterVpnLink.isPresent()); + if ( optInterVpnLink.isPresent() ) { + InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get(); String vpnUuid = optVpnUuid.get(); String routeNexthop = vrfEntry.getNextHopAddressList().get(0); - if ( isNexthopTheOtherVpnLinkEndpoint(routeNexthop, vpnUuid, interVpnLink.get()) ) { + if ( interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid) ) { // This is an static route that points to the other endpoint of an InterVpnLink // In that case, we should add another entry in FIB table pointing to LPortDispatcher table. - installRouteInInterVpnLink(interVpnLink.get(), vpnUuid, vrfEntry, vpnId); + installRouteInInterVpnLink(interVpnLink, vpnUuid, vrfEntry, vpnId); + installInterVpnRouteInLFib(rd, vrfEntry); } } } @@ -386,20 +399,6 @@ public class VrfEntryListener extends AbstractDataChangeListener imple } } - /* - * Returns true if the specified nexthop is the other endpoint in an - * InterVpnLink, regarding one of the VPN's point of view. - */ - private boolean isNexthopTheOtherVpnLinkEndpoint(String nexthop, String thisVpnUuid, InterVpnLink interVpnLink) { - return - interVpnLink != null - && ( (interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(thisVpnUuid) - && interVpnLink.getSecondEndpoint().getIpAddress().getValue().equals(nexthop)) - || (interVpnLink.getSecondEndpoint().getVpnUuid().getValue().equals(thisVpnUuid ) - && interVpnLink.getFirstEndpoint().getIpAddress().getValue().equals(nexthop)) ); - } - - // FIXME: Refactoring needed here. // This kind of logic must be taken to an 'upper' layer like BgpManager or VpnManager private void leakRouteIfNeeded(final InstanceIdentifier vrfEntryIid, final VrfEntry vrfEntry, @@ -416,7 +415,7 @@ public class VrfEntryListener extends AbstractDataChangeListener imple } } else { Preconditions.checkNotNull(vpnInstance, - "Vpn Instance not available with rd " + vrfTableKey.getRouteDistinguisher()); + "Vpn Instance not available with rd " + vrfTableKey.getRouteDistinguisher()); } String vpnUuid = vpnInstance.getVpnInstanceName(); Preconditions.checkArgument(vpnUuid != null && !vpnUuid.isEmpty(), @@ -428,8 +427,8 @@ public class VrfEntryListener extends AbstractDataChangeListener imple // For leaking, we need the InterVpnLink to be active. For removal, we just need a InterVpnLink. Optional interVpnLink = - (addOrRemove == NwConstants.ADD_FLOW) ? FibUtil.getActiveInterVpnLinkFromRd(dataBroker, rd) - : FibUtil.getInterVpnLinkByRd(dataBroker, rd); + (addOrRemove == NwConstants.ADD_FLOW) ? FibUtil.getActiveInterVpnLinkFromRd(dataBroker, rd) + : FibUtil.getInterVpnLinkByRd(dataBroker, rd); if ( !interVpnLink.isPresent() ) { LOG.debug("Could not find an InterVpnLink for Route-Distinguisher={}", rd); return; @@ -437,17 +436,18 @@ public class VrfEntryListener extends AbstractDataChangeListener imple // Ok, at this point everything is ready for the leaking/removal... but should it be performed? // For removal, we remove all leaked routes, but we only leak a route if the corresponding flag is enabled. - boolean proceed = (addOrRemove == NwConstants.DEL_FLOW ) - || ( RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP - && interVpnLink.get().isBgpRoutesLeaking() ); + boolean proceed = + (addOrRemove == NwConstants.DEL_FLOW) || ( RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP + && interVpnLink.get().isBgpRoutesLeaking() ); if ( proceed ) { - String theOtherVpnId = ( interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue().equals(vpnUuid) ) - ? interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue() - : vpnUuid; + boolean isVpnFirstEndpoint = interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue().equals(vpnUuid); + String theOtherVpnId = isVpnFirstEndpoint ? interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue() + : vpnUuid; String dstVpnRd = FibUtil.getVpnRd(dataBroker, theOtherVpnId); - String endpointIp = vrfEntry.getNextHopAddressList().get(0); + String endpointIp = isVpnFirstEndpoint ? interVpnLink.get().getFirstEndpoint().getIpAddress().toString() + : interVpnLink.get().getSecondEndpoint().getIpAddress().toString(); InstanceIdentifier vrfEntryIidInOtherVpn = InstanceIdentifier.builder(FibEntries.class) @@ -541,7 +541,7 @@ public class VrfEntryListener extends AbstractDataChangeListener imple } } - private void installInterVpnRouteInLFib(final String rd, final VrfEntry vrfEntry) { + public void installInterVpnRouteInLFib(final String rd, final VrfEntry vrfEntry) { // INTERVPN routes are routes in a Vpn1 that have been leaked to Vpn2. In DC-GW, this Vpn2 route is pointing // to a list of DPNs where Vpn2's VpnLink was instantiated. In these DPNs LFIB must be programmed so that the // packet is commuted from Vpn2 to Vpn1. @@ -570,12 +570,12 @@ public class VrfEntryListener extends AbstractDataChangeListener imple return; } - List targetDpns = - ( vpnIs1stEndpoint ) ? vpnLinkState.get().getFirstEndpointState().getDpId() - : vpnLinkState.get().getSecondEndpointState().getDpId(); - Long lportTag = - ( vpnIs1stEndpoint ) ? vpnLinkState.get().getSecondEndpointState().getLportTag() - : vpnLinkState.get().getFirstEndpointState().getLportTag(); + List targetDpns = vpnIs1stEndpoint ? vpnLinkState.get().getFirstEndpointState().getDpId() + : vpnLinkState.get().getSecondEndpointState().getDpId(); + Long lportTag = vpnIs1stEndpoint ? vpnLinkState.get().getSecondEndpointState().getLportTag() + : vpnLinkState.get().getFirstEndpointState().getLportTag(); + + LOG.trace("Installing flow in LFIB table for interVpnLink {}", interVpnLink.getName()); for ( BigInteger dpId : targetDpns ) { List actionsInfos = Arrays.asList(new ActionInfo(ActionType.pop_mpls, new String[]{})); @@ -590,6 +590,10 @@ public class VrfEntryListener extends AbstractDataChangeListener imple new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_INTERFACE_TABLE })); + LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB", + vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), + dpId, interVpnLink.getName()); + makeLFibTableEntry(dpId, vrfEntry.getLabel(), instructions, LFIB_INTERVPN_PRIORITY, NwConstants.ADD_FLOW, null); } @@ -606,109 +610,79 @@ public class VrfEntryListener extends AbstractDataChangeListener imple - private void installRouteInInterVpnLink(final InterVpnLink interVpnLink, final String vpnUuid, + private void installRouteInInterVpnLink(final InterVpnLinkDataComposite interVpnLink, final String vpnUuid, final VrfEntry vrfEntry, long vpnTag) { Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null"); Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null && vrfEntry.getNextHopAddressList().size() == 1); String destination = vrfEntry.getDestPrefix(); String nextHop = vrfEntry.getNextHopAddressList().get(0); + String iVpnLinkName = interVpnLink.getInterVpnLinkName(); // After having received a static route, we should check if the vpn is part of an inter-vpn-link. // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table // using as metadata the LPortTag associated to that vpn in the inter-vpn-link. - Optional interVpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLink.getName()); - if ( !interVpnLinkState.isPresent() ) { - LOG.warn("Could not find State for InterVpnLink {}", interVpnLink.getName()); - return; - } - if ( ! interVpnLinkState.get().getState().equals(InterVpnLinkState.State.Active) ) { + if ( interVpnLink.getState().or(State.Error) != State.Active ) { LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active", - destination, nextHop, interVpnLink.getName()); + destination, nextHop, iVpnLinkName); return; } + Optional optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid); + if ( !optOtherEndpointLportTag.isPresent() ) { + LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}", + vpnUuid, iVpnLinkName); + return; + } - // Everything Ok - boolean vpnIsFirstEndpoint = isVpnFirstEndPoint(interVpnLink, vpnUuid); - List targetDpns = - vpnIsFirstEndpoint ? interVpnLinkState.get().getFirstEndpointState().getDpId() - : interVpnLinkState.get().getSecondEndpointState().getDpId(); - - Long otherEndpointlportTag = - vpnIsFirstEndpoint ? interVpnLinkState.get().getSecondEndpointState().getLportTag() - : interVpnLinkState.get().getFirstEndpointState().getLportTag(); + List targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid); + if ( targetDpns.isEmpty() ) { + LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}", vpnUuid, iVpnLinkName); + return; + } BigInteger[] metadata = new BigInteger[] { - MetaDataUtil.getMetaDataForLPortDispatcher(otherEndpointlportTag.intValue(), + MetaDataUtil.getMetaDataForLPortDispatcher(optOtherEndpointLportTag.get().intValue(), ServiceIndex.getIndex(NwConstants.L3VPN_SERVICE_NAME, NwConstants.L3VPN_SERVICE_INDEX)), MetaDataUtil.getMetaDataMaskForLPortDispatcher() }; - int instIdx = 0; List instructions = Arrays.asList(new InstructionInfo(InstructionType.write_metadata, metadata).buildInstruction(0), - new InstructionInfo(InstructionType.goto_table, - new long[] { NwConstants.L3_INTERFACE_TABLE }).buildInstruction(1)); + new InstructionInfo(InstructionType.goto_table, + new long[] { NwConstants.L3_INTERFACE_TABLE }).buildInstruction(1)); String values[] = destination.split("/"); String destPrefixIpAddress = values[0]; int prefixLength = (values.length == 1) ? 0 : Integer.parseInt(values[1]); List matches = new ArrayList<>(); - matches.add(new MatchInfo(MatchFieldType.metadata, - new BigInteger[] { MetaDataUtil.getVpnIdMetadata(vpnTag), - MetaDataUtil.METADATA_MASK_VRFID })); + matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] { MetaDataUtil.getVpnIdMetadata(vpnTag), + MetaDataUtil.METADATA_MASK_VRFID })); matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_IPV4 })); if (prefixLength != 0) { matches.add(new MatchInfo(MatchFieldType.ipv4_destination, - new String[] { destPrefixIpAddress, Integer.toString(prefixLength) })); + new String[] { destPrefixIpAddress, Integer.toString(prefixLength) })); } int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength; - String flowRef = getInterVpnFibFlowRef(interVpnLink.getName(), destination, nextHop); + String flowRef = getInterVpnFibFlowRef(iVpnLinkName, destination, nextHop); Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0, - COOKIE_VM_FIB_TABLE, matches, instructions); - - for ( BigInteger dpId : targetDpns ) { - mdsalManager.installFlow(dpId, flowEntity); - } - } - - private void removeRouteFromInterVpnLink(final InterVpnLink interVpnLink, final String vpnUuid, - final VrfEntry vrfEntry) { - - Preconditions.checkNotNull(interVpnLink, "InterVpnLink cannot be null"); - Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null - && vrfEntry.getNextHopAddressList().size() == 1); + COOKIE_VM_FIB_TABLE, matches, instructions); - Optional interVpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLink.getName()); - if ( !interVpnLinkState.isPresent() ) { - LOG.warn("Could not find State for InterVpnLink {}", interVpnLink.getName()); - return; - } + LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}" , + vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef); - // Everything Ok - boolean vpnIsFirstEndpoint = isVpnFirstEndPoint(interVpnLink, vpnUuid); - List targetDpns = - vpnIsFirstEndpoint ? interVpnLinkState.get().getFirstEndpointState().getDpId() - : interVpnLinkState.get().getSecondEndpointState().getDpId(); + for ( BigInteger dpId : targetDpns ) { - String nextHop = vrfEntry.getNextHopAddressList().get(0); - String flowRef = getInterVpnFibFlowRef(interVpnLink.getName(), vrfEntry.getDestPrefix(), nextHop); - FlowKey flowKey = new FlowKey(new FlowId(flowRef)); - Flow flow = new FlowBuilder().setKey(flowKey).setId(new FlowId(flowRef)).setTableId(NwConstants.L3_FIB_TABLE) - .setFlowName(flowRef).build(); + LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nextHop={}] dpn {} for InterVpnLink {} in FIB", + vrfEntry.getDestPrefix(), vrfEntry.getLabel(), vrfEntry.getNextHopAddressList(), + dpId, interVpnLink.getInterVpnLinkName()); - for ( BigInteger dpId : targetDpns ) { - mdsalManager.removeFlow(dpId, flow); + mdsalManager.installFlow(dpId, flowEntity); } - } - private boolean isVpnFirstEndPoint(InterVpnLink interVpnLink, String vpnName) { - return interVpnLink.getFirstEndpoint().getVpnUuid().getValue().equals(vpnName); - } private Optional read(DataBroker broker, LogicalDatastoreType datastoreType, InstanceIdentifier path) { @@ -725,27 +699,9 @@ public class VrfEntryListener extends AbstractDataChangeListener imple return result; } - private void makeSubnetRouteTableMissFlow(BigInteger dpnId, int addOrRemove) { - final BigInteger COOKIE_TABLE_MISS = new BigInteger("8000004", 16); - List actionsInfos = new ArrayList(); - List instructions = new ArrayList(); - actionsInfos.add(new ActionInfo(ActionType.punt_to_controller, new String[]{})); - instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos)); - List matches = new ArrayList(); - String flowRef = getTableMissFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, NwConstants.TABLE_MISS_FLOW); - FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef, - NwConstants.TABLE_MISS_PRIORITY, "Subnet Route Table Miss", 0, 0, COOKIE_TABLE_MISS, matches, instructions); - - if (addOrRemove == NwConstants.ADD_FLOW) { - mdsalManager.installFlow(flowEntity); - } else { - mdsalManager.removeFlow(flowEntity); - } - } - private List createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) { - List returnLocalDpnId = new ArrayList(); - Prefixes localNextHopInfo = getPrefixToInterface(vpnId, vrfEntry.getDestPrefix()); + List returnLocalDpnId = new ArrayList<>(); + Prefixes localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix()); String localNextHopIP = vrfEntry.getDestPrefix(); if (localNextHopInfo == null) { @@ -755,7 +711,7 @@ public class VrfEntryListener extends AbstractDataChangeListener imple for (String nextHopIp : extraRoute.getNexthopIpList()) { LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp); if (nextHopIp != null) { - localNextHopInfo = getPrefixToInterface(vpnId, nextHopIp + "/32"); + localNextHopInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, nextHopIp + "/32"); localNextHopIP = nextHopIp + "/32"; BigInteger dpnId = checkCreateLocalFibEntry(localNextHopInfo, localNextHopIP, vpnId, rd, vrfEntry, vpnId); returnLocalDpnId.add(dpnId); @@ -803,18 +759,24 @@ public class VrfEntryListener extends AbstractDataChangeListener imple if (localNextHopInfo != null) { final BigInteger dpnId = localNextHopInfo.getDpnId(); if (!isVpnPresentInDpn(rd, dpnId)) { + LOG.error("The vpnName with vpnId {} rd {} is not available on dpn {}", vpnId, rd, dpnId.toString()); return BigInteger.ZERO; } - final long groupId = nextHopManager.createLocalNextHop(parentVpnId, dpnId, localNextHopInfo.getVpnInterfaceName(), localNextHopIP); - + final long groupId = nextHopManager.createLocalNextHop(parentVpnId, dpnId, + localNextHopInfo.getVpnInterfaceName(), localNextHopIP, vrfEntry.getDestPrefix()); + if (groupId == 0) { + LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}", + vrfEntry.getDestPrefix(), rd, localNextHopInfo.getVpnInterfaceName(), dpnId.toString()); + return BigInteger.ZERO; + } List actionsInfos = Arrays.asList(new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId)})); final List instructions = - Arrays.asList(new InstructionInfo(InstructionType.write_actions, actionsInfos)); + Arrays.asList(new InstructionInfo(InstructionType.apply_actions, actionsInfos)); actionsInfos = Arrays.asList(new ActionInfo(ActionType.pop_mpls, new String[]{}), new ActionInfo(ActionType.group, new String[] { String.valueOf(groupId) }) ); - final List lfibinstructions = Arrays.asList(new InstructionInfo(InstructionType.write_actions, actionsInfos)); + final List lfibinstructions = Arrays.asList(new InstructionInfo(InstructionType.apply_actions, actionsInfos)); if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) { LOG.debug("Installing tunnel table entry on dpn {} for interface {} with label {}", dpnId, localNextHopInfo.getVpnInterfaceName(), vrfEntry.getLabel()); @@ -868,7 +830,7 @@ public class VrfEntryListener extends AbstractDataChangeListener imple if (lri == null) { return true; } - List vpnInstancesList = lri.getVpnInstanceList(); + List vpnInstancesList = lri.getVpnInstanceList() != null ? lri.getVpnInstanceList() : new ArrayList(); if (vpnInstancesList.contains(vpnInstanceName)) { LOG.debug("vpninstance {} name is present", vpnInstanceName); vpnInstancesList.remove(vpnInstanceName); @@ -908,7 +870,7 @@ public class VrfEntryListener extends AbstractDataChangeListener imple mkMatches.add(new MatchInfo(MatchFieldType.tunnel_id, new BigInteger[] {BigInteger.valueOf(label)})); List mkInstructions = new ArrayList<>(); - mkInstructions.add(new InstructionInfo(InstructionType.write_actions, actionsInfos)); + mkInstructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos)); FlowEntity terminatingServiceTableFlowEntity = MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE,label), 5, String.format("%s:%d","TST Flow Entry ",label), @@ -1027,24 +989,12 @@ public class VrfEntryListener extends AbstractDataChangeListener imple } }); //TODO: verify below adjacency call need to be optimized (?) - deleteLocalAdjacency(dpnId, vpnId, localNextHopIP); + deleteLocalAdjacency(dpnId, vpnId, localNextHopIP, vrfEntry.getDestPrefix()); return dpnId; } return BigInteger.ZERO; } - - private InstanceIdentifier getPrefixToInterfaceIdentifier(Long vpnId, String ipPrefix) { - return InstanceIdentifier.builder(PrefixToInterface.class) - .child(VpnIds.class, new VpnIdsKey(vpnId)).child(Prefixes.class, new PrefixesKey(ipPrefix)).build(); - } - - private Prefixes getPrefixToInterface(Long vpnId, String ipPrefix) { - Optional localNextHopInfoData = - FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, getPrefixToInterfaceIdentifier(vpnId, ipPrefix)); - return localNextHopInfoData.isPresent() ? localNextHopInfoData.get() : null; - } - private InstanceIdentifier getVpnToExtrarouteIdentifier(String vrfId, String ipPrefix) { return InstanceIdentifier.builder(VpnToExtraroute.class) .child(Vpn.class, new VpnKey(vrfId)).child(Extraroute.class, @@ -1070,7 +1020,7 @@ public class VrfEntryListener extends AbstractDataChangeListener imple } } catch (InterruptedException | ExecutionException e) { - LOG.warn("Exception when getting tunnel interface Id for tunnel type {}", e); + LOG.warn("Exception when getting tunnel interface Id for tunnel type", e); } return null; @@ -1087,9 +1037,9 @@ public class VrfEntryListener extends AbstractDataChangeListener imple LOG.debug( "createremotefibentry: adding route {} for rd {} with transaction {}", vrfEntry.getDestPrefix(), rd, tx); /********************************************/ - List tunnelInterfaceList = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd); + List adjacencyResults = resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd); - if (tunnelInterfaceList.isEmpty()) { + if (adjacencyResults.isEmpty()) { LOG.error("Could not get interface for nexthop: {} in vpn {}", vrfEntry.getNextHopAddressList(), rd); LOG.warn("Failed to add Route: {} in vpn: {}", @@ -1097,34 +1047,24 @@ public class VrfEntryListener extends AbstractDataChangeListener imple return; } - for (String tunnelInterface : tunnelInterfaceList) { + for (AdjacencyResult adjacencyResult : adjacencyResults) { List instructions = new ArrayList<>(); List actionInfos = new ArrayList<>(); - Class tunnel_type = getTunnelType(tunnelInterface); - if (tunnel_type.equals(TunnelTypeMplsOverGre.class)) { - LOG.debug("Push label action for prefix {}", vrfEntry.getDestPrefix()); - actionInfos.add(new ActionInfo(ActionType.push_mpls, new String[]{null})); - actionInfos.add(new ActionInfo(ActionType.set_field_mpls_label, new String[]{Long.toString(vrfEntry.getLabel())})); + String egressInterface = adjacencyResult.getInterfaceName(); + if (Tunnel.class.equals(adjacencyResult.getInterfaceType())) { + addTunnelInterfaceActions(egressInterface, vpnId, vrfEntry, actionInfos); } else { - int label = vrfEntry.getLabel().intValue(); - BigInteger tunnelId; - // FIXME vxlan vni bit set is not working properly with OVS.need to revisit - if (tunnel_type.equals(TunnelTypeVxlan.class)) { - tunnelId = BigInteger.valueOf(label); - } else { - tunnelId = BigInteger.valueOf(label); - } - - LOG.debug("adding set tunnel id action for label {}", label); - actionInfos.add(new ActionInfo(ActionType.set_field_tunnel_id, new BigInteger[]{tunnelId})); + addRewriteDstMacAction(vpnId, vrfEntry, actionInfos); } - List egressActions = nextHopManager.getEgressActionsForInterface(tunnelInterface); - if(egressActions.isEmpty()){ - LOG.error("Failed to retrieve egress action for prefix {} nextHop {} interface {}. Aborting remote FIB entry creation.", vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), tunnelInterface); + List egressActions = nextHopManager.getEgressActionsForInterface(egressInterface); + if (egressActions.isEmpty()) { + LOG.error( + "Failed to retrieve egress action for prefix {} nextHop {} interface {}. Aborting remote FIB entry creation.", + vrfEntry.getDestPrefix(), vrfEntry.getNextHopAddressList(), egressInterface); return; } actionInfos.addAll(egressActions); - instructions.add(new InstructionInfo(InstructionType.write_actions, actionInfos)); + instructions.add(new InstructionInfo(InstructionType.apply_actions, actionInfos)); makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, tx); } if(!wrTxPresent ){ @@ -1133,6 +1073,54 @@ public class VrfEntryListener extends AbstractDataChangeListener imple LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId); } + private void addRewriteDstMacAction(long vpnId, VrfEntry vrfEntry, List actionInfos) { + String ipPrefix = vrfEntry.getDestPrefix(); + Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, ipPrefix); + if (prefixInfo == null) { + LOG.debug("No prefix info found for prefix {}", ipPrefix); + return; + } + + String ifName = prefixInfo.getVpnInterfaceName(); + if (ifName == null) { + LOG.warn("Failed to get VPN interface for prefix {}", ipPrefix); + return; + } + + String macAddress = FibUtil.getMacAddressFromPrefix(dataBroker, ifName, ipPrefix); + if (macAddress == null) { + LOG.warn("No MAC address found for VPN interface {} prefix {}", ifName, ipPrefix); + return; + } + + actionInfos.add(new ActionInfo(ActionType.set_field_eth_dest, new String[] { macAddress }, actionInfos.size())); + } + + private void addTunnelInterfaceActions(String tunnelInterface, long vpnId, VrfEntry vrfEntry, + List actionInfos) { + Class tunnel_type = getTunnelType(tunnelInterface); + if (tunnel_type.equals(TunnelTypeMplsOverGre.class)) { + LOG.debug("Push label action for prefix {}", vrfEntry.getDestPrefix()); + actionInfos.add(new ActionInfo(ActionType.push_mpls, new String[] { null })); + actionInfos.add(new ActionInfo(ActionType.set_field_mpls_label, + new String[] { Long.toString(vrfEntry.getLabel()) })); + } else { + int label = vrfEntry.getLabel().intValue(); + BigInteger tunnelId; + // FIXME vxlan vni bit set is not working properly with OVS.need to + // revisit + if (tunnel_type.equals(TunnelTypeVxlan.class)) { + tunnelId = BigInteger.valueOf(label); + } else { + tunnelId = BigInteger.valueOf(label); + } + + LOG.debug("adding set tunnel id action for label {}", label); + actionInfos.add(new ActionInfo(ActionType.set_field_tunnel_id, new BigInteger[] { tunnelId })); + addRewriteDstMacAction(vpnId, vrfEntry, actionInfos); + } + } + private void delIntfFromDpnToVpnList(long vpnId, BigInteger dpnId, String intfName, String rd) { InstanceIdentifier id = FibUtil.getVpnToDpnListIdentifier(rd, dpnId); Optional dpnInVpn = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id); @@ -1165,7 +1153,7 @@ public class VrfEntryListener extends AbstractDataChangeListener imple - vpn interface op DS */ LOG.debug("Cleanup of prefix {} in VPN {}", vrfEntry.getDestPrefix(), vpnId); - Prefixes prefixInfo = getPrefixToInterface(vpnId, vrfEntry.getDestPrefix()); + Prefixes prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, vrfEntry.getDestPrefix()); Extraroute extraRoute = null; if (prefixInfo == null) { extraRoute = getVpnToExtraroute(rd, vrfEntry.getDestPrefix()); @@ -1174,7 +1162,7 @@ public class VrfEntryListener extends AbstractDataChangeListener imple LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp); if (nextHopIp != null) { - prefixInfo = getPrefixToInterface(vpnId, nextHopIp + "/32"); + prefixInfo = FibUtil.getPrefixToInterface(dataBroker, vpnId, nextHopIp + "/32"); checkCleanUpOpDataForFib(prefixInfo, vpnId, rd, vrfEntry, extraRoute); } } @@ -1232,6 +1220,28 @@ public class VrfEntryListener extends AbstractDataChangeListener imple public List> call() throws Exception { // If another renderer(for eg : CSS) needs to be supported, check can be performed here // to call the respective helpers. + + //First Cleanup LabelRouteInfo + synchronized (vrfEntry.getLabel().toString().intern()) { + LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel()); + if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) && + vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) { + Optional vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd); + String vpnInstanceName = ""; + if (vpnInstanceOpDataEntryOptional.isPresent()) { + vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName(); + } + boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName); + if (lriRemoved) { + String parentRd = lri.getParentVpnRd(); + FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME, + FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix())); + } + } else { + FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME, + FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix())); + } + } String ifName = prefixInfo.getVpnInterfaceName(); Optional optvpnInterface = FibUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, FibUtil.getVpnInterfaceIdentifier(ifName)); @@ -1240,9 +1250,6 @@ public class VrfEntryListener extends AbstractDataChangeListener imple if (vpnId != associatedVpnId) { LOG.warn("Prefixes {} are associated with different vpn instance with id : {} rather than {}", vrfEntry.getDestPrefix(), associatedVpnId, vpnId); - LOG.trace("Releasing prefix label - rd {}, prefix {}", rd, vrfEntry.getDestPrefix()); - FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME, - FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix())); LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix()); return null; } else { @@ -1272,27 +1279,6 @@ public class VrfEntryListener extends AbstractDataChangeListener imple FibUtil.delete(dataBroker, LogicalDatastoreType.OPERATIONAL, FibUtil.getVpnInterfaceIdentifier(ifName)); } - - synchronized (vrfEntry.getLabel().toString().intern()) { - LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel()); - if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) && - vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) { - Optional vpnInstanceOpDataEntryOptional = FibUtil.getVpnInstanceOpData(dataBroker, rd); - String vpnInstanceName = ""; - if (vpnInstanceOpDataEntryOptional.isPresent()) { - vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName(); - } - boolean lriRemoved = deleteLabelRouteInfo(lri, vpnInstanceName); - if (lriRemoved) { - String parentRd = lri.getParentVpnRd(); - FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME, - FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix())); - } - } else { - FibUtil.releaseId(idManager, FibConstants.VPN_IDPOOL_NAME, - FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix())); - } - } return null; } } @@ -1360,6 +1346,9 @@ public class VrfEntryListener extends AbstractDataChangeListener imple } return; } + if (installRouterFibEntries(vrfEntry, vpnToDpnList, vpnInstance.getVpnId(), NwConstants.DEL_FLOW)) { + return; + } final List localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry); @@ -1409,19 +1398,26 @@ public class VrfEntryListener extends AbstractDataChangeListener imple // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint // of the interVpnLink. - Optional vpnUuid = FibUtil.getVpnNameFromRd(dataBroker, rd); - if ( vpnUuid.isPresent() ) { - Optional interVpnLink = FibUtil.getInterVpnLinkByVpnUuid(dataBroker, vpnUuid.get()); - String routeNexthop = vrfEntry.getNextHopAddressList().get(0); - - if ( interVpnLink.isPresent() - && ( (interVpnLink.get().getFirstEndpoint().getVpnUuid().getValue().equals(vpnUuid.get()) - && interVpnLink.get().getSecondEndpoint().getIpAddress().getValue().equals(routeNexthop)) - || (interVpnLink.get().getSecondEndpoint().getVpnUuid().getValue().equals(vpnUuid.get() ) - && interVpnLink.get().getFirstEndpoint().getIpAddress().getValue().equals(routeNexthop)) ) ) { - // This is route that points to the other endpoint of an InterVpnLink - // In that case, we should look for the FIB table pointing to LPortDispatcher table and remove it. - removeRouteFromInterVpnLink(interVpnLink.get(), rd, vrfEntry); + Optional optVpnUuid = FibUtil.getVpnNameFromRd(this.dataBroker, rd); + if ( optVpnUuid.isPresent() ) { + String vpnUuid = optVpnUuid.get(); + List routeNexthoplist = vrfEntry.getNextHopAddressList(); + if(routeNexthoplist.isEmpty()) { + LOG.trace("NextHopList is empty for VrfEntry {}", vrfEntry); + return; + } + String routeNexthop = routeNexthoplist.get(0); + Optional optInterVpnLink = InterVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid); + if ( optInterVpnLink.isPresent() ) { + InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get(); + if ( interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) + { + // This is route that points to the other endpoint of an InterVpnLink + // In that case, we should look for the FIB table pointing to LPortDispatcher table and remove it. + removeInterVPNLinkRouteFlows(interVpnLink.getInterVpnLinkName(), + interVpnLink.isFirstEndpointVpnName(rd), + vrfEntry); + } } } @@ -1492,7 +1488,7 @@ public class VrfEntryListener extends AbstractDataChangeListener imple VrfEntry vrfEntry, String rd, WriteTransaction tx){ boolean isRemoteRoute = true; if (localNextHopInfo != null) { - isRemoteRoute = (!remoteDpnId.equals(localNextHopInfo.getDpnId())); + isRemoteRoute = !remoteDpnId.equals(localNextHopInfo.getDpnId()); } if (isRemoteRoute) { deleteFibEntry(remoteDpnId, vpnId, vrfEntry, rd, tx); @@ -1544,13 +1540,22 @@ public class VrfEntryListener extends AbstractDataChangeListener imple matches.add(new MatchInfo(MatchFieldType.metadata, new BigInteger[] { MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID })); - matches.add(new MatchInfo(MatchFieldType.eth_type, - new long[] { NwConstants.ETHTYPE_IPV4 })); - - if(prefixLength != 0) { - matches.add(new MatchInfo(MatchFieldType.ipv4_destination, new String[] { - destPrefix.getHostAddress(), Integer.toString(prefixLength)})); + if (destPrefix instanceof Inet4Address) { + matches.add(new MatchInfo(MatchFieldType.eth_type, + new long[] { NwConstants.ETHTYPE_IPV4 })); + if(prefixLength != 0) { + matches.add(new MatchInfo(MatchFieldType.ipv4_destination, new String[] { + destPrefix.getHostAddress(), Integer.toString(prefixLength)})); + } + } else { + matches.add(new MatchInfo(MatchFieldType.eth_type, + new long[] { NwConstants.ETHTYPE_IPV6 })); + if(prefixLength != 0) { + matches.add(new MatchInfo(MatchFieldType.ipv6_destination, new String[] { + destPrefix.getHostAddress() + "/" + Integer.toString(prefixLength)})); + } } + int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength; String flowRef = getFlowRef(dpId, NwConstants.L3_FIB_TABLE, rd, priority, destPrefix); FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0, @@ -1564,6 +1569,20 @@ public class VrfEntryListener extends AbstractDataChangeListener imple InstanceIdentifier flowInstanceId = InstanceIdentifier.builder(Nodes.class) .child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class) .child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build(); + + if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) { + SubTransaction subTransaction = new SubTransactionImpl(); + if (addOrRemove == NwConstants.ADD_FLOW) { + subTransaction.setInstanceIdentifier(flowInstanceId); + subTransaction.setInstance(flow); + subTransaction.setAction(SubTransaction.CREATE); + } else { + subTransaction.setInstanceIdentifier(flowInstanceId); + subTransaction.setAction(SubTransaction.DELETE); + } + transactionObjects.add(subTransaction); + } + if (addOrRemove == NwConstants.ADD_FLOW) { tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId,flow, true); } else { @@ -1618,71 +1637,86 @@ public class VrfEntryListener extends AbstractDataChangeListener imple if(!wrTxPresent ){ tx.submit(); } - LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} modified successfully {}", - dpId, label, instructions ); + + LOG.debug("LFIB Entry for dpID {} : label : {} instructions {} : key {} {} successfully", + dpId, label, instructions, flowKey, (NwConstants.ADD_FLOW == addOrRemove) ? "ADDED" : "REMOVED"); } - private void deleteLocalAdjacency(final BigInteger dpId, final long vpnId, final String ipAddress) { + private void deleteLocalAdjacency(final BigInteger dpId, final long vpnId, final String ipAddress, + final String ipPrefixAddress) { LOG.trace("deleteLocalAdjacency called with dpid {}, vpnId{}, ipAddress {}",dpId, vpnId, ipAddress); try { - nextHopManager.removeLocalNextHop(dpId, vpnId, ipAddress); + nextHopManager.removeLocalNextHop(dpId, vpnId, ipAddress, ipPrefixAddress); } catch (NullPointerException e) { LOG.trace("", e); } } public void populateFibOnNewDpn(final BigInteger dpnId, final long vpnId, final String rd, - final FutureCallback> callback) { + final FutureCallback> callback) { LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd); InstanceIdentifier id = buildVrfId(rd); final VpnInstanceOpDataEntry vpnInstance = getVpnInstance(rd); final Optional vrfTable = FibUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id); - if (vrfTable.isPresent()) { - DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance(); - dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(), - new Callable>>() { - @Override - public List> call() throws Exception { - List> futures = new ArrayList<>(); - synchronized (vpnInstance.getVpnInstanceName().intern()) { - WriteTransaction tx = dataBroker.newWriteOnlyTransaction(); - for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) { - - SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class); - if (subnetRoute != null) { - long elanTag = subnetRoute.getElantag(); - installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx); - continue; - } - if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) { - //Handle local flow creation for imports - LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel()); - if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix() - ) && vrfEntry.getNextHopAddressList() - .contains(lri.getNextHopIpList().get(0))) { - if (lri.getDpnId().equals(dpnId)) { - createLocalFibEntry(vpnId, rd, vrfEntry); - continue; - } - } - } - // Passing null as we don't know the dpn - // to which prefix is attached at this point - createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx); - } - //TODO: if we have 100K entries in FIB, can it fit in one Tranasaction (?) - futures.add(tx.submit()); - if (callback != null) { - ListenableFuture> listenableFuture = Futures.allAsList(futures); - Futures.addCallback(listenableFuture, callback); + if (!vrfTable.isPresent()) { + LOG.warn("VRF Table not yet available for RD {}", rd); + if (callback != null) { + List> futures = new ArrayList<>(); + ListenableFuture> listenableFuture = Futures.allAsList(futures); + Futures.addCallback(listenableFuture, callback); + } + return; + } + DataStoreJobCoordinator dataStoreCoordinator = DataStoreJobCoordinator.getInstance(); + dataStoreCoordinator.enqueueJob("FIB-" + vpnId + "-" + dpnId.toString(), + new Callable>>() { + @Override + public List> call() throws Exception { + List> futures = new ArrayList<>(); + synchronized (vpnInstance.getVpnInstanceName().intern()) { + WriteTransaction tx = dataBroker.newWriteOnlyTransaction(); + for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) { + SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class); + if (subnetRoute != null) { + long elanTag = subnetRoute.getElantag(); + installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx); + continue; + } + RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class); + if (routerInt != null) { + LOG.trace( "Router augmented vrfentry found rd:{}, uuid:{}, ip:{}, mac:{}", + rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress()); + installRouterFibEntry(vrfEntry, dpnId, vpnId, routerInt.getUuid(), routerInt.getIpAddress(), + new MacAddress(routerInt.getMacAddress()), NwConstants.ADD_FLOW); + continue; + } + if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) { //Handle local flow creation for imports + LabelRouteInfo lri = getLabelRouteInfo(vrfEntry.getLabel()); + if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) + && vrfEntry.getNextHopAddressList().contains(lri.getNextHopIpList().get(0))) { + if (lri.getDpnId().equals(dpnId)) { + createLocalFibEntry(vpnId, rd, vrfEntry); + continue; } } - return futures; } - }); - } + // Passing null as we don't know the dpn + // to which prefix is attached at this point + createRemoteFibEntry(dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx); + } + //TODO: if we have 100K entries in FIB, can it fit in one Tranasaction (?) + futures.add(tx.submit()); + } + if (callback != null) { + ListenableFuture> listenableFuture = Futures.allAsList(futures); + Futures.addCallback(listenableFuture, callback); + } + return futures; + } + }); } + public void populateFibOnDpn(final BigInteger dpnId, final long vpnId, final String rd, final String localNextHopIp, final String remoteNextHopIp) { LOG.trace("dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} : populateFibOnDpn", @@ -1789,8 +1823,9 @@ public class VrfEntryListener extends AbstractDataChangeListener imple WriteTransaction writeTransaction = dataBroker.newWriteOnlyTransaction(); VrfTablesKey vrfTablesKey = new VrfTablesKey(rd); VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix); - if (vrfEntry == null) + if (vrfEntry == null) { return futures; + } LOG.trace("handleRemoteRoute :: action {}, localDpnId {}, " + "remoteDpnId {} , vpnId {}, rd {}, destPfx {}", action, localDpnId, remoteDpnId, vpnId, rd, destPrefix); @@ -1836,6 +1871,15 @@ public class VrfEntryListener extends AbstractDataChangeListener imple vrfEntry.getDestPrefix()); continue; } + // ping responder for router interfaces + RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class); + if (routerInt != null) { + LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}", + rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress()); + installRouterFibEntry(vrfEntry, dpnId, vpnId, routerInt.getUuid(), routerInt.getIpAddress(), + new MacAddress(routerInt.getMacAddress()), NwConstants.DEL_FLOW); + continue; + } // Passing null as we don't know the dpn // to which prefix is attached at this point deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, tx); @@ -1945,9 +1989,9 @@ public class VrfEntryListener extends AbstractDataChangeListener imple .append(nextHop).toString(); } - protected List resolveAdjacency(final BigInteger remoteDpnId, final long vpnId, + protected List resolveAdjacency(final BigInteger remoteDpnId, final long vpnId, final VrfEntry vrfEntry, String rd) { - List adjacencyList = new ArrayList<>(); + List adjacencyList = new ArrayList<>(); List prefixIpList = new ArrayList<>(); LOG.trace("resolveAdjacency called with remotedpid {}, vpnId{}, VrfEntry {}", remoteDpnId, vpnId, vrfEntry); @@ -1969,10 +2013,10 @@ public class VrfEntryListener extends AbstractDataChangeListener imple for (String prefixIp : prefixIpList) { for (String nextHopIp : vrfEntry.getNextHopAddressList()) { LOG.debug("NextHop IP for destination {} is {}", prefixIp, nextHopIp); - String adjacency = nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId, + AdjacencyResult adjacencyResult = nextHopManager.getRemoteNextHopPointer(remoteDpnId, vpnId, prefixIp, nextHopIp); - if (adjacency != null && !adjacency.isEmpty() && !adjacencyList.contains(adjacency)) { - adjacencyList.add(adjacency); + if (adjacencyResult != null && !adjacencyList.contains(adjacencyResult)) { + adjacencyList.add(adjacencyResult); } } } @@ -1991,41 +2035,6 @@ public class VrfEntryListener extends AbstractDataChangeListener imple return vpnInstanceOpData.isPresent() ? vpnInstanceOpData.get() : null; } - public void processNodeAdd(BigInteger dpnId) { - LOG.debug("Received notification to install TableMiss entries for dpn {} ", dpnId); - makeTableMissFlow(dpnId, NwConstants.ADD_FLOW); - makeL3IntfTblMissFlow(dpnId, NwConstants.ADD_FLOW); - makeSubnetRouteTableMissFlow(dpnId, NwConstants.ADD_FLOW); - } - - private void makeTableMissFlow(BigInteger dpnId, int addOrRemove) { - final BigInteger COOKIE_TABLE_MISS = new BigInteger("1030000", 16); - // Instruction to goto L3 InterfaceTable - List instructions = new ArrayList(); - instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.L3_INTERFACE_TABLE })); - List matches = new ArrayList(); - FlowEntity flowEntityLfib = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_LFIB_TABLE, - getTableMissFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, NwConstants.TABLE_MISS_FLOW), - NwConstants.TABLE_MISS_PRIORITY, "Table Miss", 0, 0, COOKIE_TABLE_MISS, matches, instructions); - - FlowEntity flowEntityFib = MDSALUtil.buildFlowEntity(dpnId,NwConstants.L3_FIB_TABLE, - getTableMissFlowRef(dpnId, NwConstants.L3_FIB_TABLE, - NwConstants.TABLE_MISS_FLOW), - NwConstants.TABLE_MISS_PRIORITY, "FIB Table Miss Flow", - 0, 0, COOKIE_VM_FIB_TABLE, - matches, instructions); - - if (addOrRemove == NwConstants.ADD_FLOW) { - LOG.debug("Invoking MDSAL to install Table Miss Entries"); - mdsalManager.installFlow(flowEntityLfib); - mdsalManager.installFlow(flowEntityFib); - } else { - mdsalManager.removeFlow(flowEntityLfib); - mdsalManager.removeFlow(flowEntityFib); - - } - } - private String getTableMissFlowRef(BigInteger dpnId, short tableId, int tableMiss) { return new StringBuffer().append(FLOWID_PREFIX).append(dpnId).append(NwConstants.FLOWID_SEPARATOR) .append(tableId).append(NwConstants.FLOWID_SEPARATOR).append(tableMiss) @@ -2062,7 +2071,7 @@ public class VrfEntryListener extends AbstractDataChangeListener imple } public List printFibEntries() { - List result = new ArrayList(); + List result = new ArrayList<>(); result.add(String.format(" %-7s %-20s %-20s %-7s %-7s", "RD", "Prefix", "NextHop", "Label", "Origin")); result.add("-------------------------------------------------------------------"); InstanceIdentifier id = InstanceIdentifier.create(FibEntries.class); @@ -2087,29 +2096,6 @@ public class VrfEntryListener extends AbstractDataChangeListener imple return result; } - private void makeL3IntfTblMissFlow(BigInteger dpnId, int addOrRemove) { - List instructions = new ArrayList(); - List matches = new ArrayList(); - final BigInteger COOKIE_TABLE_MISS = new BigInteger("1030000", 16); - // Instruction to goto L3 InterfaceTable - - List actionsInfos = new ArrayList (); - actionsInfos.add(new ActionInfo(ActionType.nx_resubmit, new String[]{ - Short.toString(NwConstants.LPORT_DISPATCHER_TABLE)})); - instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos)); - //instructions.add(new InstructionInfo(InstructionType.goto_table, new long[] { NwConstants.LPORT_DISPATCHER_TABLE })); - - FlowEntity flowEntityL3Intf = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_INTERFACE_TABLE, - getTableMissFlowRef(dpnId, NwConstants.L3_INTERFACE_TABLE, NwConstants.TABLE_MISS_FLOW), - NwConstants.TABLE_MISS_PRIORITY, "L3 Interface Table Miss", 0, 0, COOKIE_TABLE_MISS, - matches, instructions); - if (addOrRemove == NwConstants.ADD_FLOW) { - LOG.debug("Invoking MDSAL to install L3 interface Table Miss Entries"); - mdsalManager.installFlow(flowEntityL3Intf); - } else { - mdsalManager.removeFlow(flowEntityL3Intf); - } - } private VrfEntry getVrfEntry(DataBroker broker, String rd, String ipPrefix) { InstanceIdentifier vrfEntryId = @@ -2117,7 +2103,7 @@ public class VrfEntryListener extends AbstractDataChangeListener imple child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build(); Optional vrfEntry = read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId); if (vrfEntry.isPresent()) { - return (vrfEntry.get()); + return vrfEntry.get(); } return null; } @@ -2128,4 +2114,145 @@ public class VrfEntryListener extends AbstractDataChangeListener imple child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build(); return vrfEntryId; } + + protected Boolean isIpv4Address(String ipAddress) { + try { + InetAddress address = InetAddress.getByName(ipAddress); + if (address instanceof Inet4Address) { + return true; + } + } catch (UnknownHostException e) { + LOG.warn("Invalid ip address {}", ipAddress, e); + return false; + } + return false; + } + + protected Boolean installRouterFibEntries(final VrfEntry vrfEntry, final Collection vpnToDpnList, + long vpnId, int addOrRemove) { + RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class); + if (routerInt != null && vpnToDpnList != null) { + String routerId = routerInt.getUuid(); + String macAddress = routerInt.getMacAddress(); + String ipValue = routerInt.getIpAddress(); + LOG.trace("createFibEntries - Router augmented vrfentry found for for router uuid:{}, ip:{}, mac:{}", + routerId, ipValue, macAddress); + for (VpnToDpnList vpnDpn : vpnToDpnList) { + if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) { + installRouterFibEntry(vrfEntry, vpnDpn.getDpnId(), vpnId, routerId, ipValue, + new MacAddress(macAddress), addOrRemove); + } + } + return true; + } + return false; + } + + public void installRouterFibEntry(final VrfEntry vrfEntry, BigInteger dpnId, long vpnId, String routerUuid, + String routerInternalIp, MacAddress routerMac, int addOrRemove) { + String[] subSplit = routerInternalIp.split("/"); + if (!isIpv4Address(subSplit[0])) { + // Ping responder using OpenFlow rules is only supported for IPv4, hence skipping. + return; + } + + String addRemoveStr = (addOrRemove == NwConstants.ADD_FLOW) ? "ADD_FLOW" : "DELETE_FLOW"; + LOG.trace("{}: bulding Echo Flow entity for dpid:{}, router_ip:{}, vpnId:{}, subSplit:{} ", addRemoveStr, + dpnId, routerInternalIp, vpnId, subSplit[0]); + + List matches = new ArrayList<>(); + + matches.add(new MatchInfo(MatchFieldType.ip_proto, new long[] { IPProtocols.ICMP.intValue() })); + matches.add(new MatchInfo(MatchFieldType.metadata, + new BigInteger[] { MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID })); + matches.add(new MatchInfo(MatchFieldType.icmp_v4, new long[] { (short) 8, (short) 0 })); + matches.add(new MatchInfo(MatchFieldType.eth_type, new long[] { NwConstants.ETHTYPE_IPV4 })); + matches.add(new MatchInfo(MatchFieldType.ipv4_destination, new String[] { subSplit[0], "32" })); + + List actionsInfos = new ArrayList<>(); + + // Set Eth Src and Eth Dst + actionsInfos.add(new ActionInfo(ActionType.move_src_dst_eth, new String[] {})); + actionsInfos.add(new ActionInfo(ActionType.set_field_eth_src, new String[] { routerMac.getValue() })); + + // Move Ip Src to Ip Dst + actionsInfos.add(new ActionInfo(ActionType.move_src_dst_ip, new String[] {})); + actionsInfos.add(new ActionInfo(ActionType.set_source_ip, new String[] { subSplit[0], "32" })); + + // Set the ICMP type to 0 (echo reply) + actionsInfos.add(new ActionInfo(ActionType.set_icmp_type, new String[] { "0" })); + + actionsInfos.add(new ActionInfo(ActionType.nx_load_in_port, new BigInteger[]{ BigInteger.ZERO })); + + actionsInfos.add(new ActionInfo(ActionType.nx_resubmit, + new String[] { Short.toString(NwConstants.L3_FIB_TABLE) })); + + List instructions = new ArrayList<>(); + + instructions.add(new InstructionInfo(InstructionType.apply_actions, actionsInfos)); + + int priority = FibConstants.DEFAULT_FIB_FLOW_PRIORITY + FibConstants.DEFAULT_PREFIX_LENGTH; + String flowRef = getFlowRef(dpnId, NwConstants.L3_FIB_TABLE, vrfEntry.getLabel(), priority); + + FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, + 0, 0, NwConstants.COOKIE_VM_FIB_TABLE, matches, instructions); + + if (addOrRemove == NwConstants.ADD_FLOW) { + mdsalManager.installFlow(flowEntity); + } else { + mdsalManager.removeFlow(flowEntity); + } + } + + public void removeInterVPNLinkRouteFlows(final String interVpnLinkName, + final boolean isVpnFirstEndPoint, + final VrfEntry vrfEntry) + { + Preconditions.checkArgument(vrfEntry.getNextHopAddressList() != null + && vrfEntry.getNextHopAddressList().size() == 1); + Optional interVpnLinkState = FibUtil.getInterVpnLinkState(dataBroker, interVpnLinkName); + + if ( !interVpnLinkState.isPresent()) { + LOG.warn("Could not find State for InterVpnLink {}", interVpnLinkName); + return; + } + + List targetDpns = + isVpnFirstEndPoint ? interVpnLinkState.get().getFirstEndpointState().getDpId() + : interVpnLinkState.get().getSecondEndpointState().getDpId(); + + String nextHop = vrfEntry.getNextHopAddressList().get(0); + + // delete from FIB + // + String flowRef = getInterVpnFibFlowRef(interVpnLinkName, vrfEntry.getDestPrefix(), nextHop); + FlowKey flowKey = new FlowKey(new FlowId(flowRef)); + Flow flow = new FlowBuilder().setKey(flowKey).setId(new FlowId(flowRef)).setTableId(NwConstants.L3_FIB_TABLE) + .setFlowName(flowRef).build(); + + LOG.trace("Removing flow in FIB table for interVpnLink {} key {}", + interVpnLinkName, flowRef); + + for ( BigInteger dpId : targetDpns ) { + LOG.debug("Removing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in FIB", + vrfEntry.getDestPrefix(), vrfEntry.getLabel(), nextHop, + dpId, interVpnLinkName); + + mdsalManager.removeFlow(dpId, flow); + } + + // delete from LFIB + // + LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName); + + WriteTransaction tx = dataBroker.newWriteOnlyTransaction(); + for ( BigInteger dpId : targetDpns ) { + LOG.debug("Removing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB", + vrfEntry.getDestPrefix(), vrfEntry.getLabel(), nextHop, + dpId, interVpnLinkName); + makeLFibTableEntry(dpId, vrfEntry.getLabel(), null /* no instructions */, + LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW, tx); + } + tx.submit(); + } }