X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=fibmanager%2Fimpl%2Fsrc%2Fmain%2Fjava%2Forg%2Fopendaylight%2Fnetvirt%2Ffibmanager%2FVrfEntryListener.java;h=10d646d8fbfc9c8de81780c8ba2d5d63ae9ed698;hb=97c8a6961453727e929878b58399626485382a1f;hp=6003a11c26f1530128f54eda0fc9fd3cfb0c7e37;hpb=38059c30d7abcf158eb1c2bec53172bdcd85b3b7;p=netvirt.git diff --git a/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java b/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java old mode 100755 new mode 100644 index 6003a11c26..10d646d8fb --- a/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java +++ b/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/VrfEntryListener.java @@ -7,13 +7,12 @@ */ package org.opendaylight.netvirt.fibmanager; -import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS; import static org.opendaylight.genius.infra.Datastore.CONFIGURATION; import static org.opendaylight.genius.infra.Datastore.OPERATIONAL; import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address; -import com.google.common.base.Optional; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -24,18 +23,21 @@ import java.net.InetAddress; import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.concurrent.Callable; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; -import javax.annotation.PostConstruct; +import java.util.concurrent.locks.ReentrantLock; import javax.inject.Inject; import javax.inject.Singleton; -import org.opendaylight.controller.md.sal.binding.api.DataBroker; -import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType; -import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase; +import org.eclipse.jdt.annotation.NonNull; +import org.eclipse.jdt.annotation.Nullable; +import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker; import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar; import org.opendaylight.genius.infra.Datastore.Configuration; import org.opendaylight.genius.infra.Datastore.Operational; @@ -53,7 +55,6 @@ import org.opendaylight.genius.mdsalutil.MatchInfo; import org.opendaylight.genius.mdsalutil.MetaDataUtil; import org.opendaylight.genius.mdsalutil.NWUtil; import org.opendaylight.genius.mdsalutil.NwConstants; -import org.opendaylight.genius.mdsalutil.UpgradeState; import org.opendaylight.genius.mdsalutil.actions.ActionDrop; import org.opendaylight.genius.mdsalutil.actions.ActionGroup; import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls; @@ -66,10 +67,14 @@ import org.opendaylight.genius.mdsalutil.matches.MatchIpv4Destination; import org.opendaylight.genius.mdsalutil.matches.MatchMetadata; import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel; import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId; +import org.opendaylight.genius.utils.JvmGlobalLocks; import org.opendaylight.genius.utils.ServiceIndex; import org.opendaylight.genius.utils.batching.SubTransaction; import org.opendaylight.infrautils.jobcoordinator.JobCoordinator; -import org.opendaylight.infrautils.utils.concurrent.ListenableFutures; +import org.opendaylight.infrautils.utils.concurrent.Executors; +import org.opendaylight.infrautils.utils.concurrent.LoggingFutures; +import org.opendaylight.mdsal.binding.api.DataBroker; +import org.opendaylight.mdsal.common.api.LogicalDatastoreType; import org.opendaylight.netvirt.elanmanager.api.IElanService; import org.opendaylight.netvirt.fibmanager.NexthopManager.AdjacencyResult; import org.opendaylight.netvirt.fibmanager.api.FibHelper; @@ -78,6 +83,8 @@ import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper; import org.opendaylight.netvirt.vpnmanager.api.VpnHelper; import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache; import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite; +import org.opendaylight.serviceutils.tools.listener.AbstractAsyncDataTreeChangeListener; +import org.opendaylight.serviceutils.upgrade.UpgradeState; import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowCapableNode; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId; @@ -87,6 +94,7 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.ta import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey; import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction; +import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionKey; import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.Nodes; import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.nodes.Node; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries; @@ -101,34 +109,37 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev15033 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePathsKey; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.PrefixesBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnListKey; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.Adjacency; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.AdjacencyBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.netvirt.inter.vpn.link.rev160311.inter.vpn.link.states.InterVpnLinkState.State; import org.opendaylight.yangtools.yang.binding.InstanceIdentifier; import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder; +import org.opendaylight.yangtools.yang.common.Uint32; +import org.opendaylight.yangtools.yang.common.Uint64; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - @Singleton -public class VrfEntryListener extends AsyncDataTreeChangeListenerBase { +public class VrfEntryListener extends AbstractAsyncDataTreeChangeListener { private static final Logger LOG = LoggerFactory.getLogger(VrfEntryListener.class); private static final String FLOWID_PREFIX = "L3."; - private static final BigInteger COOKIE_VM_FIB_TABLE = new BigInteger("8000003", 16); + private static final Uint64 COOKIE_VM_FIB_TABLE = Uint64.valueOf("8000003", 16).intern(); private static final int DEFAULT_FIB_FLOW_PRIORITY = 10; private static final int IPV4_ADDR_PREFIX_LENGTH = 32; private static final int LFIB_INTERVPN_PRIORITY = 15; - public static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16); + public static final Uint64 COOKIE_TUNNEL = Uint64.valueOf("9000000", 16).intern(); private static final int MAX_RETRIES = 3; - private static final BigInteger COOKIE_TABLE_MISS = new BigInteger("8000004", 16); + private static final Uint64 COOKIE_TABLE_MISS = Uint64.valueOf("8000004", 16).intern(); private final DataBroker dataBroker; private final ManagedNewTransactionRunner txRunner; @@ -158,7 +169,9 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase getWildCardPath() { - return InstanceIdentifier.create(FibEntries.class).child(VrfTables.class).child(VrfEntry.class); - } - - @Override - protected void add(final InstanceIdentifier identifier, final VrfEntry vrfEntry) { + public void add(final InstanceIdentifier identifier, final VrfEntry vrfEntry) { Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty."); String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher(); LOG.debug("ADD: Adding Fib Entry rd {} prefix {} route-paths {}", @@ -225,7 +226,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase identifier, VrfEntry vrfEntry) { + public void remove(InstanceIdentifier identifier, VrfEntry vrfEntry) { Preconditions.checkNotNull(vrfEntry, "VrfEntry should not be null or empty."); String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher(); LOG.debug("REMOVE: Removing Fib Entry rd {} prefix {} route-paths {}", @@ -259,7 +260,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase identifier, VrfEntry original, VrfEntry update) { + public void update(InstanceIdentifier identifier, VrfEntry original, VrfEntry update) { Preconditions.checkNotNull(update, "VrfEntry should not be null or empty."); final String rd = identifier.firstKeyOf(VrfTables.class).getRouteDistinguisher(); LOG.debug("UPDATE: Updating Fib Entries to rd {} prefix {} route-paths {} origin {} old-origin {}", rd, @@ -298,8 +299,8 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase originalRoutePath = original.getRoutePaths(); - List updateRoutePath = update.getRoutePaths(); + List originalRoutePath = new ArrayList(original.nonnullRoutePaths().values()); + List updateRoutePath = new ArrayList(update.nonnullRoutePaths().values()); LOG.info("UPDATE: Original route-path {} update route-path {} ", originalRoutePath, updateRoutePath); //Updates need to be handled for extraroute even if original vrf entry route path is null or @@ -384,34 +385,35 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase vpnToDpnList; + final Map keyVpnToDpnListMap; if (vrfEntry.getParentVpnRd() != null && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) { // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries. VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd()); - vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() : + keyVpnToDpnListMap = parentVpnInstance != null ? parentVpnInstance.nonnullVpnToDpnList() : vpnInstance.getVpnToDpnList(); LOG.info("createFibEntries: Processing creation of PNF FIB entry with rd {} prefix {}", vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix()); } else { - vpnToDpnList = vpnInstance.getVpnToDpnList(); + keyVpnToDpnListMap = vpnInstance.nonnullVpnToDpnList(); } - final Long vpnId = vpnInstance.getVpnId(); + final Uint32 vpnId = vpnInstance.getVpnId(); final String rd = vrfTableKey.getRouteDistinguisher(); SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class); if (subnetRoute != null) { - final long elanTag = subnetRoute.getElantag(); + final long elanTag = subnetRoute.getElantag().toJava(); LOG.trace("SUBNETROUTE: createFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}" + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag); - if (vpnToDpnList != null) { + if (keyVpnToDpnListMap != null) { jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()), () -> Collections.singletonList( txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> { - for (final VpnToDpnList curDpn : vpnToDpnList) { + for (final VpnToDpnList curDpn : keyVpnToDpnListMap.values()) { if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) { - installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId, vrfEntry, tx); - installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnId.longValue(), - vrfEntry, NwConstants.ADD_FLOW, tx); + installSubnetRouteInFib(curDpn.getDpnId(), + elanTag, rd, vpnId, vrfEntry, tx); + installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, + vpnId, vrfEntry, NwConstants.ADD_FLOW, tx); } } }))); @@ -427,22 +429,25 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry, etherType); - if (!localDpnIdList.isEmpty() && vpnToDpnList != null) { + final List localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), + rd, vrfEntry, etherType); + if (!localDpnIdList.isEmpty() && keyVpnToDpnListMap != null) { jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()), () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> { - synchronized (vpnInstance.getVpnInstanceName().intern()) { - for (VpnToDpnList vpnDpn : vpnToDpnList) { + final ReentrantLock lock = lockFor(vpnInstance); + lock.lock(); + try { + for (VpnToDpnList vpnDpn : keyVpnToDpnListMap.values()) { if (!localDpnIdList.contains(vpnDpn.getDpnId())) { if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) { try { if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) { bgpRouteVrfEntryHandler.createRemoteFibEntry(vpnDpn.getDpnId(), vpnId, vrfTableKey.getRouteDistinguisher(), vrfEntry, - TransactionAdapter.toWriteTransaction(tx), - txnObjects); + TransactionAdapter.toWriteTransaction(tx), txnObjects); } else { - createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), + createRemoteFibEntry(vpnDpn.getDpnId(), + vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry, tx); } } catch (NullPointerException e) { @@ -452,6 +457,8 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase optVpnUuid = fibUtil.getVpnNameFromRd(rd); if (optVpnUuid.isPresent()) { String vpnUuid = optVpnUuid.get(); - InterVpnLinkDataComposite interVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid).orNull(); + InterVpnLinkDataComposite interVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid).orElse(null); if (interVpnLink != null) { LOG.debug("InterVpnLink {} found in Cache linking Vpn {}", interVpnLink.getInterVpnLinkName(), vpnUuid); FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> { @@ -478,7 +485,14 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase vrfEntryId = InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd)) .child(VrfEntry.class, new VrfEntryKey(prefix)).build(); - Optional vrfEntry = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfEntryId); + Optional vrfEntry; + try { + vrfEntry = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION, + vrfEntryId); + } catch (ExecutionException | InterruptedException e) { + LOG.error("refreshFibTables: Exception while reading VrfEntry Ds for the prefix {} rd {}", prefix, rd, e); + return; + } if (vrfEntry.isPresent()) { createFibEntries(vrfEntryId, vrfEntry.get()); } @@ -496,7 +510,8 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase vpnInstanceNames = lri.getVpnInstanceList(); + List vpnInstanceNames = + lri.getVpnInstanceList() != null ? new ArrayList<>(lri.getVpnInstanceList()) : new ArrayList<>(); vpnInstanceNames.add(vpnInstanceName); builder.setVpnInstanceList(vpnInstanceNames); MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriId, builder.build()); @@ -506,10 +521,10 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase tx) { + void installSubnetRouteInFib(final Uint64 dpnId, final long elanTag, final String rd, + final Uint32 vpnId, final VrfEntry vrfEntry, TypedWriteTransaction tx) { if (tx == null) { - ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, + LoggingFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, newTx -> installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, newTx)), LOG, "Error installing subnet route in FIB"); return; @@ -523,8 +538,11 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase { List nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry); - synchronized (label.toString().intern()) { - LabelRouteInfo lri = getLabelRouteInfo(label); + final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label); + final ReentrantLock lock = lockFor(lriKey); + lock.lock(); + try { + LabelRouteInfo lri = getLabelRouteInfo(lriKey); if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) { if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) { @@ -540,17 +558,19 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase instructions = new ArrayList<>(); - BigInteger subnetRouteMeta = BigInteger.valueOf(elanTag).shiftLeft(24) - .or(BigInteger.valueOf(vpnId).shiftLeft(1)); + Uint64 subnetRouteMeta = Uint64.valueOf(BigInteger.valueOf(elanTag).shiftLeft(24) + .or(BigInteger.valueOf(vpnId.longValue()).shiftLeft(1))); instructions.add(new InstructionWriteMetadata(subnetRouteMeta, MetaDataUtil.METADATA_MASK_SUBNET_ROUTE)); instructions.add(new InstructionGotoTable(NwConstants.L3_SUBNET_ROUTE_TABLE)); baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, instructions, NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null); if (vrfEntry.getRoutePaths() != null) { - for (RoutePaths routePath : vrfEntry.getRoutePaths()) { + for (RoutePaths routePath : vrfEntry.getRoutePaths().values()) { if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) { List actionsInfos = new ArrayList<>(); // reinitialize instructions list for LFIB Table @@ -561,14 +581,14 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase tx) { List matches = new ArrayList<>(); @@ -588,7 +608,8 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase optLportTag = interVpnLink.getEndpointLportTagByVpnName(vpnName); + Optional optLportTag = interVpnLink.getEndpointLportTagByVpnName(vpnName); if (!optLportTag.isPresent()) { LOG.warn("Could not retrieve lportTag for VPN {} endpoint in InterVpnLink {}", vpnName, interVpnLinkName); return; } - Long lportTag = optLportTag.get(); - Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(null); + Long lportTag = optLportTag.get().toJava(); + Uint32 label = FibUtil.getLabelFromRoutePaths(vrfEntry).orElse(null); if (label == null) { LOG.error("Could not find label in vrfEntry=[prefix={} routePaths={}]. LFIB entry for InterVpnLink skipped", vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths()); @@ -660,9 +681,9 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase interVpnNextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry); - List targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName); + List targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName); - for (BigInteger dpId : targetDpns) { + for (Uint64 dpId : targetDpns) { LOG.debug("Installing flow: VrfEntry=[prefix={} label={} nexthop={}] dpn {} for InterVpnLink {} in LFIB", vrfEntry.getDestPrefix(), label, interVpnNextHopList, dpId, interVpnLink.getInterVpnLinkName()); @@ -676,31 +697,31 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase(vrfEntry.getRoutePaths().values()).get(0).getNexthopAddress(); String interVpnLinkName = interVpnLink.getInterVpnLinkName(); // After having received a static route, we should check if the vpn is part of an inter-vpn-link. // In that case, we should populate the FIB table of the VPN pointing to LPortDisptacher table // using as metadata the LPortTag associated to that vpn in the inter-vpn-link. - if (interVpnLink.getState().or(State.Error) != State.Active) { + if (interVpnLink.getState().orElse(State.Error) != State.Active) { LOG.warn("Route to {} with nexthop={} cannot be installed because the interVpnLink {} is not active", destination, nextHop, interVpnLinkName); return; } - Optional optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid); + Optional optOtherEndpointLportTag = interVpnLink.getOtherEndpointLportTagByVpnName(vpnUuid); if (!optOtherEndpointLportTag.isPresent()) { LOG.warn("Could not find suitable LportTag for the endpoint opposite to vpn {} in interVpnLink {}", vpnUuid, interVpnLinkName); return; } - List targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid); + List targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnUuid); if (targetDpns.isEmpty()) { LOG.warn("Could not find DPNs for endpoint opposite to vpn {} in interVpnLink {}", vpnUuid, interVpnLinkName); @@ -712,7 +733,8 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase matches = new ArrayList<>(); - matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag), MetaDataUtil.METADATA_MASK_VRFID)); + matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnTag.longValue()), + MetaDataUtil.METADATA_MASK_VRFID)); matches.add(MatchEthernetType.IPV4); if (prefixLength != 0) { @@ -726,16 +748,21 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase instructionsMap = new HashMap(); + int instructionKey = 0; + for (Instruction instructionObj : instructions) { + instructionsMap.put(new InstructionKey(++instructionKey), instructionObj); + } int priority = DEFAULT_FIB_FLOW_PRIORITY + prefixLength; String flowRef = getInterVpnFibFlowRef(interVpnLinkName, destination, nextHop); Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_FIB_TABLE, flowRef, priority, flowRef, 0, 0, - COOKIE_VM_FIB_TABLE, matches, instructions); + COOKIE_VM_FIB_TABLE, matches, instructionsMap); LOG.trace("Installing flow in FIB table for vpn {} interVpnLink {} nextHop {} key {}", vpnUuid, interVpnLink.getInterVpnLinkName(), nextHop, flowRef); - for (BigInteger dpId : targetDpns) { + for (Uint64 dpId : targetDpns) { LOG.debug("Installing flow: VrfEntry=[prefix={} route-paths={}] dpn {} for InterVpnLink {} in FIB", vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), @@ -745,51 +772,68 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase createLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry, int etherType) { - List returnLocalDpnId = new ArrayList<>(); + private List createLocalFibEntry(Uint32 vpnId, String rd, VrfEntry vrfEntry, int etherType) { + List returnLocalDpnId = new ArrayList<>(); String localNextHopIP = vrfEntry.getDestPrefix(); Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP); String vpnName = fibUtil.getVpnNameFromId(vpnId); if (localNextHopInfo == null) { - List usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, localNextHopIP); - List vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker, - vpnName, usedRds, localNextHopIP); - if (LOG.isDebugEnabled()) { - LOG.debug("Creating Local fib entry with vpnName {} usedRds {} localNextHopIP {} vpnExtraRoutes {}", - vpnName, usedRds, localNextHopIP, vpnExtraRoutes); - } boolean localNextHopSeen = false; - //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn - for (Routes vpnExtraRoute : vpnExtraRoutes) { - String ipPrefix; - if (isIpv4Address(vpnExtraRoute.getNexthopIpList().get(0))) { - ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX; - } else { - ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX; + List vpnExtraRoutes = null; + //Synchronized to prevent missing bucket action due to race condition between refreshFib and + // add/updateFib threads on missing nexthop in VpnToExtraroutes + // FIXME: use an Identifier structure? + final ReentrantLock lock = JvmGlobalLocks.getLockForString(localNextHopIP + FibConstants.SEPARATOR + rd); + lock.lock(); + try { + List usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, localNextHopIP); + vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker, + vpnName, usedRds, localNextHopIP); + if (LOG.isDebugEnabled()) { + LOG.debug("Creating Local fib entry with vpnName {} usedRds {} localNextHopIP {} vpnExtraRoutes {}", + vpnName, usedRds, localNextHopIP, vpnExtraRoutes); } - Prefixes localNextHopInfoLocal = fibUtil.getPrefixToInterface(vpnId, - ipPrefix); - if (localNextHopInfoLocal != null) { - localNextHopSeen = true; - BigInteger dpnId = - checkCreateLocalFibEntry(localNextHopInfoLocal, localNextHopInfoLocal.getIpAddress(), - vpnId, rd, vrfEntry, vpnExtraRoute, vpnExtraRoutes, etherType); - returnLocalDpnId.add(dpnId); + + //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn + for (Routes vpnExtraRoute : vpnExtraRoutes) { + String ipPrefix; + if (isIpv4Address(vpnExtraRoute.getNexthopIpList().get(0))) { + ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV4PREFIX; + } else { + ipPrefix = vpnExtraRoute.getNexthopIpList().get(0) + NwConstants.IPV6PREFIX; + } + Prefixes localNextHopInfoLocal = fibUtil.getPrefixToInterface(vpnId, + ipPrefix); + if (localNextHopInfoLocal != null) { + localNextHopSeen = true; + Uint64 dpnId = + checkCreateLocalFibEntry(localNextHopInfoLocal, localNextHopInfoLocal.getIpAddress(), + vpnId, rd, vrfEntry, vpnExtraRoute, vpnExtraRoutes, etherType, + /*parentVpnId*/ null); + returnLocalDpnId.add(dpnId); + } } + } finally { + lock.unlock(); } if (!localNextHopSeen && RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) { - java.util.Optional optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry); + java.util.Optional optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry); if (optionalLabel.isPresent()) { - Long label = optionalLabel.get(); + Uint32 label = optionalLabel.get(); List nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry); - synchronized (label.toString().intern()) { - LabelRouteInfo lri = getLabelRouteInfo(label); + final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label); + final ReentrantLock labelLock = lockFor(lriKey); + labelLock.lock(); + try { + LabelRouteInfo lri = getLabelRouteInfo(lriKey); + Uint32 parentVpnId = lri.getParentVpnid(); if (isPrefixAndNextHopPresentInLri(localNextHopIP, nextHopAddressList, lri)) { Optional vpnInstanceOpDataEntryOptional = fibUtil.getVpnInstanceOpData(rd); if (vpnInstanceOpDataEntryOptional.isPresent()) { String vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName(); - if (lri.getVpnInstanceList().contains(vpnInstanceName)) { + if (lri.getVpnInstanceList() != null && lri.getVpnInstanceList().contains( + vpnInstanceName)) { localNextHopInfo = updateVpnReferencesInLri(lri, vpnInstanceName, true); localNextHopIP = lri.getPrefix(); } else { @@ -801,18 +845,21 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase vpnExtraRoutes, - int etherType) { + @Nullable Routes routes, @Nullable List vpnExtraRoutes, + int etherType, Uint32 parentVpnId) { String vpnName = fibUtil.getVpnNameFromId(vpnId); if (localNextHopInfo != null) { long groupId; long localGroupId; - final BigInteger dpnId = localNextHopInfo.getDpnId(); + final Uint64 dpnId = localNextHopInfo.getDpnId(); if (Prefixes.PrefixCue.Nat.equals(localNextHopInfo.getPrefixCue())) { LOG.debug("checkCreateLocalFibEntry: NAT Prefix {} with vpnId {} rd {}. Skip local dpn {}" + " FIB processing", vrfEntry.getDestPrefix(), vpnId, rd, dpnId); @@ -852,47 +900,39 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase 1) { groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes); - localGroupId = nextHopManager.getLocalNextHopGroup(vpnId, localNextHopIP); - } else if (routes.getNexthopIpList().size() > 1) { - groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes); - localGroupId = groupId; + localGroupId = nextHopManager.getLocalSelectGroup(vpnId, vrfEntry.getDestPrefix()); } else { - groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP, - prefix, gwMacAddress); + groupId = nextHopManager.createNextHopGroups(vpnId, rd, dpnId, vrfEntry, routes, vpnExtraRoutes); localGroupId = groupId; } } else { groupId = nextHopManager.createLocalNextHop(vpnId, dpnId, interfaceName, localNextHopIP, prefix, - gwMacAddress); + gwMacAddress, parentVpnId); localGroupId = groupId; } if (groupId == FibConstants.INVALID_GROUP_ID) { LOG.error("Unable to create Group for local prefix {} on rd {} for vpninterface {} on Node {}", prefix, rd, interfaceName, dpnId.toString()); - return BigInteger.ZERO; + return Uint64.ZERO; } final List instructions = Collections.singletonList( new InstructionApplyActions( Collections.singletonList(new ActionGroup(groupId)))); final List lfibinstructions = Collections.singletonList( new InstructionApplyActions( - Arrays.asList(new ActionPopMpls(etherType), new ActionGroup(groupId)))); - java.util.Optional optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry); + Arrays.asList(new ActionPopMpls(etherType), new ActionGroup(localGroupId)))); + java.util.Optional optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry); List nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry); String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, vrfEntry.getDestPrefix()); jobCoordinator.enqueueJob(jobKey, @@ -909,15 +949,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase id = VpnHelper.getVpnToDpnListIdentifier(rd, dpnId); - Optional dpnInVpn = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id); - if (dpnInVpn.isPresent()) { - return true; + Optional dpnInVpn; + try { + dpnInVpn = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, id); + } catch (ExecutionException | InterruptedException e) { + LOG.error("isVpnPresentInDpn: Exception while reading VpnToDpnList Ds for the rd {} dpnId {}", rd, + dpnId, e); + return false; } - return false; + return dpnInVpn.isPresent(); + } + + @Nullable + private LabelRouteInfo getLabelRouteInfo(Uint32 label) { + return getLabelRouteInfo(new LabelRouteInfoKey(label)); } - private LabelRouteInfo getLabelRouteInfo(Long label) { + @Nullable + private LabelRouteInfo getLabelRouteInfo(LabelRouteInfoKey label) { InstanceIdentifier lriIid = InstanceIdentifier.builder(LabelRouteMap.class) - .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build(); - Optional opResult = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid); + .child(LabelRouteInfo.class, label).build(); + Optional opResult = null; + try { + opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, + lriIid); + } catch (ExecutionException | InterruptedException e) { + LOG.error("refreshFibTables: Exception while reading LabelRouteInfo Ds for the label {}", label, e); + return null; + } if (opResult.isPresent()) { return opResult.get(); } @@ -953,7 +1002,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase tx) { + @Nullable TypedWriteTransaction tx) { if (lri == null) { return true; } @@ -963,7 +1012,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase vpnInstancesList = lri.getVpnInstanceList() != null - ? lri.getVpnInstanceList() : new ArrayList<>(); + ? new ArrayList<>(lri.getVpnInstanceList()) : new ArrayList<>(); if (vpnInstancesList.contains(vpnInstanceName)) { LOG.debug("vpninstance {} name is present", vpnInstanceName); vpnInstancesList.remove(vpnInstanceName); @@ -984,17 +1033,17 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase tx) { List actionsInfos = Collections.singletonList(new ActionGroup(groupId)); - createTerminatingServiceActions(dpId, (int) label, actionsInfos, tx); + createTerminatingServiceActions(dpId, label, actionsInfos, tx); LOG.debug("Terminating service Entry for dpID {} : label : {} egress : {} installed successfully", dpId, label, groupId); } - public void createTerminatingServiceActions(BigInteger destDpId, int label, List actionsInfos, + public void createTerminatingServiceActions(Uint64 destDpId, Uint32 label, List actionsInfos, TypedWriteTransaction tx) { List mkMatches = new ArrayList<>(); @@ -1003,16 +1052,18 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase mkInstructions = new ArrayList<>(); mkInstructions.add(new InstructionApplyActions(actionsInfos)); FlowEntity terminatingServiceTableFlowEntity = MDSALUtil.buildFlowEntity(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, - getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), 5, - String.format("%s:%d", "TST Flow Entry ", label), - 0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, mkInstructions); + getTableMissFlowRef(destDpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), + FibConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY, + String.format("%s:%s", "TST Flow Entry ", label), 0, 0, + Uint64.valueOf(COOKIE_TUNNEL.longValue() + label.longValue()), + mkMatches, mkInstructions); FlowKey flowKey = new FlowKey(new FlowId(terminatingServiceTableFlowEntity.getFlowId())); @@ -1023,20 +1074,21 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase tx) { + private void removeTunnelTableEntry(Uint64 dpId, Uint32 label, TypedWriteTransaction tx) { FlowEntity flowEntity; LOG.debug("remove terminatingServiceActions called with DpnId = {} and label = {}", dpId, label); List mkMatches = new ArrayList<>(); // Matching metadata - mkMatches.add(new MatchTunnelId(BigInteger.valueOf(label))); + mkMatches.add(new MatchTunnelId(Uint64.valueOf(label.longValue()))); flowEntity = MDSALUtil.buildFlowEntity(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, - getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, (int) label), - 5, String.format("%s:%d", "TST Flow Entry ", label), 0, 0, - COOKIE_TUNNEL.add(BigInteger.valueOf(label)), mkMatches, null); + getTableMissFlowRef(dpId, NwConstants.INTERNAL_TUNNEL_TABLE, label), + FibConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY, + String.format("%s:%s", "TST Flow Entry ", label), 0, 0, + Uint64.valueOf(COOKIE_TUNNEL.longValue() + label.longValue()), mkMatches, null); Node nodeDpn = FibUtil.buildDpnNode(flowEntity.getDpnId()); FlowKey flowKey = new FlowKey(new FlowId(flowEntity.getFlowId())); InstanceIdentifier flowInstanceId = InstanceIdentifier.builder(Nodes.class) @@ -1047,8 +1099,8 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase deleteLocalFibEntry(Long vpnId, String rd, VrfEntry vrfEntry) { - List returnLocalDpnId = new ArrayList<>(); + public List deleteLocalFibEntry(Uint32 vpnId, String rd, VrfEntry vrfEntry) { + List returnLocalDpnId = new ArrayList<>(); Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix()); String vpnName = fibUtil.getVpnNameFromId(vpnId); boolean shouldUpdateNonEcmpLocalNextHop = true; @@ -1078,13 +1130,11 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry); + java.util.Optional optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry); if (optionalLabel.isPresent()) { - Long label = optionalLabel.get(); + Uint32 label = optionalLabel.get(); List nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry); LabelRouteInfo lri = getLabelRouteInfo(label); if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) { PrefixesBuilder prefixBuilder = new PrefixesBuilder(); prefixBuilder.setDpnId(lri.getDpnId()); - BigInteger dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), nextHopAddressList.get(0), + Uint64 dpnId = checkDeleteLocalFibEntry(prefixBuilder.build(), nextHopAddressList.get(0), vpnName, vpnId, rd, vrfEntry, shouldUpdateNonEcmpLocalNextHop); - if (!dpnId.equals(BigInteger.ZERO)) { + if (!dpnId.equals(Uint64.ZERO)) { returnLocalDpnId.add(dpnId); } } @@ -1115,9 +1165,9 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase tx) { if (tx == null) { - ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, + LoggingFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, newTx -> createRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, newTx)), LOG, "Error creating remote FIB entry"); return; } String vpnName = fibUtil.getVpnNameFromId(vpnId); - LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}", - vrfEntry.getDestPrefix(), rd, remoteDpnId); + LOG.debug("createremotefibentry: adding route {} for rd {} on remoteDpnId {}", vrfEntry.getDestPrefix(), rd, + remoteDpnId); - List adjacencyResults = baseVrfEntryHandler.resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd); - if (adjacencyResults.isEmpty()) { - LOG.error("Could not get interface for route-paths: {} in vpn {} on DPN {}", - vrfEntry.getRoutePaths(), rd, remoteDpnId); - LOG.error("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd); + if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.STATIC) { + programRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, tx); return; } - + // Handling static VRF entries List usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix()); - List vpnExtraRoutes = VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker, - vpnName, usedRds, vrfEntry.getDestPrefix()); - // create loadbalancing groups for extra routes only when the extra route is present behind - // multiple VMs - if (!vpnExtraRoutes.isEmpty() && (vpnExtraRoutes.size() > 1 - || vpnExtraRoutes.get(0).getNexthopIpList().size() > 1)) { - List instructions = new ArrayList<>(); - // Obtain the local routes for this particular dpn. - java.util.Optional routes = vpnExtraRoutes - .stream() - .filter(route -> { - Prefixes prefixToInterface = fibUtil.getPrefixToInterface(vpnId, - fibUtil.getIpPrefix(route.getNexthopIpList().get(0))); - if (prefixToInterface == null) { - return false; - } - return remoteDpnId.equals(prefixToInterface.getDpnId()); - }).findFirst(); - long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry, - routes.isPresent() ? routes.get() : null, vpnExtraRoutes); - if (groupId == FibConstants.INVALID_GROUP_ID) { - LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}", - vrfEntry.getDestPrefix(), rd, remoteDpnId.toString()); - return; - } - List actionInfos = - Collections.singletonList(new ActionGroup(groupId)); - instructions.add(new InstructionApplyActions(actionInfos)); - baseVrfEntryHandler.makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, - NwConstants.ADD_FLOW, TransactionAdapter.toWriteTransaction(tx), null); + List vpnExtraRoutes = + VpnExtraRouteHelper.getAllVpnExtraRoutes(dataBroker, vpnName, usedRds, vrfEntry.getDestPrefix()); + if (!vpnExtraRoutes.isEmpty()) { + programRemoteFibWithLoadBalancingGroups(remoteDpnId, vpnId, rd, vrfEntry, vpnExtraRoutes); } else { - baseVrfEntryHandler.programRemoteFib(remoteDpnId, vpnId, vrfEntry, - TransactionAdapter.toWriteTransaction(tx), rd, adjacencyResults, null); + // Program in case of other static VRF entries like floating IPs + programRemoteFibEntry(remoteDpnId, vpnId, rd, vrfEntry, tx); } + } + + // Allow deprecated TransactionRunner calls for now + @SuppressWarnings("ForbidCertainMethod") + private void programRemoteFibWithLoadBalancingGroups(final Uint64 remoteDpnId, final Uint32 vpnId, String rd, + final VrfEntry vrfEntry, List vpnExtraRoutes) { + // create loadbalancing groups for extra routes only when the extra route is + // present behind multiple VMs + // Obtain the local routes for this particular dpn. + java.util.Optional routes = vpnExtraRoutes.stream().filter(route -> { + Prefixes prefixToInterface = + fibUtil.getPrefixToInterface(vpnId, FibUtil.getIpPrefix(route.getNexthopIpList().get(0))); + if (prefixToInterface == null) { + return false; + } + return remoteDpnId.equals(prefixToInterface.getDpnId()); + }).findFirst(); + long groupId = nextHopManager.createNextHopGroups(vpnId, rd, remoteDpnId, vrfEntry, + routes.isPresent() ? routes.get() : null, vpnExtraRoutes); + if (groupId == FibConstants.INVALID_GROUP_ID) { + LOG.error("Unable to create Group for local prefix {} on rd {} on Node {}", vrfEntry.getDestPrefix(), rd, + remoteDpnId); + return; + } + List actionInfos = Collections.singletonList(new ActionGroup(groupId)); + List instructions = Lists.newArrayList(new InstructionApplyActions(actionInfos)); + String jobKey = FibUtil.getCreateRemoteNextHopJobKey(vpnId, remoteDpnId, vrfEntry.getDestPrefix()); + jobCoordinator.enqueueJob(jobKey, + () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(txn -> { + baseVrfEntryHandler.makeConnectedRoute(remoteDpnId, vpnId, vrfEntry, rd, instructions, + NwConstants.ADD_FLOW, txn, null); + }))); LOG.debug("Successfully added FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId); } - protected void cleanUpOpDataForFib(Long vpnId, String primaryRd, final VrfEntry vrfEntry) { + private void programRemoteFibEntry(final Uint64 remoteDpnId, final Uint32 vpnId, String rd, + final VrfEntry vrfEntry, TypedWriteTransaction tx) { + List adjacencyResults = baseVrfEntryHandler.resolveAdjacency(remoteDpnId, vpnId, vrfEntry, rd); + if (adjacencyResults.isEmpty()) { + LOG.error("Could not get interface for route-paths: {} in vpn {} on DPN {}", vrfEntry.getRoutePaths(), rd, + remoteDpnId); + LOG.error("Failed to add Route: {} in vpn: {}", vrfEntry.getDestPrefix(), rd); + return; + } + baseVrfEntryHandler.programRemoteFib(remoteDpnId, vpnId, vrfEntry, TransactionAdapter.toWriteTransaction(tx), + rd, adjacencyResults, null); + LOG.debug("Successfully programmed FIB entry for prefix {} in vpnId {}", vrfEntry.getDestPrefix(), vpnId); + } + + protected void cleanUpOpDataForFib(Uint32 vpnId, String primaryRd, final VrfEntry vrfEntry) { /* Get interface info from prefix to interface mapping; Use the interface info to get the corresponding vpn interface op DS entry, remove the adjacency corresponding to this fib entry. @@ -1240,7 +1307,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnId, vrfEntry.getDestPrefix()); String usedRd = usedRds.isEmpty() ? primaryRd : usedRds.get(0); Routes extraRoute = baseVrfEntryHandler.getVpnToExtraroute(vpnId, usedRd, vrfEntry.getDestPrefix()); - if (extraRoute != null) { + if (extraRoute != null && extraRoute.getNexthopIpList() != null) { for (String nextHopIp : extraRoute.getNexthopIpList()) { LOG.debug("NextHop IP for destination {} is {}", vrfEntry.getDestPrefix(), nextHopIp); if (nextHopIp != null) { @@ -1256,9 +1323,9 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry); + java.util.Optional optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry); if (optionalLabel.isPresent()) { - Long label = optionalLabel.get(); + Uint32 label = optionalLabel.get(); List nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry); LabelRouteInfo lri = getLabelRouteInfo(label); if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) { @@ -1278,8 +1345,8 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase>> { + private class CleanupVpnInterfaceWorker implements Callable>> { Prefixes prefixInfo; - Long vpnId; + Uint32 vpnId; String rd; VrfEntry vrfEntry; Routes extraRoute; - CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Long vpnId, final String rd, + CleanupVpnInterfaceWorker(final Prefixes prefixInfo, final Uint32 vpnId, final String rd, final VrfEntry vrfEntry, final Routes extraRoute) { this.prefixInfo = prefixInfo; this.vpnId = vpnId; @@ -1325,9 +1392,12 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase { List nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry); - synchronized (label.toString().intern()) { - LabelRouteInfo lri = getLabelRouteInfo(label); - if (lri != null && lri.getPrefix().equals(vrfEntry.getDestPrefix()) + final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label); + final ReentrantLock lock = lockFor(lriKey); + lock.lock(); + try { + LabelRouteInfo lri = getLabelRouteInfo(lriKey); + if (lri != null && Objects.equals(lri.getPrefix(), vrfEntry.getDestPrefix()) && nextHopAddressList.contains(lri.getNextHopIpList().get(0))) { Optional vpnInstanceOpDataEntryOptional = fibUtil.getVpnInstanceOpData(rd); @@ -1345,6 +1415,8 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase opVpnInterface = tx .read(FibUtil.getVpnInterfaceOpDataEntryIdentifier(ifName, vpnName)).get(); if (opVpnInterface.isPresent()) { - long associatedVpnId = fibUtil.getVpnId(vpnName); - if (vpnId != associatedVpnId) { + Uint32 associatedVpnId = fibUtil.getVpnId(vpnName); + if (!Objects.equals(vpnId, associatedVpnId)) { LOG.warn("Prefixes {} are associated with different vpn instance with id {} rather than {}", vrfEntry.getDestPrefix(), associatedVpnId, vpnId); LOG.warn("Not proceeding with Cleanup op data for prefix {}", vrfEntry.getDestPrefix()); @@ -1404,6 +1476,8 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase tx) throws ExecutionException, InterruptedException { @@ -1425,8 +1499,10 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase + @NonNull List adjacencies + = new ArrayList(optAdjacencies.get().nonnullAdjacency().values()); + if (adjacencies.size() <= 2 + && adjacencies.stream().allMatch(adjacency -> adjacency.getAdjacencyType() == Adjacency.AdjacencyType.PrimaryAdjacency && adjacency.isMarkedForDeletion() != null && adjacency.isMarkedForDeletion() @@ -1444,49 +1520,54 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase vpnToDpnList; + final Map keyVpnToDpnListMap; if (vrfEntry.getParentVpnRd() != null && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) { // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries. VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd()); - vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() : + keyVpnToDpnListMap = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() : vpnInstance.getVpnToDpnList(); LOG.info("deleteFibEntries: Processing deletion of PNF FIB entry with rd {} prefix {}", vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix()); } else { - vpnToDpnList = vpnInstance.getVpnToDpnList(); + keyVpnToDpnListMap = vpnInstance.getVpnToDpnList(); } SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class); - final java.util.Optional optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry); + final java.util.Optional optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry); List nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry); String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId()); if (subnetRoute != null) { - long elanTag = subnetRoute.getElantag(); + long elanTag = subnetRoute.getElantag().toJava(); LOG.trace("SUBNETROUTE: deleteFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}" + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag); - if (vpnToDpnList != null) { + if (keyVpnToDpnListMap != null) { jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()), () -> Collections.singletonList( txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> { - for (final VpnToDpnList curDpn : vpnToDpnList) { + for (final VpnToDpnList curDpn : keyVpnToDpnListMap.values()) { - baseVrfEntryHandler.makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), - vrfEntry, vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, - TransactionAdapter.toWriteTransaction(tx), null); + baseVrfEntryHandler.makeConnectedRoute(curDpn.getDpnId(), + vpnInstance.getVpnId(), + vrfEntry, vrfTableKey.getRouteDistinguisher(), null, + NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null); if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) { - optionalLabel.ifPresent(label -> makeLFibTableEntry(curDpn.getDpnId(), label, null, - DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx)); + optionalLabel.ifPresent(label -> makeLFibTableEntry(curDpn.getDpnId(), + label, null, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx)); } - installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnInstance.getVpnId(), + installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, + vpnInstance.getVpnId(), vrfEntry, NwConstants.DEL_FLOW, tx); } }))); } optionalLabel.ifPresent(label -> { - synchronized (label.toString().intern()) { - LabelRouteInfo lri = getLabelRouteInfo(label); + final LabelRouteInfoKey lriKey = new LabelRouteInfoKey(label); + final ReentrantLock lock = lockFor(lriKey); + lock.lock(); + try { + LabelRouteInfo lri = getLabelRouteInfo(lriKey); if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) { Optional vpnInstanceOpDataEntryOptional = fibUtil.getVpnInstanceOpData(rd); @@ -1508,21 +1589,22 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(), + final List localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry); - if (vpnToDpnList != null) { + if (keyVpnToDpnListMap != null) { List usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(), vrfEntry.getDestPrefix()); String jobKey; Optional extraRouteOptional; //Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn if (usedRds != null && !usedRds.isEmpty()) { - jobKey = FibUtil.getJobKeyForRdPrefix(usedRds.get(0), vrfEntry.getDestPrefix()); if (usedRds.size() > 1) { LOG.error("The extra route prefix is still present in some DPNs"); return ; @@ -1533,29 +1615,35 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> { if (localDpnIdList.size() <= 0) { - for (VpnToDpnList curDpn : vpnToDpnList) { - baseVrfEntryHandler.deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), + for (VpnToDpnList curDpn : keyVpnToDpnListMap.values()) { + baseVrfEntryHandler.deleteRemoteRoute(Uint64.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, - TransactionAdapter.toWriteTransaction(tx)); + TransactionAdapter.toWriteTransaction(tx)); } } else { - for (BigInteger localDpnId : localDpnIdList) { - for (VpnToDpnList curDpn : vpnToDpnList) { - if (!curDpn.getDpnId().equals(localDpnId)) { + for (Uint64 localDpnId : localDpnIdList) { + for (VpnToDpnList curDpn : keyVpnToDpnListMap.values()) { + if (!Objects.equals(curDpn.getDpnId(), localDpnId)) { baseVrfEntryHandler.deleteRemoteRoute(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, - TransactionAdapter.toWriteTransaction(tx)); + TransactionAdapter.toWriteTransaction(tx)); } } } } + if (extraRouteOptional.isPresent()) { + //Remove select groups only for extra-routes + nextHopManager.removeNextHopPointer(nextHopManager + .getRemoteSelectGroupKey(vpnInstance.getVpnId(), vrfEntry.getDestPrefix())); + nextHopManager.removeNextHopPointer(nextHopManager + .getLocalSelectGroupKey(vpnInstance.getVpnId(), vrfEntry.getDestPrefix())); + } })), MAX_RETRIES); } @@ -1584,10 +1672,10 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase instructions, int priority, - int addOrRemove, TypedWriteTransaction tx) { + private void makeLFibTableEntry(Uint64 dpId, Uint32 label, @Nullable List instructions, + int priority, int addOrRemove, TypedWriteTransaction tx) { if (tx == null) { - ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, + LoggingFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, newTx -> makeLFibTableEntry(dpId, label, instructions, priority, addOrRemove, newTx)), LOG, "Error making LFIB table entry"); return; @@ -1595,7 +1683,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase matches = new ArrayList<>(); matches.add(MatchEthernetType.MPLS_UNICAST); - matches.add(new MatchMplsLabel(label)); + matches.add(new MatchMplsLabel(label.longValue())); // Install the flow entry in L3_LFIB_TABLE String flowRef = FibUtil.getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, label, priority); @@ -1612,7 +1700,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase> callback) { LOG.trace("New dpn {} for vpn {} : populateFibOnNewDpn", dpnId, rd); jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId), @@ -1639,12 +1727,15 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase { - for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) { + for (final VrfEntry vrfEntry : vrfTable.get().nonnullVrfEntry().values()) { SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class); if (subnetRoute != null) { - long elanTag = subnetRoute.getElantag(); + long elanTag = subnetRoute.getElantag().toJava(); installSubnetRouteInFib(dpnId, elanTag, rd, vpnId, vrfEntry, tx); installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry, NwConstants.ADD_FLOW, tx); @@ -1661,12 +1752,12 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry); + java.util.Optional optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry); if (optionalLabel.isPresent()) { List nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry); LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get()); if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri)) { - if (lri.getDpnId().equals(dpnId)) { + if (Objects.equals(lri.getDpnId(), dpnId)) { try { int etherType = NWUtil.getEtherTypeFromIpPrefix( vrfEntry.getDestPrefix()); @@ -1700,39 +1791,51 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase> listenableFuture = Futures.allAsList(futures); Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor()); } + } finally { + lock.unlock(); } return futures; }); } - public void populateExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd, + public void populateExternalRoutesOnDpn(final Uint64 dpnId, final Uint32 vpnId, final String rd, final String localNextHopIp, final String remoteNextHopIp) { LOG.trace("populateExternalRoutesOnDpn : dpn {}, vpn {}, rd {}, localNexthopIp {} , remoteNextHopIp {} ", dpnId, vpnId, rd, localNextHopIp, remoteNextHopIp); InstanceIdentifier id = buildVrfId(rd); final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd); List txnObjects = new ArrayList<>(); - final Optional vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id); + final Optional vrfTable; + try { + vrfTable = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION, id); + } catch (ExecutionException | InterruptedException e) { + LOG.error("populateExternalRoutesOnDpn: Exception while reading the VrfTable for the rd {}", rd, e); + return; + } if (vrfTable.isPresent()) { jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId), () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> { - synchronized (vpnInstance.getVpnInstanceName().intern()) { - vrfTable.get().getVrfEntry().stream() + final ReentrantLock lock = lockFor(vpnInstance); + lock.lock(); + try { + vrfTable.get().nonnullVrfEntry().values().stream() .filter(vrfEntry -> RouteOrigin.BGP == RouteOrigin.value(vrfEntry.getOrigin())) .forEach(bgpRouteVrfEntryHandler.getConsumerForCreatingRemoteFib(dpnId, vpnId, rd, remoteNextHopIp, vrfTable, TransactionAdapter.toWriteTransaction(tx), txnObjects)); + } finally { + lock.unlock(); } }))); } } public void manageRemoteRouteOnDPN(final boolean action, - final BigInteger localDpnId, - final long vpnId, + final Uint64 localDpnId, + final Uint32 vpnId, final String rd, final String destPrefix, final String destTepIp, - final long label) { + final Uint32 label) { final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd); if (vpnInstance == null) { @@ -1742,7 +1845,9 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> { - synchronized (vpnInstance.getVpnInstanceName().intern()) { + final ReentrantLock lock = lockFor(vpnInstance); + lock.lock(); + try { VrfTablesKey vrfTablesKey = new VrfTablesKey(rd); VrfEntry vrfEntry = getVrfEntry(dataBroker, rd, destPrefix); if (vrfEntry == null) { @@ -1750,9 +1855,9 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase routePathList = vrfEntry.getRoutePaths(); + Map keyRoutePathsMap = vrfEntry.getRoutePaths(); VrfEntry modVrfEntry; - if (routePathList == null || routePathList.isEmpty()) { + if (keyRoutePathsMap == null || keyRoutePathsMap.isEmpty()) { modVrfEntry = FibHelper.getVrfEntryBuilder(vrfEntry, label, Collections.singletonList(destTepIp), RouteOrigin.value(vrfEntry.getOrigin()), null /* parentVpnRd */).build(); @@ -1766,16 +1871,16 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(), - vrfEntry.getDestPrefix()); + List usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, + vpnInstance.getVpnId(), vrfEntry.getDestPrefix()); if (usedRds.size() > 1) { LOG.debug("The extra route prefix is still present in some DPNs"); return; } //Is this fib route an extra route? If yes, get the nexthop which would be //an adjacency in the vpn - Optional extraRouteOptional = Optional.absent(); - if (usedRds.size() != 0) { + Optional extraRouteOptional = Optional.empty(); + if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.STATIC && usedRds.size() != 0) { extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, fibUtil.getVpnNameFromId(vpnInstance.getVpnId()), usedRds.get(0), vrfEntry.getDestPrefix()); @@ -1783,121 +1888,134 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase> callback) { LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd); jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId), () -> { InstanceIdentifier id = buildVrfId(rd); final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd); - List txnObjects = new ArrayList<>(); + List txnObjects = new ArrayList<>(); final Optional vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id); List> futures = new ArrayList<>(); - if (vrfTable.isPresent()) { - synchronized (vpnInstance.getVpnInstanceName().intern()) { - futures.add(retryingTxRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> { - String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId()); - for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) { - /* Handle subnet routes here */ - SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class); - if (subnetRoute != null && !fibUtil - .isInterfacePresentInDpn(vrfEntry.getParentVpnRd(), dpnId)) { - LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Cleaning subnetroute {} on dpn {}" - + " for vpn {}", vrfEntry.getDestPrefix(), dpnId, rd); - baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null, - NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null); - List routePaths = vrfEntry.getRoutePaths(); - if (routePaths != null) { - for (RoutePaths routePath : routePaths) { - makeLFibTableEntry(dpnId, routePath.getLabel(), null, - DEFAULT_FIB_FLOW_PRIORITY, - NwConstants.DEL_FLOW, tx); - LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Released subnetroute label {}" - + " for rd {} prefix {}", routePath.getLabel(), rd, - vrfEntry.getDestPrefix()); - } + if (!vrfTable.isPresent()) { + LOG.error("cleanUpDpnForVpn: VRF Table not available for RD {}", rd); + if (callback != null) { + ListenableFuture> listenableFuture = Futures.allAsList(futures); + Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor()); + } + return futures; + } + final ReentrantLock lock = lockFor(vpnInstance); + lock.lock(); + try { + futures.add(retryingTxRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> { + String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId()); + for (final VrfEntry vrfEntry : vrfTable.get().nonnullVrfEntry().values()) { + /* parentRd is only filled for external PNF cases where the interface on the external + * network VPN are used to cleanup the flows. For all other cases, use "rd" for + * #fibUtil.isInterfacePresentInDpn(). + * */ + String parentRd = vrfEntry.getParentVpnRd() != null ? vrfEntry.getParentVpnRd() + : rd; + /* Handle subnet routes here */ + SubnetRoute subnetRoute = vrfEntry.augmentation(SubnetRoute.class); + if (subnetRoute != null && !fibUtil + .isInterfacePresentInDpn(parentRd, dpnId)) { + LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Cleaning subnetroute {} on dpn {}" + + " for vpn {}", vrfEntry.getDestPrefix(), dpnId, rd); + baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null, + NwConstants.DEL_FLOW, TransactionAdapter.toWriteTransaction(tx), null); + Map keyRoutePathsMap = vrfEntry.getRoutePaths(); + if (keyRoutePathsMap != null) { + for (RoutePaths routePath : keyRoutePathsMap.values()) { + makeLFibTableEntry(dpnId, routePath.getLabel(), null, + DEFAULT_FIB_FLOW_PRIORITY, + NwConstants.DEL_FLOW, tx); + LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Released subnetroute label {}" + + " for rd {} prefix {}", routePath.getLabel(), rd, + vrfEntry.getDestPrefix()); } - installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry, - NwConstants.DEL_FLOW, tx); - continue; } - // ping responder for router interfaces - RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class); - if (routerInt != null) { - LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}", - rd, routerInt.getUuid(), routerInt.getIpAddress(), - routerInt.getMacAddress()); - routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId, - routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()), - NwConstants.DEL_FLOW); - continue; - } - - //Handle local flow deletion for imports - if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) { - java.util.Optional optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry); - if (optionalLabel.isPresent()) { - List nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry); - LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get()); - if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, - lri) && lri.getDpnId().equals(dpnId)) { - deleteLocalFibEntry(vpnId, rd, vrfEntry); - } + installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry, + NwConstants.DEL_FLOW, tx); + continue; + } + // ping responder for router interfaces + RouterInterface routerInt = vrfEntry.augmentation(RouterInterface.class); + if (routerInt != null) { + LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}", + rd, routerInt.getUuid(), routerInt.getIpAddress(), + routerInt.getMacAddress()); + routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId, + routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()), + NwConstants.DEL_FLOW); + continue; + } + //Handle local flow deletion for imports + if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) { + java.util.Optional optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry); + if (optionalLabel.isPresent()) { + List nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry); + LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get()); + if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, + lri) && Objects.equals(lri.getDpnId(), dpnId)) { + deleteLocalFibEntry(vpnId, rd, vrfEntry); } } - - // Passing null as we don't know the dpn - // to which prefix is attached at this point - List usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, - vpnInstance.getVpnId(), vrfEntry.getDestPrefix()); - Optional extraRouteOptional; - //Is this fib route an extra route? If yes, get the nexthop which would be - //an adjacency in the vpn - if (usedRds != null && !usedRds.isEmpty()) { - if (usedRds.size() > 1) { - LOG.error("The extra route prefix is still present in some DPNs"); - return; - } else { - extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName, - usedRds.get(0), vrfEntry.getDestPrefix()); - - } + } + // Passing null as we don't know the dpn + // to which prefix is attached at this point + List usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, + vpnInstance.getVpnId(), vrfEntry.getDestPrefix()); + Optional extraRouteOptional; + //Is this fib route an extra route? If yes, get the nexthop which would be + //an adjacency in the vpn + if (usedRds != null && !usedRds.isEmpty()) { + if (usedRds.size() > 1) { + LOG.error("The extra route prefix is still present in some DPNs"); + return; } else { - extraRouteOptional = Optional.absent(); + extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName, + usedRds.get(0), vrfEntry.getDestPrefix()); + } - if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) { - bgpRouteVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId, + } else { + extraRouteOptional = Optional.empty(); + } + if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) { + bgpRouteVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().key(), vrfEntry, extraRouteOptional, TransactionAdapter.toWriteTransaction(tx), txnObjects); - } else { - if (subnetRoute == null || !fibUtil - .isInterfacePresentInDpn(vrfEntry.getParentVpnRd(), dpnId)) { - baseVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId, + } else { + if (subnetRoute == null || !fibUtil + .isInterfacePresentInDpn(parentRd, dpnId)) { + baseVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().key(), vrfEntry, extraRouteOptional, TransactionAdapter.toWriteTransaction(tx)); - } } } - })); - } - if (callback != null) { - ListenableFuture> listenableFuture = Futures.allAsList(futures); - Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor()); - } - } else { - LOG.error("cleanUpDpnForVpn: No vrf table found for rd {} vpnId {} dpn {}", rd, vpnId, dpnId); + } + })); + } finally { + lock.unlock(); + } + if (callback != null) { + ListenableFuture> listenableFuture = Futures.allAsList(futures); + Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor()); } return futures; }); - } - public void cleanUpExternalRoutesOnDpn(final BigInteger dpnId, final long vpnId, final String rd, + public void cleanUpExternalRoutesOnDpn(final Uint64 dpnId, final Uint32 vpnId, final String rd, final String localNextHopIp, final String remoteNextHopIp) { LOG.trace("cleanUpExternalRoutesOnDpn : cleanup remote routes on dpn {} for vpn {}, rd {}, " + " localNexthopIp {} , remoteNexhtHopIp {}", @@ -1905,18 +2023,28 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase id = buildVrfId(rd); final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd); List txnObjects = new ArrayList<>(); - final Optional vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id); + final Optional vrfTable; + try { + vrfTable = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION, id); + } catch (ExecutionException | InterruptedException e) { + LOG.error("getVrfEntry: Exception while reading VrfTable for the rd {} vpnId {}", rd, vpnId, e); + return; + } if (vrfTable.isPresent()) { jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId), () -> { - synchronized (vpnInstance.getVpnInstanceName().intern()) { + final ReentrantLock lock = lockFor(vpnInstance); + lock.lock(); + try { return Collections.singletonList( txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, - tx -> vrfTable.get().getVrfEntry().stream() + tx -> vrfTable.get().nonnullVrfEntry().values().stream() .filter(vrfEntry -> RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) .forEach(bgpRouteVrfEntryHandler.getConsumerForDeletingRemoteFib(dpnId, vpnId, remoteNextHopIp, vrfTable, TransactionAdapter.toWriteTransaction(tx), - txnObjects)))); + txnObjects)))); + } finally { + lock.unlock(); } }); } @@ -1933,16 +2061,24 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase vrfEntryId = InstanceIdentifier.builder(FibEntries.class) .child(VrfTables.class, new VrfTablesKey(rd)) .child(VrfEntry.class, new VrfEntryKey(ipPrefix)).build(); - Optional vrfEntry = MDSALUtil.read(broker, LogicalDatastoreType.CONFIGURATION, vrfEntryId); + Optional vrfEntry; + try { + vrfEntry = SingleTransactionDataBroker.syncReadOptional(broker, LogicalDatastoreType.CONFIGURATION, + vrfEntryId); + } catch (ExecutionException | InterruptedException e) { + LOG.error("getVrfEntry: Exception while reading VrfEntry for the prefix {} rd {}", ipPrefix, rd, e); + return null; + } if (vrfEntry.isPresent()) { return vrfEntry.get(); } @@ -1955,7 +2091,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName); + List targetDpns = interVpnLink.getEndpointDpnsByVpnName(vpnName); if (targetDpns.isEmpty()) { LOG.warn("Could not find DPNs for VPN {} in InterVpnLink {}", vpnName, interVpnLinkName); @@ -1963,7 +2099,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase optNextHop = FibUtil.getFirstNextHopAddress(vrfEntry); - java.util.Optional optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry); + java.util.Optional optLabel = FibUtil.getLabelFromRoutePaths(vrfEntry); // delete from FIB // @@ -1974,7 +2110,7 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase { LOG.trace("Removing flow in FIB table for interVpnLink {}", interVpnLinkName); - ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> { - for (BigInteger dpId : targetDpns) { + LoggingFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> { + for (Uint64 dpId : targetDpns) { LOG.debug("Removing flow: VrfEntry=[prefix={} label={}] dpn {} for InterVpnLink {} in LFIB", vrfEntry.getDestPrefix(), label, dpId, interVpnLinkName); - makeLFibTableEntry(dpId, label, /*instructions*/null, LFIB_INTERVPN_PRIORITY, NwConstants.DEL_FLOW, - tx); + makeLFibTableEntry(dpId, label, /*instructions*/null, LFIB_INTERVPN_PRIORITY, + NwConstants.DEL_FLOW, tx); } }), LOG, "Error removing flows"); }); } - private boolean isPrefixAndNextHopPresentInLri(String prefix, + private static boolean isPrefixAndNextHopPresentInLri(String prefix, List nextHopAddressList, LabelRouteInfo lri) { - return lri != null && lri.getPrefix().equals(prefix) + return lri != null && Objects.equals(lri.getPrefix(), prefix) && nextHopAddressList.contains(lri.getNextHopIpList().get(0)); } - private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Long vpnId, VrfEntry vrfEntry, BigInteger dpnId) { + private boolean shouldCreateFibEntryForVrfAndVpnIdOnDpn(Uint32 vpnId, VrfEntry vrfEntry, Uint64 dpnId) { if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.BGP) { return true; } Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, vrfEntry.getDestPrefix()); if (prefix != null) { - BigInteger prefixDpnId = prefix.getDpnId(); + Uint64 prefixDpnId = prefix.getDpnId(); if (dpnId.equals(prefixDpnId)) { LOG.trace("Should not create remote FIB entry for vrfEntry {} on DPN {}", vrfEntry, dpnId); @@ -2020,4 +2156,13 @@ public class VrfEntryListener extends AsyncDataTreeChangeListenerBase