From 6282ebd1f91fc59c28537f33f68b2e1fd74af88c Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 4 Jul 2018 15:19:22 +0530 Subject: [PATCH] ECMP UC2 Enhancement: Support for more than two DC-GWs Following changes are done as part of this enhancement: 1. Creates nc2 load balancing groups with each group having two buckets pointing towards DC-GWs. Quagga bgp will only advertise two routes and based on the routes advertised, table 21 will point to one of the groups. 2. Marks buckets down when external tunnel end point gets deleted. 3. Yang changes to store DC-GWs. Note: After this change, DC-GW load balancing gropus will not be deleted forever, which is fine because Neighbhor add/delete are initial operations and will not be performed frequently. Following are the scenarios which will end up having stale groups: a) When tunnel is deleted and BGP neighbor comes up with different tunnel ip. b) When tunnel and BGP neighbor both are deleted. Change-Id: I74b1025751b58a7d0d19be19e2edd13c8205bde7 Signed-off-by: Ankit Jain Signed-off-by: Amitesh Soni --- .../bgpmanager/BgpConfigurationManager.java | 5 +- .../netvirt/bgpmanager/BgpUtil.java | 29 +-- .../netvirt/fibmanager/api/IFibManager.java | 4 +- .../netvirt/fibmanager/FibManagerImpl.java | 4 +- .../netvirt/fibmanager/FibUtil.java | 68 ++--- .../netvirt/fibmanager/NexthopManager.java | 232 +++++++++++------- vpnmanager/api/src/main/yang/odl-l3vpn.yang | 12 +- .../TunnelInterfaceStateListener.java | 36 +-- 8 files changed, 206 insertions(+), 184 deletions(-) diff --git a/bgpmanager/impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpConfigurationManager.java b/bgpmanager/impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpConfigurationManager.java index d0b16e121d..0eca8f01fe 100755 --- a/bgpmanager/impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpConfigurationManager.java +++ b/bgpmanager/impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpConfigurationManager.java @@ -2084,7 +2084,7 @@ public class BgpConfigurationManager { return; } tepIpList.forEach(tepIp -> { - bgpUtil.removeOrUpdateLBGroups(tepIp, NwConstants.MOD_FLOW, false); + bgpUtil.removeOrUpdateLBGroups(tepIp, NwConstants.MOD_FLOW); }); } @@ -2095,7 +2095,7 @@ public class BgpConfigurationManager { return; } tepIpList.forEach(tepIp -> { - bgpUtil.removeOrUpdateLBGroups(tepIp, NwConstants.MOD_FLOW, true); + bgpUtil.removeOrUpdateLBGroups(tepIp, NwConstants.MOD_FLOW); }); } @@ -2549,6 +2549,7 @@ public class BgpConfigurationManager { DcgwTep dto = new DcgwTepBuilder().setDcGwIp(dcgwIp).setTepIps(tepList) .build(); update(iid, dto); + bgpUtil.removeOrUpdateLBGroups(tepIp,NwConstants.MOD_FLOW); } public void addLogging(String fileName, String logLevel) { diff --git a/bgpmanager/impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpUtil.java b/bgpmanager/impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpUtil.java index 6f61b3a670..67c9ec9086 100755 --- a/bgpmanager/impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpUtil.java +++ b/bgpmanager/impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpUtil.java @@ -7,14 +7,12 @@ */ package org.opendaylight.netvirt.bgpmanager; -import static java.util.stream.Collectors.toList; - import com.google.common.base.Optional; + import java.net.Inet4Address; import java.net.Inet6Address; import java.net.InetAddress; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; @@ -52,7 +50,6 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre; import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.DpnEndpoints; import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.dpn.endpoints.DPNTEPsInfo; -import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rev160406.DcGatewayIpList; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInstances; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceKey; @@ -377,25 +374,7 @@ public class BgpUtil implements AutoCloseable { } } - private List getDcGwIps() { - InstanceIdentifier dcGatewayIpListid = - InstanceIdentifier.builder(DcGatewayIpList.class).build(); - DcGatewayIpList dcGatewayIpListConfig = - MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, dcGatewayIpListid).orNull(); - if (dcGatewayIpListConfig == null) { - return Collections.EMPTY_LIST; - } - return dcGatewayIpListConfig.getDcGatewayIp() - .stream() - .filter(dcGwIp -> dcGwIp.getTunnnelType().equals(TunnelTypeMplsOverGre.class)) - .map(dcGwIp -> String.valueOf(dcGwIp.getIpAddress().getIpv4Address())).sorted() - .collect(toList()); - } - - - public void removeOrUpdateLBGroups(String tepIp, int addRemoveOrUpdate, boolean isTunnelUp) { - LOG.debug("removing bucket towards DCGW {}", tepIp); - List availableDcGws = getDcGwIps(); + public void removeOrUpdateLBGroups(String tepIp, int addRemoveOrUpdate) { getDpnTEPsInfos(dataBroker).forEach(dpnInfo -> { if (NwConstants.MOD_FLOW == addRemoveOrUpdate) { LOG.debug("Updating bucket in DPN {}", dpnInfo.getDPNID()); @@ -403,8 +382,8 @@ public class BgpUtil implements AutoCloseable { LOG.debug("Deleting groups in DPN {}", dpnInfo.getDPNID()); } Class tunType = TunnelTypeMplsOverGre.class; - fibManager.programDcGwLoadBalancingGroup(availableDcGws, dpnInfo.getDPNID(), - tepIp, addRemoveOrUpdate, isTunnelUp, tunType); + fibManager.programDcGwLoadBalancingGroup(dpnInfo.getDPNID(), + tepIp, addRemoveOrUpdate, false, tunType); }); } } diff --git a/fibmanager/api/src/main/java/org/opendaylight/netvirt/fibmanager/api/IFibManager.java b/fibmanager/api/src/main/java/org/opendaylight/netvirt/fibmanager/api/IFibManager.java index d2f7200559..de42f30588 100644 --- a/fibmanager/api/src/main/java/org/opendaylight/netvirt/fibmanager/api/IFibManager.java +++ b/fibmanager/api/src/main/java/org/opendaylight/netvirt/fibmanager/api/IFibManager.java @@ -79,8 +79,8 @@ public interface IFibManager { boolean isVpnFirstEndPoint, VrfEntry vrfEntry); - void programDcGwLoadBalancingGroup(List availableDcGws, BigInteger dpnId, String destinationIp, - int addRemoveOrUpdate, boolean isTunnelUp, + void programDcGwLoadBalancingGroup(BigInteger dpnId, + String destinationIp, int addRemoveOrUpdate, boolean isTunnelUp, Class tunnelType); void refreshVrfEntry(String rd, String prefix); diff --git a/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/FibManagerImpl.java b/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/FibManagerImpl.java index 0d1515cc14..91d6f84b44 100755 --- a/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/FibManagerImpl.java +++ b/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/FibManagerImpl.java @@ -200,10 +200,10 @@ public class FibManagerImpl implements IFibManager { } @Override - public void programDcGwLoadBalancingGroup(List availableDcGws, BigInteger dpnId, String destinationIp, + public void programDcGwLoadBalancingGroup(BigInteger dpnId, String destinationIp, int addRemoveOrUpdate, boolean isTunnelUp, Class tunnelType) { - nexthopManager.programDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp, + nexthopManager.programDcGwLoadBalancingGroup(dpnId, destinationIp, addRemoveOrUpdate, isTunnelUp, tunnelType); } diff --git a/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/FibUtil.java b/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/FibUtil.java index 6314d5e174..7d134c115b 100644 --- a/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/FibUtil.java +++ b/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/FibUtil.java @@ -92,15 +92,15 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev15033 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.DpidL3vpnLbNexthops; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.L3vpnDcGws; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.L3vpnLbNexthops; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnIdToVpnInstance; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceToVpnId; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.dpid.l3vpn.lb.nexthops.DpnLbNexthops; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.dpid.l3vpn.lb.nexthops.DpnLbNexthopsBuilder; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.dpid.l3vpn.lb.nexthops.DpnLbNexthopsKey; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.dc.gws.DcGateway; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.dc.gws.DcGatewayBuilder; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.dc.gws.DcGatewayKey; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.lb.nexthops.Nexthops; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.lb.nexthops.NexthopsBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.lb.nexthops.NexthopsKey; @@ -681,19 +681,40 @@ public class FibUtil { return "gre-" + availableDcGws.stream().sorted().collect(joining(":")); } - public static void updateLbGroupInfo(BigInteger dpnId, String destinationIp, String groupIdKey, + public static void updateLbGroupInfo(BigInteger dpnId, String groupIdKey, String groupId, TypedWriteTransaction tx) { - InstanceIdentifier id = getDpnLbNexthopsIdentifier(dpnId, destinationIp); - DpnLbNexthops dpnToLbNextHop = buildDpnLbNextHops(dpnId, destinationIp, groupIdKey); - tx.merge(id, dpnToLbNextHop); InstanceIdentifier nextHopsId = getNextHopsIdentifier(groupIdKey); Nexthops nextHopsToGroupId = buildNextHops(dpnId, groupIdKey, groupId); tx.merge(nextHopsId, nextHopsToGroupId); } - public static void removeDpnIdToNextHopInfo(String destinationIp, BigInteger dpnId, - TypedWriteTransaction tx) { - tx.delete(getDpnLbNexthopsIdentifier(dpnId, destinationIp)); + public static void removeL3vpnDcGateWay(String destinationIp, TypedReadWriteTransaction tx) + throws InterruptedException, ExecutionException { + InstanceIdentifier dcGateWayId = getDcGwInstance(destinationIp); + Optional dcGateWayOpt = tx.read(dcGateWayId).get(); + if (!dcGateWayOpt.isPresent()) { + return; + } + tx.delete(dcGateWayId); + } + + public static void addL3vpnDcGateWay(String destinationIp, TypedReadWriteTransaction tx) + throws InterruptedException, ExecutionException { + InstanceIdentifier dcGateWayId = getDcGwInstance(destinationIp); + Optional dcGateWayOpt = tx.read(dcGateWayId).get(); + if (!dcGateWayOpt.isPresent()) { + tx.put(dcGateWayId, + new DcGatewayBuilder() + .withKey(new DcGatewayKey(destinationIp)) + .setIpAddress(destinationIp).build() + ); + } + } + + private static InstanceIdentifier getDcGwInstance(String destinationIp) { + return InstanceIdentifier.builder(L3vpnDcGws.class) + .child(DcGateway.class, new DcGatewayKey(destinationIp)) + .build(); } public static void removeOrUpdateNextHopInfo(BigInteger dpnId, String nextHopKey, String groupId, @@ -713,13 +734,6 @@ public class FibUtil { } } - private static InstanceIdentifier getDpnLbNexthopsIdentifier(BigInteger dpnId, - String destinationIp) { - return InstanceIdentifier.builder(DpidL3vpnLbNexthops.class) - .child(DpnLbNexthops.class, new DpnLbNexthopsKey(destinationIp, dpnId)) - .build(); - } - private static InstanceIdentifier getNextHopsIdentifier(String groupIdKey) { return InstanceIdentifier.builder(L3vpnLbNexthops.class) .child(Nexthops.class, new NexthopsKey(groupIdKey)).build(); @@ -732,24 +746,20 @@ public class FibUtil { .setTargetDeviceId(Collections.singletonList(dpnId.toString())).build(); } - private static DpnLbNexthops buildDpnLbNextHops(BigInteger dpnId, String destinationIp, - String groupIdKey) { - return new DpnLbNexthopsBuilder().withKey(new DpnLbNexthopsKey(destinationIp, dpnId)) - .setDstDeviceId(destinationIp).setSrcDpId(dpnId) - .setNexthopKey(Collections.singletonList(groupIdKey)).build(); - } - public Optional getNexthops(String nextHopKey) { InstanceIdentifier nextHopsId = InstanceIdentifier.builder(L3vpnLbNexthops.class) .child(Nexthops.class, new NexthopsKey(nextHopKey)).build(); return MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, nextHopsId); } - public Optional getDpnLbNexthops(BigInteger dpnId, String destinationIp) { - InstanceIdentifier id = InstanceIdentifier.builder(DpidL3vpnLbNexthops.class) - .child(DpnLbNexthops.class, new DpnLbNexthopsKey(destinationIp, dpnId)) + public List getL3VpnDcGateWays() { + InstanceIdentifier id = InstanceIdentifier.builder(L3vpnDcGws.class) .build(); - return MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id); + Optional dcGwsOpt = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id); + if (!dcGwsOpt.isPresent()) { + return Collections.emptyList(); + } + return dcGwsOpt.get().getDcGateway().stream().map(DcGateway::getIpAddress).collect(toList()); } static boolean isVxlanNetwork(NetworkType networkType) { diff --git a/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/NexthopManager.java b/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/NexthopManager.java index 9424d12de1..211049d31d 100644 --- a/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/NexthopManager.java +++ b/fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/NexthopManager.java @@ -7,14 +7,18 @@ */ package org.opendaylight.netvirt.fibmanager; +import static java.util.stream.Collectors.toList; import static org.opendaylight.genius.infra.Datastore.CONFIGURATION; import static org.opendaylight.genius.infra.Datastore.OPERATIONAL; import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address; import com.google.common.base.Optional; import com.google.common.base.Preconditions; +import com.google.common.util.concurrent.ListenableFuture; + import java.math.BigInteger; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -24,9 +28,11 @@ import java.util.Objects; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; + import javax.annotation.PreDestroy; import javax.inject.Inject; import javax.inject.Singleton; + import org.eclipse.jdt.annotation.Nullable; import org.opendaylight.controller.md.sal.binding.api.DataBroker; import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType; @@ -53,7 +59,6 @@ import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId; import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldVlanVid; import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager; import org.opendaylight.infrautils.jobcoordinator.JobCoordinator; -import org.opendaylight.infrautils.utils.concurrent.ListenableFutures; import org.opendaylight.netvirt.elanmanager.api.IElanService; import org.opendaylight.netvirt.fibmanager.api.L3VPNTransportTypes; import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper; @@ -87,6 +92,7 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.Tun import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelsState; import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList; import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelListKey; +import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rev160406.DcGatewayIpList; import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelInputBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelOutput; import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameInputBuilder; @@ -122,7 +128,6 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacenciesKey; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpn; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpnBuilder; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.dpid.l3vpn.lb.nexthops.DpnLbNexthops; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.lb.nexthops.Nexthops; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes; @@ -143,6 +148,8 @@ public class NexthopManager implements AutoCloseable { private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L); private static final long WAIT_TIME_TO_ACQUIRE_LOCK = 3000L; private static final int SELECT_GROUP_WEIGHT = 1; + private static final int RETRY_COUNT = 6; + private static final String NEXTHOPMANAGER_JOB_KEY_PREFIX = "NextHopManager"; private final DataBroker dataBroker; private final ManagedNewTransactionRunner txRunner; @@ -1058,31 +1065,72 @@ public class NexthopManager implements AutoCloseable { return listBucketInfo; } - public void createDcGwLoadBalancingGroup(List availableDcGws, BigInteger dpnId, String destinationIp, + public void createDcGwLoadBalancingGroup(BigInteger dpnId, String destinationIp, Class tunnelType) { - Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present"); - int noOfDcGws = availableDcGws.size(); - if (noOfDcGws == 1) { - LOG.trace("There are no enough DC GateWays {} present to program LB group", availableDcGws); - return; - } - // TODO : Place the logic to construct all possible DC-GW combination here. - String groupIdKey = FibUtil.getGreLbGroupKey(availableDcGws); - Long groupId = createNextHopPointer(groupIdKey); - List listBucket = new ArrayList<>(); - for (int index = 0; index < noOfDcGws; index++) { - if (isTunnelUp(availableDcGws.get(index), dpnId, tunnelType)) { - listBucket.add(buildBucketForDcGwLbGroup(availableDcGws.get(index), dpnId, index, tunnelType)); - } + jobCoordinator.enqueueJob(getJobKey(dpnId), () -> { + List> futures = new ArrayList<>(); + futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operationalTx -> { + synchronized (getDcGateWaySyncKey(destinationIp)) { + FibUtil.addL3vpnDcGateWay(destinationIp, operationalTx); + } + futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, configTx -> { + List availableDcGws = getDcGwIps(); + Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present"); + int noOfDcGws = availableDcGws.size(); + if (noOfDcGws == 1) { + LOG.trace("There are no enough DC GateWays {} present to program LB group", availableDcGws); + return; + } + if (availableDcGws.contains(destinationIp)) { + availableDcGws.remove(destinationIp); + } + availableDcGws.forEach(dcGwIp -> { + List dcGws = Arrays.asList(dcGwIp, destinationIp); + Collections.sort(dcGws); + String groupIdKey = FibUtil.getGreLbGroupKey(dcGws); + Long groupId = createNextHopPointer(groupIdKey); + List listBucket = new ArrayList<>(); + for (int index = 0; index < dcGws.size(); index++) { + if (isTunnelUp(dcGws.get(index), dpnId, tunnelType)) { + listBucket.add(buildBucketForDcGwLbGroup(dcGws.get(index), + dpnId, index, tunnelType, true)); + } + } + Group group = MDSALUtil.buildGroup(groupId, groupIdKey, GroupTypes.GroupSelect, + MDSALUtil.buildBucketLists(listBucket)); + mdsalApiManager.addGroup(configTx, dpnId, group); + FibUtil.updateLbGroupInfo(dpnId, groupIdKey, groupId.toString(), operationalTx); + LOG.trace("LB group {} towards DC-GW installed on dpn {}. Group - {}", + groupIdKey, dpnId, group); + }); + })); + })); + return futures; + }, RETRY_COUNT); + } + + private String getJobKey(BigInteger dpnId) { + return new StringBuilder().append(NEXTHOPMANAGER_JOB_KEY_PREFIX).append(dpnId).toString(); + } + + private String getDcGateWaySyncKey(String destinationIp) { + String mutex = new StringBuilder().append("L3vpncDcGateWay").append(destinationIp).toString(); + return mutex.intern(); + } + + private List getDcGwIps() { + InstanceIdentifier dcGatewayIpListid = + InstanceIdentifier.builder(DcGatewayIpList.class).build(); + DcGatewayIpList dcGatewayIpListConfig = + MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, dcGatewayIpListid).orNull(); + if (dcGatewayIpListConfig == null) { + return Collections.emptyList(); } - Group group = MDSALUtil.buildGroup(groupId, groupIdKey, GroupTypes.GroupSelect, - MDSALUtil.buildBucketLists(listBucket)); - ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, - confTx -> mdsalApiManager.addGroup(confTx, dpnId, group)), LOG, "Error adding load-balancing group"); - ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, - operTx -> FibUtil.updateLbGroupInfo(dpnId, destinationIp, groupIdKey, groupId.toString(), operTx)), LOG, - "Error updating load-balancing group info"); - LOG.trace("LB group {} towards DC-GW installed on dpn {}. Group - {}", groupIdKey, dpnId, group); + return dcGatewayIpListConfig.getDcGatewayIp() + .stream() + .filter(dcGwIp -> dcGwIp.getTunnnelType().equals(TunnelTypeMplsOverGre.class)) + .map(dcGwIp -> dcGwIp.getIpAddress().stringValue()).sorted() + .collect(toList()); } private boolean isTunnelUp(String dcGwIp, BigInteger dpnId, Class tunnelType) { @@ -1119,30 +1167,23 @@ public class NexthopManager implements AutoCloseable { } /** - * This method is invoked when the tunnel state is removed from DS. - * If the there is just one DC-GW left in configuration then the LB groups can be deleted. - * Otherwise, the groups are just updated. + * This method is invoked when the neighbor is removed from DS. + * All the LB groups which point to the given destination will be deleted. */ - public void removeOrUpdateDcGwLoadBalancingGroup(List availableDcGws, BigInteger dpnId, + public void removeDcGwLoadBalancingGroup(BigInteger dpnId, String destinationIp) { - Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present"); - ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> { - ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> { - int noOfDcGws = availableDcGws.size(); - // If availableDcGws does not contain the destination Ip it means this is a configuration delete. - if (!availableDcGws.contains(destinationIp)) { - availableDcGws.add(destinationIp); - Collections.sort(availableDcGws); - } - // TODO : Place the logic to construct all possible DC-GW combination here. - int bucketId = availableDcGws.indexOf(destinationIp); - Optional dpnLbNextHops = fibUtil.getDpnLbNexthops(dpnId, destinationIp); - if (!dpnLbNextHops.isPresent()) { - return; - } - List nextHopKeys = dpnLbNextHops.get().getNexthopKey(); - if (nextHopKeys != null) { - for (String nextHopKey : nextHopKeys) { + jobCoordinator.enqueueJob(getJobKey(dpnId), () -> { + List availableDcGws = fibUtil.getL3VpnDcGateWays(); + if (availableDcGws.contains(destinationIp)) { + availableDcGws.remove(destinationIp); + } + List> futures = new ArrayList<>(); + futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operationalTx -> { + futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx -> { + availableDcGws.forEach(dcGwIp -> { + List dcGws = Arrays.asList(dcGwIp, destinationIp); + Collections.sort(dcGws); + String nextHopKey = FibUtil.getGreLbGroupKey(dcGws); Optional optionalNextHops = fibUtil.getNexthops(nextHopKey); if (!optionalNextHops.isPresent()) { return; @@ -1150,41 +1191,44 @@ public class NexthopManager implements AutoCloseable { Nexthops nexthops = optionalNextHops.get(); final String groupId = nexthops.getGroupId(); final long groupIdValue = Long.parseLong(groupId); - if (noOfDcGws > 1) { - mdsalApiManager.removeBucket(confTx, dpnId, groupIdValue, bucketId); - } else { - LOG.trace("Removed LB group {} on dpn {}", groupIdValue, dpnId); - mdsalApiManager.removeGroup(confTx, dpnId, groupIdValue); - removeNextHopPointer(nextHopKey); - } - // When the DC-GW is removed from configuration. - if (noOfDcGws != availableDcGws.size()) { - FibUtil.removeOrUpdateNextHopInfo(dpnId, nextHopKey, groupId, nexthops, operTx); + Group group = MDSALUtil.buildGroup(groupIdValue, nextHopKey, GroupTypes.GroupSelect, + MDSALUtil.buildBucketLists(Collections.emptyList())); + LOG.trace("Removed LB group {} on dpn {}", group, dpnId); + try { + mdsalApiManager.removeGroup(configTx, dpnId, group); + } catch (ExecutionException | InterruptedException e) { + LOG.error("Group removal failed for group {} with exception", groupId, e); } + removeNextHopPointer(nextHopKey); + FibUtil.removeOrUpdateNextHopInfo(dpnId, nextHopKey, groupId, nexthops, operationalTx); + }); + synchronized (getDcGateWaySyncKey(destinationIp)) { + FibUtil.removeL3vpnDcGateWay(destinationIp, operationalTx); } - } - FibUtil.removeDpnIdToNextHopInfo(destinationIp, dpnId, operTx); - }), LOG, "Error removing or updating load-balancing group"); - }), LOG, "Error removing or updating load-balancing group"); + })); + })); + return futures; + }, RETRY_COUNT); } /** - * This method is invoked when the tunnel status is updated. - * The bucket is directly removed/added based on the operational status of the tunnel. + * This method is invoked when the tunnel status is deleted. + * All the buckets which point to given destination will be marked down. */ - public void updateDcGwLoadBalancingGroup(List availableDcGws, - BigInteger dpnId, String destinationIp, boolean isTunnelUp, Class tunnelType) { - Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present"); - ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> { - // TODO : Place the logic to construct all possible DC-GW combination here. - int bucketId = availableDcGws.indexOf(destinationIp); - Optional dpnLbNextHops = fibUtil.getDpnLbNexthops(dpnId, destinationIp); - if (!dpnLbNextHops.isPresent()) { - return; + public void updateDcGwLoadBalancingGroup(BigInteger dpnId, String destinationIp, + boolean isTunnelUp, Class tunnelType) { + jobCoordinator.enqueueJob(getJobKey(dpnId), () -> { + List availableDcGws = fibUtil.getL3VpnDcGateWays(); + if (availableDcGws.contains(destinationIp)) { + availableDcGws.remove(destinationIp); } - List nextHopKeys = dpnLbNextHops.get().getNexthopKey(); - if (nextHopKeys != null) { - for (String nextHopKey : nextHopKeys) { + List> futures = new ArrayList<>(); + futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx -> { + availableDcGws.forEach(dcGwIp -> { + List dcGws = Arrays.asList(dcGwIp, destinationIp); + Collections.sort(dcGws); + String nextHopKey = FibUtil.getGreLbGroupKey(dcGws); + int bucketId = dcGws.indexOf(destinationIp); Optional optionalNextHops = fibUtil.getNexthops(nextHopKey); if (!optionalNextHops.isPresent()) { return; @@ -1192,21 +1236,21 @@ public class NexthopManager implements AutoCloseable { Nexthops nexthops = optionalNextHops.get(); final String groupId = nexthops.getGroupId(); final long groupIdValue = Long.parseLong(groupId); - if (isTunnelUp) { - Bucket bucket = buildBucketForDcGwLbGroup(destinationIp, dpnId, bucketId, tunnelType); - LOG.trace("Added bucket {} to group {} on dpn {}.", bucket, groupId, dpnId); - mdsalApiManager.addBucket(confTx, dpnId, groupIdValue, bucket); - } else { - LOG.trace("Removed bucketId {} from group {} on dpn {}.", bucketId, groupId, dpnId); - mdsalApiManager.removeBucket(confTx, dpnId, groupIdValue, bucketId); + Bucket bucket = buildBucketForDcGwLbGroup(destinationIp, dpnId, bucketId, tunnelType, isTunnelUp); + LOG.trace("updated bucket {} to group {} on dpn {}.", bucket, groupId, dpnId); + try { + mdsalApiManager.addBucket(configTx, dpnId, groupIdValue, bucket); + } catch (ExecutionException | InterruptedException e) { + LOG.error("Bucket addition failed for bucket {} with exception", bucketId, e); } - } - } - }), LOG, "Error updating load-balancing group"); + }); + })); + return futures; + }, RETRY_COUNT); } - private Bucket buildBucketForDcGwLbGroup(String ipAddress, BigInteger dpnId, int index, - Class tunnelType) { + private Bucket buildBucketForDcGwLbGroup(String ipAddress, BigInteger dpnId, + int index, Class tunnelType, boolean isTunnelUp) { List listAction = new ArrayList<>(); // ActionKey 0 goes to mpls label. int actionKey = 1; @@ -1222,20 +1266,24 @@ public class NexthopManager implements AutoCloseable { // clear off actions if there is no egress actions. listAction = Collections.emptyList(); } + long watchPort = MDSALUtil.WATCH_PORT; + if (!isTunnelUp) { + watchPort = 0xFFFFFFFEL; + } //OVS expects a non-zero weight value for load balancing to happen in select groups return MDSALUtil.buildBucket(listAction, SELECT_GROUP_WEIGHT, index, - MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP); + watchPort, MDSALUtil.WATCH_GROUP); } - public void programDcGwLoadBalancingGroup(List availableDcGws, BigInteger dpnId, String destinationIp, + public void programDcGwLoadBalancingGroup(BigInteger dpnId, String destinationIp, int addRemoveOrUpdate, boolean isTunnelUp, Class tunnelType) { if (NwConstants.ADD_FLOW == addRemoveOrUpdate) { - createDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp, tunnelType); + createDcGwLoadBalancingGroup(dpnId, destinationIp, tunnelType); } else if (NwConstants.DEL_FLOW == addRemoveOrUpdate) { - removeOrUpdateDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp); + removeDcGwLoadBalancingGroup(dpnId, destinationIp); } else if (NwConstants.MOD_FLOW == addRemoveOrUpdate) { - updateDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp, isTunnelUp, tunnelType); + updateDcGwLoadBalancingGroup(dpnId, destinationIp, isTunnelUp, tunnelType); } } } diff --git a/vpnmanager/api/src/main/yang/odl-l3vpn.yang b/vpnmanager/api/src/main/yang/odl-l3vpn.yang index b03a029c2f..2d49cffa57 100644 --- a/vpnmanager/api/src/main/yang/odl-l3vpn.yang +++ b/vpnmanager/api/src/main/yang/odl-l3vpn.yang @@ -688,13 +688,13 @@ module odl-l3vpn { } } - container dpid-l3vpn-lb-nexthops { + container l3vpn-dc-gws { config false; - list dpn-lb-nexthops { - key "src-dp-id dst-device-id"; - leaf src-dp-id { type uint64; } - leaf dst-device-id { type string; } //dpId or ip-address - leaf-list nexthop-key { type string; } + list dc-gateway { + key "ip-address"; + leaf ip-address { + type string; + } } } diff --git a/vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/TunnelInterfaceStateListener.java b/vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/TunnelInterfaceStateListener.java index 8e5dd06024..b998ce246c 100644 --- a/vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/TunnelInterfaceStateListener.java +++ b/vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/TunnelInterfaceStateListener.java @@ -8,13 +8,13 @@ package org.opendaylight.netvirt.vpnmanager; import static java.util.Collections.emptyList; -import static java.util.stream.Collectors.toList; import static org.opendaylight.genius.infra.Datastore.CONFIGURATION; import static org.opendaylight.genius.infra.Datastore.OPERATIONAL; import com.google.common.base.Optional; import com.google.common.base.Strings; import com.google.common.util.concurrent.ListenableFuture; + import java.math.BigInteger; import java.util.ArrayList; import java.util.HashMap; @@ -29,6 +29,7 @@ import java.util.concurrent.locks.ReentrantLock; import javax.annotation.PostConstruct; import javax.inject.Inject; import javax.inject.Singleton; + import org.opendaylight.controller.md.sal.binding.api.DataBroker; import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType; import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException; @@ -36,7 +37,6 @@ import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase; import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker; import org.opendaylight.genius.infra.ManagedNewTransactionRunner; import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl; -import org.opendaylight.genius.mdsalutil.MDSALUtil; import org.opendaylight.genius.mdsalutil.NwConstants; import org.opendaylight.genius.utils.JvmGlobalLocks; import org.opendaylight.infrautils.jobcoordinator.JobCoordinator; @@ -64,7 +64,6 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.Tep import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelOperStatus; import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelsState; import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList; -import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rev160406.DcGatewayIpList; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.extraroute.rds.map.extraroute.rds.DestPrefixes; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths; @@ -154,7 +153,7 @@ public class TunnelInterfaceStateListener extends AsyncDataTreeChangeListenerBas protected void remove(InstanceIdentifier identifier, StateTunnelList del) { LOG.trace("remove: Tunnel deletion---- {}", del); if (isGreTunnel(del)) { - programDcGwLoadBalancingGroup(del, NwConstants.DEL_FLOW); + programDcGwLoadBalancingGroup(del, NwConstants.MOD_FLOW, false); } handleTunnelEventForDPN(del, TunnelAction.TUNNEL_EP_DELETE); } @@ -173,8 +172,9 @@ public class TunnelInterfaceStateListener extends AsyncDataTreeChangeListenerBas update.getTunnelInterfaceName()); return; } + boolean isTunnelUp = TunnelOperStatus.Up == update.getOperState(); if (isGreTunnel(update)) { - programDcGwLoadBalancingGroup(update, NwConstants.MOD_FLOW); + programDcGwLoadBalancingGroup(update, NwConstants.MOD_FLOW, isTunnelUp); } //Remove the corresponding nexthop from the routepath under extraroute in fibentries. @@ -230,8 +230,9 @@ public class TunnelInterfaceStateListener extends AsyncDataTreeChangeListenerBas if (tunOpStatus != TunnelOperStatus.Up) { LOG.error("add: Tunnel {} is not yet UP.", add.getTunnelInterfaceName()); } + boolean isTunnelUp = TunnelOperStatus.Up == add.getOperState(); if (isGreTunnel(add)) { - programDcGwLoadBalancingGroup(add, NwConstants.ADD_FLOW); + programDcGwLoadBalancingGroup(add, NwConstants.ADD_FLOW, isTunnelUp); } LOG.info("add: ITM Tunnel ,type {} ,added between src: {} and dest: {}", fibManager.getTransportTypeStr(add.getTransportType().toString()), @@ -564,31 +565,14 @@ public class TunnelInterfaceStateListener extends AsyncDataTreeChangeListenerBas return del.getTransportType() == TunnelTypeMplsOverGre.class; } - private void programDcGwLoadBalancingGroup(StateTunnelList tunnelState, int addOrRemove) { + private void programDcGwLoadBalancingGroup(StateTunnelList tunnelState, int addOrRemove, boolean isTunnelUp) { IpAddress dcGwIp = tunnelState.getDstInfo().getTepIp(); - String dcGwIpAddress = dcGwIp.stringValue(); - List availableDcGws = getDcGwIps(); + String dcGwIpAddress = String.valueOf(dcGwIp.stringValue()); BigInteger dpId = new BigInteger(tunnelState.getSrcInfo().getTepDeviceId()); - boolean isTunnelUp = TunnelOperStatus.Up == tunnelState.getOperState(); - fibManager.programDcGwLoadBalancingGroup(availableDcGws, dpId, dcGwIpAddress, addOrRemove, isTunnelUp, + fibManager.programDcGwLoadBalancingGroup(dpId, dcGwIpAddress, addOrRemove, isTunnelUp, tunnelState.getTransportType()); } - private List getDcGwIps() { - InstanceIdentifier dcGatewayIpListid = - InstanceIdentifier.builder(DcGatewayIpList.class).build(); - DcGatewayIpList dcGatewayIpListConfig = - MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, dcGatewayIpListid).orNull(); - if (dcGatewayIpListConfig == null) { - return emptyList(); - } - return dcGatewayIpListConfig.getDcGatewayIp() - .stream() - .filter(dcGwIp -> dcGwIp.getTunnnelType().equals(TunnelTypeMplsOverGre.class)) - .map(dcGwIp -> dcGwIp.getIpAddress().stringValue()).sorted() - .collect(toList()); - } - private boolean isTunnelInLogicalGroup(StateTunnelList stateTunnelList) { String ifaceName = stateTunnelList.getTunnelInterfaceName(); if (getTunnelType(stateTunnelList) == VpnConstants.ITMTunnelLocType.Internal.getValue()) { -- 2.36.6