ECMP UC2 Enhancement: Support for more than two DC-GWs 40/83340/5
authorAnkit Jain <ankit.j.jain@ericsson.com>
Wed, 4 Jul 2018 09:49:22 +0000 (15:19 +0530)
committerFaseela K <faseela.k@ericsson.com>
Tue, 17 Sep 2019 15:42:21 +0000 (15:42 +0000)
Following changes are done as part of this enhancement:

1. Creates nc2 load balancing groups with each group having two
buckets pointing towards DC-GWs.
Quagga bgp will only advertise two routes and based on the routes
advertised, table 21 will point to one of the groups.

2. Marks buckets down when external tunnel end point gets
deleted.

3. Yang changes to store DC-GWs.

Note: After this change, DC-GW load balancing gropus will
not be deleted forever, which is fine because Neighbhor
add/delete are initial operations and will not be performed frequently.

Following are the scenarios which will end up having stale groups:
a) When tunnel is deleted and BGP neighbor comes up with
different tunnel ip.
b) When tunnel and BGP neighbor both are deleted.

Change-Id: I74b1025751b58a7d0d19be19e2edd13c8205bde7
Signed-off-by: Ankit Jain <ankit.j.jain@ericsson.com>
Signed-off-by: Amitesh Soni <amitesh.soni@ericsson.com>
bgpmanager/impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpConfigurationManager.java
bgpmanager/impl/src/main/java/org/opendaylight/netvirt/bgpmanager/BgpUtil.java
fibmanager/api/src/main/java/org/opendaylight/netvirt/fibmanager/api/IFibManager.java
fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/FibManagerImpl.java
fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/FibUtil.java
fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/NexthopManager.java
vpnmanager/api/src/main/yang/odl-l3vpn.yang
vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/TunnelInterfaceStateListener.java

index d0b16e121d656c458b4dad75104759575dd2fb90..0eca8f01fe2c1e3c6a3df3c23d4b712f90aaf51e 100755 (executable)
@@ -2084,7 +2084,7 @@ public class BgpConfigurationManager {
             return;
         }
         tepIpList.forEach(tepIp -> {
-            bgpUtil.removeOrUpdateLBGroups(tepIp, NwConstants.MOD_FLOW, false);
+            bgpUtil.removeOrUpdateLBGroups(tepIp, NwConstants.MOD_FLOW);
         });
     }
 
@@ -2095,7 +2095,7 @@ public class BgpConfigurationManager {
             return;
         }
         tepIpList.forEach(tepIp -> {
-            bgpUtil.removeOrUpdateLBGroups(tepIp, NwConstants.MOD_FLOW, true);
+            bgpUtil.removeOrUpdateLBGroups(tepIp, NwConstants.MOD_FLOW);
         });
     }
 
@@ -2549,6 +2549,7 @@ public class BgpConfigurationManager {
         DcgwTep dto = new DcgwTepBuilder().setDcGwIp(dcgwIp).setTepIps(tepList)
                 .build();
         update(iid, dto);
+        bgpUtil.removeOrUpdateLBGroups(tepIp,NwConstants.MOD_FLOW);
     }
 
     public void addLogging(String fileName, String logLevel) {
index 6f61b3a6702614e9173251e0204ecb3905c0c496..67c9ec9086f460fe22f89a97de2341cf89baa8c7 100755 (executable)
@@ -7,14 +7,12 @@
  */
 package org.opendaylight.netvirt.bgpmanager;
 
-import static java.util.stream.Collectors.toList;
-
 import com.google.common.base.Optional;
+
 import java.net.Inet4Address;
 import java.net.Inet6Address;
 import java.net.InetAddress;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -52,7 +50,6 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.DpnEndpoints;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.dpn.endpoints.DPNTEPsInfo;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rev160406.DcGatewayIpList;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.ElanInstances;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstanceKey;
@@ -377,25 +374,7 @@ public class BgpUtil implements AutoCloseable {
         }
     }
 
-    private List<String> getDcGwIps() {
-        InstanceIdentifier<DcGatewayIpList> dcGatewayIpListid =
-                InstanceIdentifier.builder(DcGatewayIpList.class).build();
-        DcGatewayIpList dcGatewayIpListConfig =
-                MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, dcGatewayIpListid).orNull();
-        if (dcGatewayIpListConfig == null) {
-            return Collections.EMPTY_LIST;
-        }
-        return dcGatewayIpListConfig.getDcGatewayIp()
-                .stream()
-                .filter(dcGwIp -> dcGwIp.getTunnnelType().equals(TunnelTypeMplsOverGre.class))
-                .map(dcGwIp -> String.valueOf(dcGwIp.getIpAddress().getIpv4Address())).sorted()
-                .collect(toList());
-    }
-
-
-    public void removeOrUpdateLBGroups(String tepIp, int addRemoveOrUpdate, boolean isTunnelUp) {
-        LOG.debug("removing bucket towards DCGW {}", tepIp);
-        List<String> availableDcGws = getDcGwIps();
+    public void removeOrUpdateLBGroups(String tepIp, int addRemoveOrUpdate) {
         getDpnTEPsInfos(dataBroker).forEach(dpnInfo -> {
             if (NwConstants.MOD_FLOW == addRemoveOrUpdate) {
                 LOG.debug("Updating bucket in DPN {}", dpnInfo.getDPNID());
@@ -403,8 +382,8 @@ public class BgpUtil implements AutoCloseable {
                 LOG.debug("Deleting groups in DPN {}", dpnInfo.getDPNID());
             }
             Class<? extends TunnelTypeBase> tunType = TunnelTypeMplsOverGre.class;
-            fibManager.programDcGwLoadBalancingGroup(availableDcGws, dpnInfo.getDPNID(),
-                    tepIp, addRemoveOrUpdate, isTunnelUp, tunType);
+            fibManager.programDcGwLoadBalancingGroup(dpnInfo.getDPNID(),
+                    tepIp, addRemoveOrUpdate, false, tunType);
         });
     }
 }
index d2f720055968e6cec9fcd4b775c5d067d63b8786..de42f3058886ede030a6953365c83d258d4e7314 100644 (file)
@@ -79,8 +79,8 @@ public interface IFibManager {
                                       boolean isVpnFirstEndPoint,
                                       VrfEntry vrfEntry);
 
-    void programDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId, String destinationIp,
-                                       int addRemoveOrUpdate, boolean isTunnelUp,
+    void programDcGwLoadBalancingGroup(BigInteger dpnId,
+            String destinationIp, int addRemoveOrUpdate, boolean isTunnelUp,
                                        Class<? extends TunnelTypeBase> tunnelType);
 
     void refreshVrfEntry(String rd, String prefix);
index 0d1515cc149011272bdc5c89b5f6a3eca31631bc..91d6f84b44f0788b9ee92de93fdef06061e22365 100755 (executable)
@@ -200,10 +200,10 @@ public class FibManagerImpl implements IFibManager {
     }
 
     @Override
-    public void programDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId, String destinationIp,
+    public void programDcGwLoadBalancingGroup(BigInteger dpnId, String destinationIp,
                                               int addRemoveOrUpdate, boolean isTunnelUp,
                                               Class<? extends TunnelTypeBase> tunnelType) {
-        nexthopManager.programDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp,
+        nexthopManager.programDcGwLoadBalancingGroup(dpnId, destinationIp,
             addRemoveOrUpdate, isTunnelUp, tunnelType);
     }
 
index 6314d5e17425b5ea7ff454c7e237ee400382c512..7d134c115bc8824a13e0a9bfc2d5f6e550b837cb 100644 (file)
@@ -92,15 +92,15 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev15033
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntryKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.DpidL3vpnLbNexthops;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.L3vpnDcGws;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.L3vpnLbNexthops;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnIdToVpnInstance;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceToVpnId;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.dpid.l3vpn.lb.nexthops.DpnLbNexthops;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.dpid.l3vpn.lb.nexthops.DpnLbNexthopsBuilder;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.dpid.l3vpn.lb.nexthops.DpnLbNexthopsKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.dc.gws.DcGateway;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.dc.gws.DcGatewayBuilder;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.dc.gws.DcGatewayKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.lb.nexthops.Nexthops;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.lb.nexthops.NexthopsBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.lb.nexthops.NexthopsKey;
@@ -681,19 +681,40 @@ public class FibUtil {
         return "gre-" + availableDcGws.stream().sorted().collect(joining(":"));
     }
 
-    public static void updateLbGroupInfo(BigInteger dpnId, String destinationIp, String groupIdKey,
+    public static void updateLbGroupInfo(BigInteger dpnId, String groupIdKey,
             String groupId, TypedWriteTransaction<Operational> tx) {
-        InstanceIdentifier<DpnLbNexthops> id = getDpnLbNexthopsIdentifier(dpnId, destinationIp);
-        DpnLbNexthops dpnToLbNextHop = buildDpnLbNextHops(dpnId, destinationIp, groupIdKey);
-        tx.merge(id, dpnToLbNextHop);
         InstanceIdentifier<Nexthops> nextHopsId = getNextHopsIdentifier(groupIdKey);
         Nexthops nextHopsToGroupId = buildNextHops(dpnId, groupIdKey, groupId);
         tx.merge(nextHopsId, nextHopsToGroupId);
     }
 
-    public static void removeDpnIdToNextHopInfo(String destinationIp, BigInteger dpnId,
-            TypedWriteTransaction<Operational> tx) {
-        tx.delete(getDpnLbNexthopsIdentifier(dpnId, destinationIp));
+    public static void removeL3vpnDcGateWay(String destinationIp, TypedReadWriteTransaction<Operational> tx)
+            throws InterruptedException, ExecutionException {
+        InstanceIdentifier<DcGateway> dcGateWayId = getDcGwInstance(destinationIp);
+        Optional<DcGateway> dcGateWayOpt = tx.read(dcGateWayId).get();
+        if (!dcGateWayOpt.isPresent()) {
+            return;
+        }
+        tx.delete(dcGateWayId);
+    }
+
+    public static void addL3vpnDcGateWay(String destinationIp, TypedReadWriteTransaction<Operational> tx)
+            throws InterruptedException, ExecutionException {
+        InstanceIdentifier<DcGateway> dcGateWayId = getDcGwInstance(destinationIp);
+        Optional<DcGateway> dcGateWayOpt = tx.read(dcGateWayId).get();
+        if (!dcGateWayOpt.isPresent()) {
+            tx.put(dcGateWayId,
+                    new DcGatewayBuilder()
+                    .withKey(new DcGatewayKey(destinationIp))
+                    .setIpAddress(destinationIp).build()
+            );
+        }
+    }
+
+    private static InstanceIdentifier<DcGateway> getDcGwInstance(String destinationIp) {
+        return InstanceIdentifier.builder(L3vpnDcGws.class)
+                .child(DcGateway.class, new DcGatewayKey(destinationIp))
+                .build();
     }
 
     public static void removeOrUpdateNextHopInfo(BigInteger dpnId, String nextHopKey, String groupId,
@@ -713,13 +734,6 @@ public class FibUtil {
         }
     }
 
-    private static InstanceIdentifier<DpnLbNexthops> getDpnLbNexthopsIdentifier(BigInteger dpnId,
-            String destinationIp) {
-        return InstanceIdentifier.builder(DpidL3vpnLbNexthops.class)
-                .child(DpnLbNexthops.class, new DpnLbNexthopsKey(destinationIp, dpnId))
-                .build();
-    }
-
     private static InstanceIdentifier<Nexthops> getNextHopsIdentifier(String groupIdKey) {
         return InstanceIdentifier.builder(L3vpnLbNexthops.class)
                 .child(Nexthops.class, new NexthopsKey(groupIdKey)).build();
@@ -732,24 +746,20 @@ public class FibUtil {
                 .setTargetDeviceId(Collections.singletonList(dpnId.toString())).build();
     }
 
-    private static DpnLbNexthops buildDpnLbNextHops(BigInteger dpnId, String destinationIp,
-            String groupIdKey) {
-        return new DpnLbNexthopsBuilder().withKey(new DpnLbNexthopsKey(destinationIp, dpnId))
-                .setDstDeviceId(destinationIp).setSrcDpId(dpnId)
-                .setNexthopKey(Collections.singletonList(groupIdKey)).build();
-    }
-
     public Optional<Nexthops> getNexthops(String nextHopKey) {
         InstanceIdentifier<Nexthops> nextHopsId = InstanceIdentifier.builder(L3vpnLbNexthops.class)
                 .child(Nexthops.class, new NexthopsKey(nextHopKey)).build();
         return MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, nextHopsId);
     }
 
-    public Optional<DpnLbNexthops> getDpnLbNexthops(BigInteger dpnId, String destinationIp) {
-        InstanceIdentifier<DpnLbNexthops> id = InstanceIdentifier.builder(DpidL3vpnLbNexthops.class)
-                .child(DpnLbNexthops.class, new DpnLbNexthopsKey(destinationIp, dpnId))
+    public List<String> getL3VpnDcGateWays() {
+        InstanceIdentifier<L3vpnDcGws> id = InstanceIdentifier.builder(L3vpnDcGws.class)
                 .build();
-        return MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
+        Optional<L3vpnDcGws> dcGwsOpt = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
+        if (!dcGwsOpt.isPresent()) {
+            return Collections.emptyList();
+        }
+        return dcGwsOpt.get().getDcGateway().stream().map(DcGateway::getIpAddress).collect(toList());
     }
 
     static boolean isVxlanNetwork(NetworkType networkType) {
index 9424d12de161c1186b876092ad35bfd1d00b872e..211049d31d8cf03a96326ac7fcaccbedd6f6deac 100644 (file)
@@ -7,14 +7,18 @@
  */
 package org.opendaylight.netvirt.fibmanager;
 
+import static java.util.stream.Collectors.toList;
 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
 
 import com.google.common.base.Optional;
 import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.ListenableFuture;
+
 import java.math.BigInteger;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -24,9 +28,11 @@ import java.util.Objects;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
+
 import javax.annotation.PreDestroy;
 import javax.inject.Inject;
 import javax.inject.Singleton;
+
 import org.eclipse.jdt.annotation.Nullable;
 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
@@ -53,7 +59,6 @@ import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldVlanVid;
 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
-import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
 import org.opendaylight.netvirt.elanmanager.api.IElanService;
 import org.opendaylight.netvirt.fibmanager.api.L3VPNTransportTypes;
 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
@@ -87,6 +92,7 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.Tun
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelsState;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelListKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rev160406.DcGatewayIpList;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelInputBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelOutput;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameInputBuilder;
@@ -122,7 +128,6 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacenciesKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpn;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpnBuilder;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.dpid.l3vpn.lb.nexthops.DpnLbNexthops;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.lb.nexthops.Nexthops;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
@@ -143,6 +148,8 @@ public class NexthopManager implements AutoCloseable {
     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
     private static final long WAIT_TIME_TO_ACQUIRE_LOCK = 3000L;
     private static final int SELECT_GROUP_WEIGHT = 1;
+    private static final int RETRY_COUNT = 6;
+    private static final String NEXTHOPMANAGER_JOB_KEY_PREFIX = "NextHopManager";
 
     private final DataBroker dataBroker;
     private final ManagedNewTransactionRunner txRunner;
@@ -1058,31 +1065,72 @@ public class NexthopManager implements AutoCloseable {
         return listBucketInfo;
     }
 
-    public void createDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId, String destinationIp,
+    public void createDcGwLoadBalancingGroup(BigInteger dpnId, String destinationIp,
                                              Class<? extends TunnelTypeBase> tunnelType) {
-        Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
-        int noOfDcGws = availableDcGws.size();
-        if (noOfDcGws == 1) {
-            LOG.trace("There are no enough DC GateWays {} present to program LB group", availableDcGws);
-            return;
-        }
-        // TODO : Place the logic to construct all possible DC-GW combination here.
-        String groupIdKey = FibUtil.getGreLbGroupKey(availableDcGws);
-        Long groupId = createNextHopPointer(groupIdKey);
-        List<Bucket> listBucket = new ArrayList<>();
-        for (int index = 0; index < noOfDcGws; index++) {
-            if (isTunnelUp(availableDcGws.get(index), dpnId, tunnelType)) {
-                listBucket.add(buildBucketForDcGwLbGroup(availableDcGws.get(index), dpnId, index, tunnelType));
-            }
+        jobCoordinator.enqueueJob(getJobKey(dpnId), () -> {
+            List<ListenableFuture<Void>> futures = new ArrayList<>();
+            futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operationalTx -> {
+                synchronized (getDcGateWaySyncKey(destinationIp)) {
+                    FibUtil.addL3vpnDcGateWay(destinationIp, operationalTx);
+                }
+                futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, configTx -> {
+                    List<String> availableDcGws = getDcGwIps();
+                    Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
+                    int noOfDcGws = availableDcGws.size();
+                    if (noOfDcGws == 1) {
+                        LOG.trace("There are no enough DC GateWays {} present to program LB group", availableDcGws);
+                        return;
+                    }
+                    if (availableDcGws.contains(destinationIp)) {
+                        availableDcGws.remove(destinationIp);
+                    }
+                    availableDcGws.forEach(dcGwIp -> {
+                        List<String> dcGws = Arrays.asList(dcGwIp, destinationIp);
+                        Collections.sort(dcGws);
+                        String groupIdKey = FibUtil.getGreLbGroupKey(dcGws);
+                        Long groupId = createNextHopPointer(groupIdKey);
+                        List<Bucket> listBucket = new ArrayList<>();
+                        for (int index = 0; index < dcGws.size(); index++) {
+                            if (isTunnelUp(dcGws.get(index), dpnId, tunnelType)) {
+                                listBucket.add(buildBucketForDcGwLbGroup(dcGws.get(index),
+                                        dpnId, index, tunnelType, true));
+                            }
+                        }
+                        Group group = MDSALUtil.buildGroup(groupId, groupIdKey, GroupTypes.GroupSelect,
+                                        MDSALUtil.buildBucketLists(listBucket));
+                        mdsalApiManager.addGroup(configTx, dpnId, group);
+                        FibUtil.updateLbGroupInfo(dpnId, groupIdKey, groupId.toString(), operationalTx);
+                        LOG.trace("LB group {} towards DC-GW installed on dpn {}. Group - {}",
+                                groupIdKey, dpnId, group);
+                    });
+                }));
+            }));
+            return futures;
+        }, RETRY_COUNT);
+    }
+
+    private String getJobKey(BigInteger dpnId) {
+        return new StringBuilder().append(NEXTHOPMANAGER_JOB_KEY_PREFIX).append(dpnId).toString();
+    }
+
+    private String getDcGateWaySyncKey(String destinationIp) {
+        String mutex = new StringBuilder().append("L3vpncDcGateWay").append(destinationIp).toString();
+        return mutex.intern();
+    }
+
+    private List<String> getDcGwIps() {
+        InstanceIdentifier<DcGatewayIpList> dcGatewayIpListid =
+                InstanceIdentifier.builder(DcGatewayIpList.class).build();
+        DcGatewayIpList dcGatewayIpListConfig =
+                MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, dcGatewayIpListid).orNull();
+        if (dcGatewayIpListConfig == null) {
+            return Collections.emptyList();
         }
-        Group group = MDSALUtil.buildGroup(groupId, groupIdKey, GroupTypes.GroupSelect,
-                        MDSALUtil.buildBucketLists(listBucket));
-        ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
-            confTx -> mdsalApiManager.addGroup(confTx, dpnId, group)), LOG, "Error adding load-balancing group");
-        ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
-            operTx -> FibUtil.updateLbGroupInfo(dpnId, destinationIp, groupIdKey, groupId.toString(), operTx)), LOG,
-            "Error updating load-balancing group info");
-        LOG.trace("LB group {} towards DC-GW installed on dpn {}. Group - {}", groupIdKey, dpnId, group);
+        return dcGatewayIpListConfig.getDcGatewayIp()
+                .stream()
+                .filter(dcGwIp -> dcGwIp.getTunnnelType().equals(TunnelTypeMplsOverGre.class))
+                .map(dcGwIp -> dcGwIp.getIpAddress().stringValue()).sorted()
+                .collect(toList());
     }
 
     private boolean isTunnelUp(String dcGwIp, BigInteger dpnId, Class<? extends TunnelTypeBase> tunnelType) {
@@ -1119,30 +1167,23 @@ public class NexthopManager implements AutoCloseable {
     }
 
     /**
-     * This method is invoked when the tunnel state is removed from DS.
-     * If the there is just one DC-GW left in configuration then the LB groups can be deleted.
-     * Otherwise, the groups are just updated.
+     * This method is invoked when the neighbor is removed from DS.
+     * All the LB groups which point to the given destination will be deleted.
      */
-    public void removeOrUpdateDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId,
+    public void removeDcGwLoadBalancingGroup(BigInteger dpnId,
             String destinationIp) {
-        Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
-        ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
-            ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
-                int noOfDcGws = availableDcGws.size();
-                // If availableDcGws does not contain the destination Ip it means this is a configuration delete.
-                if (!availableDcGws.contains(destinationIp)) {
-                    availableDcGws.add(destinationIp);
-                    Collections.sort(availableDcGws);
-                }
-                // TODO : Place the logic to construct all possible DC-GW combination here.
-                int bucketId = availableDcGws.indexOf(destinationIp);
-                Optional<DpnLbNexthops> dpnLbNextHops = fibUtil.getDpnLbNexthops(dpnId, destinationIp);
-                if (!dpnLbNextHops.isPresent()) {
-                    return;
-                }
-                List<String> nextHopKeys = dpnLbNextHops.get().getNexthopKey();
-                if (nextHopKeys != null) {
-                    for (String nextHopKey : nextHopKeys) {
+        jobCoordinator.enqueueJob(getJobKey(dpnId), () -> {
+            List<String> availableDcGws = fibUtil.getL3VpnDcGateWays();
+            if (availableDcGws.contains(destinationIp)) {
+                availableDcGws.remove(destinationIp);
+            }
+            List<ListenableFuture<Void>> futures = new ArrayList<>();
+            futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operationalTx -> {
+                futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx -> {
+                    availableDcGws.forEach(dcGwIp -> {
+                        List<String> dcGws = Arrays.asList(dcGwIp, destinationIp);
+                        Collections.sort(dcGws);
+                        String nextHopKey = FibUtil.getGreLbGroupKey(dcGws);
                         Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
                         if (!optionalNextHops.isPresent()) {
                             return;
@@ -1150,41 +1191,44 @@ public class NexthopManager implements AutoCloseable {
                         Nexthops nexthops = optionalNextHops.get();
                         final String groupId = nexthops.getGroupId();
                         final long groupIdValue = Long.parseLong(groupId);
-                        if (noOfDcGws > 1) {
-                            mdsalApiManager.removeBucket(confTx, dpnId, groupIdValue, bucketId);
-                        } else {
-                            LOG.trace("Removed LB group {} on dpn {}", groupIdValue, dpnId);
-                            mdsalApiManager.removeGroup(confTx, dpnId, groupIdValue);
-                            removeNextHopPointer(nextHopKey);
-                        }
-                        // When the DC-GW is removed from configuration.
-                        if (noOfDcGws != availableDcGws.size()) {
-                            FibUtil.removeOrUpdateNextHopInfo(dpnId, nextHopKey, groupId, nexthops, operTx);
+                        Group group = MDSALUtil.buildGroup(groupIdValue, nextHopKey, GroupTypes.GroupSelect,
+                                MDSALUtil.buildBucketLists(Collections.emptyList()));
+                        LOG.trace("Removed LB group {} on dpn {}", group, dpnId);
+                        try {
+                            mdsalApiManager.removeGroup(configTx, dpnId, group);
+                        } catch (ExecutionException | InterruptedException e) {
+                            LOG.error("Group removal failed for group {} with exception", groupId, e);
                         }
+                        removeNextHopPointer(nextHopKey);
+                        FibUtil.removeOrUpdateNextHopInfo(dpnId, nextHopKey, groupId, nexthops, operationalTx);
+                    });
+                    synchronized (getDcGateWaySyncKey(destinationIp)) {
+                        FibUtil.removeL3vpnDcGateWay(destinationIp, operationalTx);
                     }
-                }
-                FibUtil.removeDpnIdToNextHopInfo(destinationIp, dpnId, operTx);
-            }), LOG, "Error removing or updating load-balancing group");
-        }), LOG, "Error removing or updating load-balancing group");
+                }));
+            }));
+            return futures;
+        }, RETRY_COUNT);
     }
 
     /**
-     * This method is invoked when the tunnel status is updated.
-     * The bucket is directly removed/added based on the operational status of the tunnel.
+     * This method is invoked when the tunnel status is deleted.
+     * All the buckets which point to given destination will be marked down.
      */
-    public void updateDcGwLoadBalancingGroup(List<String> availableDcGws,
-            BigInteger dpnId, String destinationIp, boolean isTunnelUp, Class<? extends TunnelTypeBase> tunnelType) {
-        Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
-        ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
-            // TODO : Place the logic to construct all possible DC-GW combination here.
-            int bucketId = availableDcGws.indexOf(destinationIp);
-            Optional<DpnLbNexthops> dpnLbNextHops = fibUtil.getDpnLbNexthops(dpnId, destinationIp);
-            if (!dpnLbNextHops.isPresent()) {
-                return;
+    public void updateDcGwLoadBalancingGroup(BigInteger dpnId, String destinationIp,
+            boolean isTunnelUp, Class<? extends TunnelTypeBase> tunnelType) {
+        jobCoordinator.enqueueJob(getJobKey(dpnId), () -> {
+            List<String> availableDcGws = fibUtil.getL3VpnDcGateWays();
+            if (availableDcGws.contains(destinationIp)) {
+                availableDcGws.remove(destinationIp);
             }
-            List<String> nextHopKeys = dpnLbNextHops.get().getNexthopKey();
-            if (nextHopKeys != null) {
-                for (String nextHopKey : nextHopKeys) {
+            List<ListenableFuture<Void>> futures = new ArrayList<>();
+            futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx -> {
+                availableDcGws.forEach(dcGwIp -> {
+                    List<String> dcGws = Arrays.asList(dcGwIp, destinationIp);
+                    Collections.sort(dcGws);
+                    String nextHopKey = FibUtil.getGreLbGroupKey(dcGws);
+                    int bucketId = dcGws.indexOf(destinationIp);
                     Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
                     if (!optionalNextHops.isPresent()) {
                         return;
@@ -1192,21 +1236,21 @@ public class NexthopManager implements AutoCloseable {
                     Nexthops nexthops = optionalNextHops.get();
                     final String groupId = nexthops.getGroupId();
                     final long groupIdValue = Long.parseLong(groupId);
-                    if (isTunnelUp) {
-                        Bucket bucket = buildBucketForDcGwLbGroup(destinationIp, dpnId, bucketId, tunnelType);
-                        LOG.trace("Added bucket {} to group {} on dpn {}.", bucket, groupId, dpnId);
-                        mdsalApiManager.addBucket(confTx, dpnId, groupIdValue, bucket);
-                    } else {
-                        LOG.trace("Removed bucketId {} from group {} on dpn {}.", bucketId, groupId, dpnId);
-                        mdsalApiManager.removeBucket(confTx, dpnId, groupIdValue, bucketId);
+                    Bucket bucket = buildBucketForDcGwLbGroup(destinationIp, dpnId, bucketId, tunnelType, isTunnelUp);
+                    LOG.trace("updated bucket {} to group {} on dpn {}.", bucket, groupId, dpnId);
+                    try {
+                        mdsalApiManager.addBucket(configTx, dpnId, groupIdValue, bucket);
+                    } catch (ExecutionException | InterruptedException e) {
+                        LOG.error("Bucket addition failed for bucket {} with exception", bucketId, e);
                     }
-                }
-            }
-        }), LOG, "Error updating load-balancing group");
+                });
+            }));
+            return futures;
+        }, RETRY_COUNT);
     }
 
-    private Bucket buildBucketForDcGwLbGroup(String ipAddress, BigInteger dpnId, int index,
-                                             Class<? extends TunnelTypeBase> tunnelType) {
+    private Bucket buildBucketForDcGwLbGroup(String ipAddress, BigInteger dpnId,
+            int index, Class<? extends TunnelTypeBase> tunnelType, boolean isTunnelUp) {
         List<Action> listAction = new ArrayList<>();
         // ActionKey 0 goes to mpls label.
         int actionKey = 1;
@@ -1222,20 +1266,24 @@ public class NexthopManager implements AutoCloseable {
             // clear off actions if there is no egress actions.
             listAction = Collections.emptyList();
         }
+        long watchPort = MDSALUtil.WATCH_PORT;
+        if (!isTunnelUp) {
+            watchPort = 0xFFFFFFFEL;
+        }
         //OVS expects a non-zero weight value for load balancing to happen in select groups
         return MDSALUtil.buildBucket(listAction, SELECT_GROUP_WEIGHT, index,
-                MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP);
+                watchPort, MDSALUtil.WATCH_GROUP);
     }
 
-    public void programDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId, String destinationIp,
+    public void programDcGwLoadBalancingGroup(BigInteger dpnId, String destinationIp,
                                               int addRemoveOrUpdate, boolean isTunnelUp,
                                               Class<? extends TunnelTypeBase> tunnelType) {
         if (NwConstants.ADD_FLOW == addRemoveOrUpdate) {
-            createDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp, tunnelType);
+            createDcGwLoadBalancingGroup(dpnId, destinationIp, tunnelType);
         } else if (NwConstants.DEL_FLOW == addRemoveOrUpdate) {
-            removeOrUpdateDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp);
+            removeDcGwLoadBalancingGroup(dpnId, destinationIp);
         } else if (NwConstants.MOD_FLOW == addRemoveOrUpdate) {
-            updateDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp, isTunnelUp, tunnelType);
+            updateDcGwLoadBalancingGroup(dpnId, destinationIp, isTunnelUp, tunnelType);
         }
     }
 }
index b03a029c2f0aeea40705f2471a9e5b8844b10d44..2d49cffa5702d53c7cbc8c68479e3a370b3c8586 100644 (file)
@@ -688,13 +688,13 @@ module odl-l3vpn {
         }
     }
 
-    container dpid-l3vpn-lb-nexthops {
+    container l3vpn-dc-gws {
         config false;
-        list dpn-lb-nexthops {
-            key "src-dp-id dst-device-id";
-            leaf src-dp-id { type uint64; }
-            leaf dst-device-id { type string; } //dpId or ip-address
-            leaf-list nexthop-key { type string; }
+        list dc-gateway {
+            key "ip-address";
+            leaf ip-address {
+                type string;
+            }
         }
     }
 
index 8e5dd06024a300a1808729565fb95bf13b3753e2..b998ce246ce0664eb487211bf6707d34e7b98675 100644 (file)
@@ -8,13 +8,13 @@
 package org.opendaylight.netvirt.vpnmanager;
 
 import static java.util.Collections.emptyList;
-import static java.util.stream.Collectors.toList;
 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
 
 import com.google.common.base.Optional;
 import com.google.common.base.Strings;
 import com.google.common.util.concurrent.ListenableFuture;
+
 import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -29,6 +29,7 @@ import java.util.concurrent.locks.ReentrantLock;
 import javax.annotation.PostConstruct;
 import javax.inject.Inject;
 import javax.inject.Singleton;
+
 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
@@ -36,7 +37,6 @@ import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
-import org.opendaylight.genius.mdsalutil.MDSALUtil;
 import org.opendaylight.genius.mdsalutil.NwConstants;
 import org.opendaylight.genius.utils.JvmGlobalLocks;
 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
@@ -64,7 +64,6 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.Tep
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelOperStatus;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelsState;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rev160406.DcGatewayIpList;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.extraroute.rds.map.extraroute.rds.DestPrefixes;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
@@ -154,7 +153,7 @@ public class TunnelInterfaceStateListener extends AsyncDataTreeChangeListenerBas
     protected void remove(InstanceIdentifier<StateTunnelList> identifier, StateTunnelList del) {
         LOG.trace("remove: Tunnel deletion---- {}", del);
         if (isGreTunnel(del)) {
-            programDcGwLoadBalancingGroup(del, NwConstants.DEL_FLOW);
+            programDcGwLoadBalancingGroup(del, NwConstants.MOD_FLOW, false);
         }
         handleTunnelEventForDPN(del, TunnelAction.TUNNEL_EP_DELETE);
     }
@@ -173,8 +172,9 @@ public class TunnelInterfaceStateListener extends AsyncDataTreeChangeListenerBas
                     update.getTunnelInterfaceName());
             return;
         }
+        boolean isTunnelUp = TunnelOperStatus.Up == update.getOperState();
         if (isGreTunnel(update)) {
-            programDcGwLoadBalancingGroup(update, NwConstants.MOD_FLOW);
+            programDcGwLoadBalancingGroup(update, NwConstants.MOD_FLOW, isTunnelUp);
         }
 
         //Remove the corresponding nexthop from the routepath under extraroute in fibentries.
@@ -230,8 +230,9 @@ public class TunnelInterfaceStateListener extends AsyncDataTreeChangeListenerBas
         if (tunOpStatus != TunnelOperStatus.Up) {
             LOG.error("add: Tunnel {} is not yet UP.", add.getTunnelInterfaceName());
         }
+        boolean isTunnelUp = TunnelOperStatus.Up == add.getOperState();
         if (isGreTunnel(add)) {
-            programDcGwLoadBalancingGroup(add, NwConstants.ADD_FLOW);
+            programDcGwLoadBalancingGroup(add, NwConstants.ADD_FLOW, isTunnelUp);
         }
         LOG.info("add: ITM Tunnel ,type {} ,added between src: {} and dest: {}",
                 fibManager.getTransportTypeStr(add.getTransportType().toString()),
@@ -564,31 +565,14 @@ public class TunnelInterfaceStateListener extends AsyncDataTreeChangeListenerBas
         return del.getTransportType() == TunnelTypeMplsOverGre.class;
     }
 
-    private void programDcGwLoadBalancingGroup(StateTunnelList tunnelState, int addOrRemove) {
+    private void programDcGwLoadBalancingGroup(StateTunnelList tunnelState, int addOrRemove, boolean isTunnelUp) {
         IpAddress dcGwIp = tunnelState.getDstInfo().getTepIp();
-        String dcGwIpAddress = dcGwIp.stringValue();
-        List<String> availableDcGws = getDcGwIps();
+        String dcGwIpAddress = String.valueOf(dcGwIp.stringValue());
         BigInteger dpId = new BigInteger(tunnelState.getSrcInfo().getTepDeviceId());
-        boolean isTunnelUp = TunnelOperStatus.Up == tunnelState.getOperState();
-        fibManager.programDcGwLoadBalancingGroup(availableDcGws, dpId, dcGwIpAddress, addOrRemove, isTunnelUp,
+        fibManager.programDcGwLoadBalancingGroup(dpId, dcGwIpAddress, addOrRemove, isTunnelUp,
                 tunnelState.getTransportType());
     }
 
-    private List<String> getDcGwIps() {
-        InstanceIdentifier<DcGatewayIpList> dcGatewayIpListid =
-                InstanceIdentifier.builder(DcGatewayIpList.class).build();
-        DcGatewayIpList dcGatewayIpListConfig =
-                MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, dcGatewayIpListid).orNull();
-        if (dcGatewayIpListConfig == null) {
-            return emptyList();
-        }
-        return dcGatewayIpListConfig.getDcGatewayIp()
-                .stream()
-                .filter(dcGwIp -> dcGwIp.getTunnnelType().equals(TunnelTypeMplsOverGre.class))
-                .map(dcGwIp -> dcGwIp.getIpAddress().stringValue()).sorted()
-                .collect(toList());
-    }
-
     private boolean isTunnelInLogicalGroup(StateTunnelList stateTunnelList) {
         String ifaceName = stateTunnelList.getTunnelInterfaceName();
         if (getTunnelType(stateTunnelList) == VpnConstants.ITMTunnelLocType.Internal.getValue()) {