VPN Imported data traffic is failed
[netvirt.git] / fibmanager / impl / src / main / java / org / opendaylight / netvirt / fibmanager / NexthopManager.java
index 8ff25aa5bc4389b14405266c0108fac717e4b579..00e65e658336a935e48f519ff89dc7ceaba08e1b 100644 (file)
@@ -7,15 +7,17 @@
  */
 package org.opendaylight.netvirt.fibmanager;
 
+import static java.util.stream.Collectors.toList;
 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
-import static org.opendaylight.netvirt.fibmanager.FibUtil.nullToEmpty;
 
 import com.google.common.base.Optional;
 import com.google.common.base.Preconditions;
-import java.math.BigInteger;
+import com.google.common.util.concurrent.ListenableFuture;
+
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -25,10 +27,12 @@ import java.util.Objects;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
-import javax.annotation.Nullable;
+
 import javax.annotation.PreDestroy;
 import javax.inject.Inject;
 import javax.inject.Singleton;
+
+import org.eclipse.jdt.annotation.Nullable;
 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
@@ -54,7 +58,6 @@ import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldVlanVid;
 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
-import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
 import org.opendaylight.netvirt.elanmanager.api.IElanService;
 import org.opendaylight.netvirt.fibmanager.api.L3VPNTransportTypes;
 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
@@ -88,6 +91,7 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.Tun
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelsState;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelListKey;
+import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rev160406.DcGatewayIpList;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelInputBuilder;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelOutput;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameInputBuilder;
@@ -123,7 +127,6 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacenciesKey;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpn;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpnBuilder;
-import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.dpid.l3vpn.lb.nexthops.DpnLbNexthops;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.lb.nexthops.Nexthops;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
@@ -134,6 +137,8 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.ni
 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 import org.opendaylight.yangtools.yang.common.RpcResult;
+import org.opendaylight.yangtools.yang.common.Uint32;
+import org.opendaylight.yangtools.yang.common.Uint64;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -144,6 +149,8 @@ public class NexthopManager implements AutoCloseable {
     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
     private static final long WAIT_TIME_TO_ACQUIRE_LOCK = 3000L;
     private static final int SELECT_GROUP_WEIGHT = 1;
+    private static final int RETRY_COUNT = 6;
+    private static final String NEXTHOPMANAGER_JOB_KEY_PREFIX = "NextHopManager";
 
     private final DataBroker dataBroker;
     private final ManagedNewTransactionRunner txRunner;
@@ -212,15 +219,15 @@ public class NexthopManager implements AutoCloseable {
         }
     }
 
-    private String getNextHopKey(long vpnId, String ipAddress) {
+    private String getNextHopKey(Uint32 vpnId, String ipAddress) {
         return "nexthop." + vpnId + ipAddress;
     }
 
-    String getRemoteSelectGroupKey(long vpnId, String ipAddress) {
+    String getRemoteSelectGroupKey(Uint32 vpnId, String ipAddress) {
         return "remote.ecmp.nexthop." + vpnId + ipAddress;
     }
 
-    String getLocalSelectGroupKey(long vpnId, String ipAddress) {
+    String getLocalSelectGroupKey(Uint32 vpnId, String ipAddress) {
         return "local.ecmp.nexthop." + vpnId + ipAddress;
     }
 
@@ -236,9 +243,10 @@ public class NexthopManager implements AutoCloseable {
         try {
             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
             RpcResult<AllocateIdOutput> rpcResult = result.get();
-            return rpcResult.getResult().getIdValue();
+            return rpcResult.getResult().getIdValue().toJava();
         } catch (NullPointerException | InterruptedException | ExecutionException e) {
-            LOG.trace("", e);
+            // FIXME: NPEs should not be caught but rather their root cause should be eliminated
+            LOG.trace("Failed to allocate {}", getIdInput, e);
         }
         return 0;
     }
@@ -259,7 +267,8 @@ public class NexthopManager implements AutoCloseable {
     }
 
     protected List<ActionInfo> getEgressActionsForInterface(final String ifName, int actionKey,
-                                                            boolean isTunnelInterface) {
+                                                            boolean isTunnelInterface,
+                                                            Uint32 vpnId, String destIpPrefix) {
         List<Action> actions;
         try {
             if (isTunnelInterface && interfaceManager.isItmDirectTunnelsEnabled()) {
@@ -271,22 +280,22 @@ public class NexthopManager implements AutoCloseable {
                             ifName, rpcResult.getErrors());
                     return Collections.emptyList();
                 } else {
-                    actions = rpcResult.getResult().getAction();
+                    actions = rpcResult.getResult().nonnullAction();
                 }
             } else {
                 RpcResult<GetEgressActionsForInterfaceOutput> rpcResult = odlInterfaceRpcService
                         .getEgressActionsForInterface(new GetEgressActionsForInterfaceInputBuilder()
                                 .setIntfName(ifName).build()).get();
                 if (!rpcResult.isSuccessful()) {
-                    LOG.error("RPC Call to Get egress vm actions for interface {} returned with Errors {}",
-                            ifName, rpcResult.getErrors());
+                    LOG.error("RPC Call to Get egress vm actions for interface {} vpnId {} ipPrefix {} returned with "
+                                    + "Errors {}", ifName, vpnId, destIpPrefix, rpcResult.getErrors());
                     return Collections.emptyList();
                 } else {
-                    actions = rpcResult.getResult().getAction();
+                    actions = rpcResult.getResult().nonnullAction();
                 }
             }
             List<ActionInfo> listActionInfo = new ArrayList<>();
-            for (Action action : nullToEmpty(actions)) {
+            for (Action action : actions) {
                 actionKey = action.key().getOrder() + actionKey;
                 org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action
                     actionClass = action.getAction();
@@ -298,30 +307,31 @@ public class NexthopManager implements AutoCloseable {
                 } else if (actionClass instanceof SetFieldCase) {
                     if (((SetFieldCase) actionClass).getSetField().getVlanMatch() != null) {
                         int vlanVid = ((SetFieldCase) actionClass).getSetField().getVlanMatch()
-                            .getVlanId().getVlanId().getValue();
+                            .getVlanId().getVlanId().getValue().toJava();
                         listActionInfo.add(new ActionSetFieldVlanVid(actionKey, vlanVid));
                     }
                 } else if (actionClass instanceof NxActionResubmitRpcAddGroupCase) {
-                    Short tableId = ((NxActionResubmitRpcAddGroupCase) actionClass).getNxResubmit().getTable();
+                    Short tableId = ((NxActionResubmitRpcAddGroupCase) actionClass).getNxResubmit().getTable().toJava();
                     listActionInfo.add(new ActionNxResubmit(actionKey, tableId));
                 } else if (actionClass instanceof NxActionRegLoadNodesNodeTableFlowApplyActionsCase) {
                     NxRegLoad nxRegLoad =
                         ((NxActionRegLoadNodesNodeTableFlowApplyActionsCase) actionClass).getNxRegLoad();
                     listActionInfo.add(new ActionRegLoad(actionKey, NxmNxReg6.class,
-                        nxRegLoad.getDst().getStart(), nxRegLoad.getDst().getEnd(),
+                        nxRegLoad.getDst().getStart().toJava(), nxRegLoad.getDst().getEnd().toJava(),
                         nxRegLoad.getValue().longValue()));
                 }
             }
             return listActionInfo;
-        } catch (InterruptedException | ExecutionException e) {
-            LOG.warn("Exception when egress actions for interface {}", ifName, e);
+        } catch (InterruptedException | ExecutionException | NullPointerException e) {
+            LOG.error("Exception when egress actions for interface {} isTunnel {} vpnId {} ipPrefix {}", ifName,
+                    isTunnelInterface, vpnId, destIpPrefix, e);
         }
         LOG.warn("Exception when egress actions for interface {}", ifName);
         return Collections.emptyList();
     }
 
     @Nullable
-    protected String getTunnelInterfaceName(BigInteger srcDpId, BigInteger dstDpId) {
+    protected String getTunnelInterfaceName(Uint64 srcDpId, Uint64 dstDpId) {
         Class<? extends TunnelTypeBase> tunType = getReqTunType(getReqTransType().toUpperCase(Locale.getDefault()));
         Future<RpcResult<GetTunnelInterfaceNameOutput>> result;
         try {
@@ -343,7 +353,7 @@ public class NexthopManager implements AutoCloseable {
     }
 
     @Nullable
-    protected String getTunnelInterfaceName(BigInteger srcDpId, org.opendaylight.yang.gen.v1.urn.ietf.params
+    protected String getTunnelInterfaceName(Uint64 srcDpId, org.opendaylight.yang.gen.v1.urn.ietf.params
         .xml.ns.yang.ietf.inet.types.rev130715.IpAddress dstIp, Class<? extends TunnelTypeBase> tunnelType) {
         Future<RpcResult<GetInternalOrExternalInterfaceNameOutput>> result;
         try {
@@ -361,13 +371,13 @@ public class NexthopManager implements AutoCloseable {
                 return rpcResult.getResult().getInterfaceName();
             }
         } catch (InterruptedException | ExecutionException e) {
-            LOG.warn("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstIp, e);
+            LOG.error("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstIp, e);
         }
         return null;
     }
 
 
-    public long getLocalNextHopGroup(long vpnId,
+    public long getLocalNextHopGroup(Uint32 vpnId,
             String ipNextHopAddress) {
         long groupId = createNextHopPointer(getNextHopKey(vpnId, ipNextHopAddress));
         if (groupId == FibConstants.INVALID_GROUP_ID) {
@@ -376,7 +386,7 @@ public class NexthopManager implements AutoCloseable {
         return groupId;
     }
 
-    public long getLocalSelectGroup(long vpnId,
+    public long getLocalSelectGroup(Uint32 vpnId,
             String ipNextHopAddress) {
         long groupId = createNextHopPointer(getLocalSelectGroupKey(vpnId, ipNextHopAddress));
         if (groupId == FibConstants.INVALID_GROUP_ID) {
@@ -385,10 +395,11 @@ public class NexthopManager implements AutoCloseable {
         return groupId;
     }
 
-    public long createLocalNextHop(long vpnId, BigInteger dpnId, String ifName,
+    public long createLocalNextHop(Uint32 vpnId, Uint64 dpnId, String ifName,
                                    String primaryIpAddress, String currDestIpPrefix,
-                                   String gwMacAddress) {
-        String vpnName = fibUtil.getVpnNameFromId(vpnId);
+                                   String gwMacAddress, Uint32 parentVpnId) {
+        //For VPN Imported routes, getting VPN Instance name using parentVpnId
+        String vpnName = parentVpnId != null ? fibUtil.getVpnNameFromId(parentVpnId) : fibUtil.getVpnNameFromId(vpnId);
         if (vpnName == null) {
             return 0;
         }
@@ -411,7 +422,6 @@ public class NexthopManager implements AutoCloseable {
                     if (nexthop == null) {
                         String encMacAddress = macAddress == null
                                 ? fibUtil.getMacAddressFromPrefix(ifName, vpnName, primaryIpAddress) : macAddress;
-                        List<BucketInfo> listBucketInfo = new ArrayList<>();
                         List<ActionInfo> listActionInfo = new ArrayList<>();
                         int actionKey = 0;
                         // MAC re-write
@@ -426,12 +436,19 @@ public class NexthopManager implements AutoCloseable {
                                     new MacAddress(encMacAddress)));
                             // listActionInfo.add(0, new ActionPopMpls());
                         } else {
-                            // FIXME: Log message here.
-                            LOG.debug("mac address for new local nexthop is null");
+                            LOG.error("mac address for new local nexthop group {} is null for vpnId {}, prefix {}, "
+                                    + "ifName {} on dpn {}", groupId, vpnId, primaryIpAddress, ifName, dpnId);
+                        }
+                        List<ActionInfo> nhActionInfoList = getEgressActionsForInterface(ifName, actionKey, false,
+                                vpnId, currDestIpPrefix);
+                        if (nhActionInfoList.isEmpty()) {
+                            LOG.error("createLocalNextHop: Skipping, Empty list of egress actions received for "
+                                    + "interface {} on dpn {} for vpn {} prefix {}", ifName, dpnId, vpnId,
+                                    currDestIpPrefix);
                         }
-                        listActionInfo.addAll(getEgressActionsForInterface(ifName, actionKey, false));
+                        listActionInfo.addAll(nhActionInfoList);
                         BucketInfo bucket = new BucketInfo(listActionInfo);
-
+                        List<BucketInfo> listBucketInfo = new ArrayList<>();
                         listBucketInfo.add(bucket);
                         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, groupId, primaryIpAddress,
                                 GroupTypes.GroupAll, listBucketInfo);
@@ -470,7 +487,7 @@ public class NexthopManager implements AutoCloseable {
         return groupId;
     }
 
-    private void installGroupOnDpn(long groupId, BigInteger dpnId, String groupName, List<BucketInfo> bucketsInfo,
+    private void installGroupOnDpn(long groupId, Uint64 dpnId, String groupName, List<BucketInfo> bucketsInfo,
                                      String nextHopKey, GroupTypes groupType) {
         NodeRef nodeRef = FibUtil.buildNodeRef(dpnId);
         Buckets buckets = FibUtil.buildBuckets(bucketsInfo);
@@ -494,7 +511,7 @@ public class NexthopManager implements AutoCloseable {
         }
     }
 
-    protected void addVpnNexthopToDS(BigInteger dpnId, long vpnId, String primaryIpAddr,
+    protected void addVpnNexthopToDS(Uint64 dpnId, Uint32 vpnId, String primaryIpAddr,
                                      String currIpAddr, long egressPointer) {
         InstanceIdentifierBuilder<VpnNexthops> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId));
@@ -516,7 +533,7 @@ public class NexthopManager implements AutoCloseable {
 
     }
 
-    protected InstanceIdentifier<IpAdjacencies> getVpnNextHopIpPrefixIdentifier(long vpnId, String primaryIpAddress,
+    protected InstanceIdentifier<IpAdjacencies> getVpnNextHopIpPrefixIdentifier(Uint32 vpnId, String primaryIpAddress,
                                                                                 String ipPrefix) {
         InstanceIdentifier<IpAdjacencies> id = InstanceIdentifier.builder(L3nexthop.class)
                 .child(VpnNexthops.class, new VpnNexthopsKey(vpnId))
@@ -526,7 +543,7 @@ public class NexthopManager implements AutoCloseable {
     }
 
     @Nullable
-    protected VpnNexthop getVpnNexthop(long vpnId, String ipAddress) {
+    protected VpnNexthop getVpnNexthop(Uint32 vpnId, String ipAddress) {
 
         // check if vpn node is there
         InstanceIdentifierBuilder<VpnNexthops> idBuilder =
@@ -536,7 +553,7 @@ public class NexthopManager implements AutoCloseable {
         Optional<VpnNexthops> vpnNexthops = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
         if (vpnNexthops.isPresent()) {
             // get nexthops list for vpn
-            List<VpnNexthop> nexthops = nullToEmpty(vpnNexthops.get().getVpnNexthop());
+            List<VpnNexthop> nexthops = vpnNexthops.get().nonnullVpnNexthop();
             for (VpnNexthop nexthop : nexthops) {
                 if (Objects.equals(nexthop.getIpAddress(), ipAddress)) {
                     // return nexthop
@@ -550,7 +567,7 @@ public class NexthopManager implements AutoCloseable {
     }
 
     @Nullable
-    public AdjacencyResult getRemoteNextHopPointer(BigInteger remoteDpnId, long vpnId, String prefixIp,
+    public AdjacencyResult getRemoteNextHopPointer(Uint64 remoteDpnId, Uint32 vpnId, String prefixIp,
             @Nullable String nextHopIp, Class<? extends TunnelTypeBase> tunnelType) {
         String egressIfName = null;
         LOG.trace("getRemoteNextHopPointer: input [remoteDpnId {}, vpnId {}, prefixIp {}, nextHopIp {} ]", remoteDpnId,
@@ -577,13 +594,7 @@ public class NexthopManager implements AutoCloseable {
                 prefixIp) : null;
     }
 
-    @Nullable
-    public BigInteger getDpnForPrefix(long vpnId, String prefixIp) {
-        VpnNexthop vpnNexthop = getVpnNexthop(vpnId, prefixIp);
-        return vpnNexthop == null ? null : vpnNexthop.getDpnId();
-    }
-
-    private void removeVpnNexthopFromDS(long vpnId, String ipPrefix) {
+    private void removeVpnNexthopFromDS(Uint32 vpnId, String ipPrefix) {
 
         InstanceIdentifierBuilder<VpnNexthop> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId))
@@ -594,18 +605,18 @@ public class NexthopManager implements AutoCloseable {
         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
     }
 
-    public void removeLocalNextHop(BigInteger dpnId, Long vpnId, String primaryIpAddress, String currDestIpPrefix) {
+    public void removeLocalNextHop(Uint64 dpnId, Uint32 vpnId, String primaryIpAddress, String currDestIpPrefix) {
         String nextHopLockStr = vpnId + primaryIpAddress;
         try {
             if (FibUtil.lockCluster(lockManager, nextHopLockStr, WAIT_TIME_TO_ACQUIRE_LOCK)) {
                 VpnNexthop nh = getVpnNexthop(vpnId, primaryIpAddress);
                 if (nh != null) {
-                    List<IpAdjacencies> prefixesList = new ArrayList<>(nullToEmpty(nh.getIpAdjacencies()));
+                    List<IpAdjacencies> prefixesList = new ArrayList<>(nh.nonnullIpAdjacencies());
                     IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currDestIpPrefix).build();
                     prefixesList.remove(prefix);
                     if (prefixesList.isEmpty()) { //remove the group only if there are no more flows using this group
-                        GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, nh.getEgressPointer(),
-                                primaryIpAddress, GroupTypes.GroupAll, Collections.EMPTY_LIST);
+                        GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, nh.getEgressPointer().toJava(),
+                                primaryIpAddress, GroupTypes.GroupAll, Collections.emptyList());
                         // remove Group ...
                         mdsalApiManager.removeGroup(groupEntity);
                         //update MD-SAL DS
@@ -738,7 +749,7 @@ public class NexthopManager implements AutoCloseable {
     // TODO Clean up the exception handling
     @SuppressWarnings("checkstyle:IllegalCatch")
     @Nullable
-    private String getTunnelRemoteNextHopPointer(BigInteger remoteDpnId, String nextHopIp,
+    private String getTunnelRemoteNextHopPointer(Uint64 remoteDpnId, String nextHopIp,
                                                  Class<? extends TunnelTypeBase> tunnelType) {
         if (nextHopIp != null && !nextHopIp.isEmpty()) {
             try {
@@ -755,7 +766,7 @@ public class NexthopManager implements AutoCloseable {
         return null;
     }
 
-    private String getExtPortRemoteNextHopPointer(BigInteger remoteDpnId, ElanInstance elanInstance) {
+    private String getExtPortRemoteNextHopPointer(Uint64 remoteDpnId, ElanInstance elanInstance) {
         return elanService.getExternalElanInterface(elanInstance.getElanInstanceName(), remoteDpnId);
     }
 
@@ -775,7 +786,7 @@ public class NexthopManager implements AutoCloseable {
         return Tunnel.class;
     }
 
-    private ElanInstance getElanInstanceForPrefix(long vpnId, String prefixIp) {
+    private ElanInstance getElanInstanceForPrefix(Uint32 vpnId, String prefixIp) {
         ElanInstance elanInstance = null;
         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, prefixIp);
         if (prefix != null) {
@@ -840,7 +851,7 @@ public class NexthopManager implements AutoCloseable {
         }
     }
 
-    protected long setupLoadBalancingNextHop(Long parentVpnId, BigInteger dpnId,
+    protected long setupLoadBalancingNextHop(Uint32 parentVpnId, Uint64 dpnId,
             String destPrefix, List<BucketInfo> localBucketInfo, List<BucketInfo> remoteBucketInfo) {
         long remoteGroupId = createNextHopPointer(getRemoteSelectGroupKey(parentVpnId, destPrefix));
         if (remoteGroupId == FibConstants.INVALID_GROUP_ID) {
@@ -884,7 +895,7 @@ public class NexthopManager implements AutoCloseable {
         return remoteGroupId;
     }
 
-    protected void deleteLoadBalancingNextHop(Long parentVpnId, BigInteger dpnId, String destPrefix) {
+    protected void deleteLoadBalancingNextHop(Uint32 parentVpnId, Uint64 dpnId, String destPrefix) {
         long remoteGroupId = createNextHopPointer(getRemoteSelectGroupKey(parentVpnId, destPrefix));
         if (remoteGroupId == FibConstants.INVALID_GROUP_ID) {
             LOG.error("Unable to allocate/retrieve remote groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
@@ -909,7 +920,7 @@ public class NexthopManager implements AutoCloseable {
         });
     }
 
-    long createNextHopGroups(Long vpnId, String rd, BigInteger dpnId, VrfEntry vrfEntry,
+    long createNextHopGroups(Uint32 vpnId, String rd, Uint64 dpnId, VrfEntry vrfEntry,
             @Nullable Routes routes, List<Routes> vpnExtraRoutes) {
         List<BucketInfo> localBucketInfo = new ArrayList<>();
         List<Routes> clonedVpnExtraRoutes  = new ArrayList<>(vpnExtraRoutes);
@@ -923,156 +934,208 @@ public class NexthopManager implements AutoCloseable {
             vrfEntry.getDestPrefix(), localBucketInfo, remoteBucketInfo);
     }
 
-    private List<BucketInfo> getBucketsForLocalNexthop(Long vpnId, BigInteger dpnId,
+    private List<BucketInfo> getBucketsForLocalNexthop(Uint32 vpnId, Uint64 dpnId,
             VrfEntry vrfEntry, Routes routes) {
+        @Nullable List<String> nexthopIpList = routes.getNexthopIpList();
         if (LOG.isDebugEnabled()) {
             LOG.debug("NexthopManager.getBucketsForLocalNexthop invoked with vpnId {} dpnId {} "
                             + " vrfEntry.routePaths {}, routes.nexthopList {}", vpnId, dpnId, vrfEntry.getRoutePaths(),
-                    routes.getNexthopIpList());
+                nexthopIpList);
         }
         List<BucketInfo> listBucketInfo = new CopyOnWriteArrayList<>();
-        nullToEmpty(routes.getNexthopIpList()).parallelStream().forEach(nextHopIp -> {
-            String localNextHopIP;
-            if (isIpv4Address(nextHopIp)) {
-                localNextHopIP = nextHopIp + NwConstants.IPV4PREFIX;
-            } else {
-                localNextHopIP = nextHopIp + NwConstants.IPV6PREFIX;
-            }
-            Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
-            if (localNextHopInfo != null) {
-                long groupId = getLocalNextHopGroup(vpnId, localNextHopIP);
-                if (groupId == FibConstants.INVALID_GROUP_ID) {
-                    LOG.error("Unable to allocate groupId for vpnId {} , prefix {} , interface {}", vpnId,
-                            vrfEntry.getDestPrefix(), localNextHopInfo.getVpnInterfaceName());
-                    return;
+        if (nexthopIpList != null) {
+            nexthopIpList.parallelStream().forEach(nextHopIp -> {
+                String localNextHopIP;
+                if (isIpv4Address(nextHopIp)) {
+                    localNextHopIP = nextHopIp + NwConstants.IPV4PREFIX;
+                } else {
+                    localNextHopIP = nextHopIp + NwConstants.IPV6PREFIX;
                 }
-                List<ActionInfo> actionsInfos =
+                Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
+                if (localNextHopInfo != null) {
+                    long groupId = getLocalNextHopGroup(vpnId, localNextHopIP);
+                    if (groupId == FibConstants.INVALID_GROUP_ID) {
+                        LOG.error("Unable to allocate groupId for vpnId {} , prefix {} , interface {}", vpnId,
+                            vrfEntry.getDestPrefix(), localNextHopInfo.getVpnInterfaceName());
+                        return;
+                    }
+                    List<ActionInfo> actionsInfos =
                         Collections.singletonList(new ActionGroup(groupId));
-                BucketInfo bucket = new BucketInfo(actionsInfos);
-                bucket.setWeight(1);
-                listBucketInfo.add(bucket);
-            }
-        });
+                    BucketInfo bucket = new BucketInfo(actionsInfos);
+                    bucket.setWeight(1);
+                    listBucketInfo.add(bucket);
+                }
+            });
+        }
         LOG.trace("LOCAL: listbucket {}, vpnId {}, dpnId {}, routes {}", listBucketInfo, vpnId, dpnId, routes);
         return listBucketInfo;
     }
 
-    private List<BucketInfo> getBucketsForRemoteNexthop(Long vpnId, BigInteger dpnId, VrfEntry vrfEntry, String rd,
+    private List<BucketInfo> getBucketsForRemoteNexthop(Uint32 vpnId, Uint64 dpnId, VrfEntry vrfEntry, String rd,
             List<Routes> vpnExtraRoutes) {
         List<BucketInfo> listBucketInfo = new ArrayList<>();
         Map<String, List<ActionInfo>> egressActionMap = new HashMap<>();
-        vpnExtraRoutes.forEach(vpnExtraRoute -> nullToEmpty(vpnExtraRoute.getNexthopIpList()).forEach(nextHopIp -> {
-            String nextHopPrefixIp;
-            if (isIpv4Address(nextHopIp)) {
-                nextHopPrefixIp = nextHopIp + NwConstants.IPV4PREFIX;
-            } else {
-                nextHopPrefixIp = nextHopIp + NwConstants.IPV6PREFIX;
-            }
-            List<String> tepIpAddresses = fibUtil.getNextHopAddresses(rd, nextHopPrefixIp);
-            if (tepIpAddresses.isEmpty()) {
-                return;
-            }
-            // There would be only one nexthop address for a VM ip which would be the tep Ip
-            String tepIp = tepIpAddresses.get(0);
-            AdjacencyResult adjacencyResult = getRemoteNextHopPointer(dpnId, vpnId,
+        vpnExtraRoutes.stream().filter(vpnExtraRoute -> vpnExtraRoute.getNexthopIpList() != null).forEach(
+            vpnExtraRoute -> vpnExtraRoute.getNexthopIpList().forEach(nextHopIp -> {
+                String nextHopPrefixIp;
+                if (isIpv4Address(nextHopIp)) {
+                    nextHopPrefixIp = nextHopIp + NwConstants.IPV4PREFIX;
+                } else {
+                    nextHopPrefixIp = nextHopIp + NwConstants.IPV6PREFIX;
+                }
+                List<String> tepIpAddresses = fibUtil.getNextHopAddresses(rd, nextHopPrefixIp);
+                if (tepIpAddresses.isEmpty()) {
+                    return;
+                }
+                // There would be only one nexthop address for a VM ip which would be the tep Ip
+                String tepIp = tepIpAddresses.get(0);
+                AdjacencyResult adjacencyResult = getRemoteNextHopPointer(dpnId, vpnId,
                     vrfEntry.getDestPrefix(), tepIp, TunnelTypeVxlan.class);
-            if (adjacencyResult == null) {
-                return;
-            }
-            String egressInterface = adjacencyResult.getInterfaceName();
-            if (!FibUtil.isTunnelInterface(adjacencyResult)) {
-                return;
-            }
-            Class<? extends TunnelTypeBase> tunnelType = VpnExtraRouteHelper.getTunnelType(itmManager, egressInterface);
-            StateTunnelList ifState = null;
-            try {
-                ifState = fibUtil.getTunnelState(egressInterface);
-                if (ifState == null || ifState.getOperState() != TunnelOperStatus.Up) {
-                    LOG.trace("Tunnel is not up for interface {}", egressInterface);
+                if (adjacencyResult == null) {
+                    return;
+                }
+                String egressInterface = adjacencyResult.getInterfaceName();
+                if (!FibUtil.isTunnelInterface(adjacencyResult)) {
                     return;
                 }
-            } catch (ReadFailedException e) {
-                LOG.error("getBucketsForRemoteNexthop: error in fetching tunnel state for interface {}",
+                Class<? extends TunnelTypeBase> tunnelType =
+                    VpnExtraRouteHelper.getTunnelType(itmManager, egressInterface);
+                StateTunnelList ifState = null;
+                try {
+                    ifState = fibUtil.getTunnelState(egressInterface);
+                    if (ifState == null || ifState.getOperState() != TunnelOperStatus.Up) {
+                        LOG.trace("Tunnel is not up for interface {}", egressInterface);
+                        return;
+                    }
+                } catch (ReadFailedException e) {
+                    LOG.error("getBucketsForRemoteNexthop: error in fetching tunnel state for interface {}",
                         egressInterface, e);
-                return;
-            }
-            if (!TunnelTypeVxlan.class.equals(tunnelType)) {
-                return;
-            }
-            Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).get();
-            Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, nextHopPrefixIp);
-            if (prefixInfo == null) {
-                LOG.error("No prefix info found for prefix {} in rd {} for VPN {}", nextHopPrefixIp, rd,
+                    return;
+                }
+                if (!TunnelTypeVxlan.class.equals(tunnelType)) {
+                    return;
+                }
+                Uint32 label = FibUtil.getLabelFromRoutePaths(vrfEntry).get();
+                Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, nextHopPrefixIp);
+                if (prefixInfo == null) {
+                    LOG.error("No prefix info found for prefix {} in rd {} for VPN {}", nextHopPrefixIp, rd,
                         vpnId);
-                return;
-            }
-            BigInteger tunnelId;
-            if (FibUtil.isVxlanNetwork(prefixInfo.getNetworkType())) {
-                tunnelId = BigInteger.valueOf(prefixInfo.getSegmentationId());
-            } else {
-                LOG.warn("Network is not of type VXLAN for prefix {}."
+                    return;
+                }
+                Uint32 tunnelId;
+                if (FibUtil.isVxlanNetwork(prefixInfo.getNetworkType())) {
+                    tunnelId = prefixInfo.getSegmentationId();
+                } else {
+                    LOG.warn("Network is not of type VXLAN for prefix {}."
                         + "Going with default Lport Tag.", prefixInfo.toString());
-                tunnelId = BigInteger.valueOf(label);
-            }
-            List<ActionInfo> actionInfos = new ArrayList<>();
-            actionInfos.add(new ActionSetFieldTunnelId(tunnelId));
-            String ifName = prefixInfo.getVpnInterfaceName();
-            String vpnName = fibUtil.getVpnNameFromId(vpnId);
-            if (vpnName == null) {
-                return;
-            }
-            String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, nextHopPrefixIp);
-            actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
+                    tunnelId = label;
+                }
+                List<ActionInfo> actionInfos = new ArrayList<>();
+                actionInfos.add(new ActionSetFieldTunnelId(Uint64.valueOf(tunnelId.longValue())));
+                String ifName = prefixInfo.getVpnInterfaceName();
+                String vpnName = fibUtil.getVpnNameFromId(vpnId);
+                if (vpnName == null) {
+                    return;
+                }
+                String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, nextHopPrefixIp);
+                actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
                     new MacAddress(macAddress)));
-            List<ActionInfo> egressActions;
-            if (egressActionMap.containsKey(egressInterface)) {
-                egressActions = egressActionMap.get(egressInterface);
-            } else {
-                egressActions = getEgressActionsForInterface(egressInterface, actionInfos.size(), true);
-                egressActionMap.put(egressInterface, egressActions);
-            }
-            if (egressActions.isEmpty()) {
-                LOG.error("Failed to retrieve egress action for prefix {} route-paths {}"
-                        + " interface {}." + " Aborting remote FIB entry creation.",
+                List<ActionInfo> egressActions;
+                if (egressActionMap.containsKey(egressInterface)) {
+                    egressActions = egressActionMap.get(egressInterface);
+                } else {
+                    egressActions = getEgressActionsForInterface(egressInterface, actionInfos.size(),
+                            true, vpnId, vrfEntry.getDestPrefix());
+                    if (egressActions.isEmpty()) {
+                        LOG.error("Skipping getBucketsForRemoteNexthop: Empty list of egress actions received for "
+                                        + "interface {} on dpn {} for vpn {} prefix {} nextHop {}", ifName, dpnId,
+                                vpnId, vrfEntry.getDestPrefix(), nextHopPrefixIp);
+                    }
+                    egressActionMap.put(egressInterface, egressActions);
+                }
+                if (egressActions.isEmpty()) {
+                    LOG.error("Failed to retrieve egress action for prefix {} route-paths {}"
+                            + " interface {}." + " Aborting remote FIB entry creation.",
                         vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), egressInterface);
-            }
-            actionInfos.addAll(egressActions);
-            BucketInfo bucket = new BucketInfo(actionInfos);
-            bucket.setWeight(1);
-            listBucketInfo.add(bucket);
-        }));
+                }
+                actionInfos.addAll(egressActions);
+                BucketInfo bucket = new BucketInfo(actionInfos);
+                bucket.setWeight(1);
+                listBucketInfo.add(bucket);
+            }));
         LOG.trace("LOCAL: listbucket {}, rd {}, dpnId {}, routes {}", listBucketInfo, rd, dpnId, vpnExtraRoutes);
         return listBucketInfo;
     }
 
-    public void createDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId, String destinationIp,
+    public void createDcGwLoadBalancingGroup(Uint64 dpnId, String destinationIp,
                                              Class<? extends TunnelTypeBase> tunnelType) {
-        Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
-        int noOfDcGws = availableDcGws.size();
-        if (noOfDcGws == 1) {
-            LOG.trace("There are no enough DC GateWays {} present to program LB group", availableDcGws);
-            return;
-        }
-        // TODO : Place the logic to construct all possible DC-GW combination here.
-        String groupIdKey = FibUtil.getGreLbGroupKey(availableDcGws);
-        Long groupId = createNextHopPointer(groupIdKey);
-        List<Bucket> listBucket = new ArrayList<>();
-        for (int index = 0; index < noOfDcGws; index++) {
-            if (isTunnelUp(availableDcGws.get(index), dpnId, tunnelType)) {
-                listBucket.add(buildBucketForDcGwLbGroup(availableDcGws.get(index), dpnId, index, tunnelType));
-            }
+        jobCoordinator.enqueueJob(getJobKey(dpnId), () -> {
+            List<ListenableFuture<Void>> futures = new ArrayList<>();
+            futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operationalTx -> {
+                synchronized (getDcGateWaySyncKey(destinationIp)) {
+                    FibUtil.addL3vpnDcGateWay(destinationIp, operationalTx);
+                }
+                futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, configTx -> {
+                    List<String> availableDcGws = getDcGwIps();
+                    Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
+                    int noOfDcGws = availableDcGws.size();
+                    if (noOfDcGws == 1) {
+                        LOG.trace("There are no enough DC GateWays {} present to program LB group", availableDcGws);
+                        return;
+                    }
+                    if (availableDcGws.contains(destinationIp)) {
+                        availableDcGws.remove(destinationIp);
+                    }
+                    availableDcGws.forEach(dcGwIp -> {
+                        List<String> dcGws = Arrays.asList(dcGwIp, destinationIp);
+                        Collections.sort(dcGws);
+                        String groupIdKey = FibUtil.getGreLbGroupKey(dcGws);
+                        Long groupId = createNextHopPointer(groupIdKey);
+                        List<Bucket> listBucket = new ArrayList<>();
+                        for (int index = 0; index < dcGws.size(); index++) {
+                            if (isTunnelUp(dcGws.get(index), dpnId, tunnelType)) {
+                                listBucket.add(buildBucketForDcGwLbGroup(dcGws.get(index),
+                                        dpnId, index, tunnelType, true));
+                            }
+                        }
+                        Group group = MDSALUtil.buildGroup(groupId, groupIdKey, GroupTypes.GroupSelect,
+                                        MDSALUtil.buildBucketLists(listBucket));
+                        mdsalApiManager.addGroup(configTx, dpnId, group);
+                        FibUtil.updateLbGroupInfo(dpnId, groupIdKey, groupId.toString(), operationalTx);
+                        LOG.trace("LB group {} towards DC-GW installed on dpn {}. Group - {}",
+                                groupIdKey, dpnId, group);
+                    });
+                }));
+            }));
+            return futures;
+        }, RETRY_COUNT);
+    }
+
+    private String getJobKey(Uint64 dpnId) {
+        return new StringBuilder().append(NEXTHOPMANAGER_JOB_KEY_PREFIX).append(dpnId).toString();
+    }
+
+    private String getDcGateWaySyncKey(String destinationIp) {
+        String mutex = new StringBuilder().append("L3vpncDcGateWay").append(destinationIp).toString();
+        return mutex.intern();
+    }
+
+    private List<String> getDcGwIps() {
+        InstanceIdentifier<DcGatewayIpList> dcGatewayIpListid =
+                InstanceIdentifier.builder(DcGatewayIpList.class).build();
+        DcGatewayIpList dcGatewayIpListConfig =
+                MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, dcGatewayIpListid).orNull();
+        if (dcGatewayIpListConfig == null) {
+            return Collections.emptyList();
         }
-        Group group = MDSALUtil.buildGroup(groupId, groupIdKey, GroupTypes.GroupSelect,
-                        MDSALUtil.buildBucketLists(listBucket));
-        ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
-            confTx -> mdsalApiManager.addGroup(confTx, dpnId, group)), LOG, "Error adding load-balancing group");
-        ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
-            operTx -> FibUtil.updateLbGroupInfo(dpnId, destinationIp, groupIdKey, groupId.toString(), operTx)), LOG,
-            "Error updating load-balancing group info");
-        LOG.trace("LB group {} towards DC-GW installed on dpn {}. Group - {}", groupIdKey, dpnId, group);
+        return dcGatewayIpListConfig.getDcGatewayIp()
+                .stream()
+                .filter(dcGwIp -> dcGwIp.getTunnnelType().equals(TunnelTypeMplsOverGre.class))
+                .map(dcGwIp -> dcGwIp.getIpAddress().stringValue()).sorted()
+                .collect(toList());
     }
 
-    private boolean isTunnelUp(String dcGwIp, BigInteger dpnId, Class<? extends TunnelTypeBase> tunnelType) {
+    private boolean isTunnelUp(String dcGwIp, Uint64 dpnId, Class<? extends TunnelTypeBase> tunnelType) {
         String tunnelName = getTunnelRemoteNextHopPointer(dpnId, dcGwIp, tunnelType);
         if (tunnelName != null) {
             InstanceIdentifier<StateTunnelList> tunnelStateId =
@@ -1097,7 +1160,7 @@ public class NexthopManager implements AutoCloseable {
                 LOG.warn("RPC Call to Get egress actions for interface {} returned with Errors {}",
                         interfaceName, rpcResult.getErrors());
             } else {
-                actions = nullToEmpty(rpcResult.getResult().getAction());
+                actions = rpcResult.getResult().nonnullAction();
             }
         } catch (InterruptedException | ExecutionException e) {
             LOG.warn("Exception when egress actions for interface {}", interfaceName, e);
@@ -1106,29 +1169,68 @@ public class NexthopManager implements AutoCloseable {
     }
 
     /**
-     * This method is invoked when the tunnel state is removed from DS.
-     * If the there is just one DC-GW left in configuration then the LB groups can be deleted.
-     * Otherwise, the groups are just updated.
+     * This method is invoked when the neighbor is removed from DS.
+     * All the LB groups which point to the given destination will be deleted.
      */
-    public void removeOrUpdateDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId,
+    public void removeDcGwLoadBalancingGroup(Uint64 dpnId,
             String destinationIp) {
-        Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
-        ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
-            ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
-                int noOfDcGws = availableDcGws.size();
-                // If availableDcGws does not contain the destination Ip it means this is a configuration delete.
-                if (!availableDcGws.contains(destinationIp)) {
-                    availableDcGws.add(destinationIp);
-                    Collections.sort(availableDcGws);
-                }
-                // TODO : Place the logic to construct all possible DC-GW combination here.
-                int bucketId = availableDcGws.indexOf(destinationIp);
-                Optional<DpnLbNexthops> dpnLbNextHops = fibUtil.getDpnLbNexthops(dpnId, destinationIp);
-                if (!dpnLbNextHops.isPresent()) {
-                    return;
-                }
-                List<String> nextHopKeys = nullToEmpty(dpnLbNextHops.get().getNexthopKey());
-                for (String nextHopKey : nextHopKeys) {
+        jobCoordinator.enqueueJob(getJobKey(dpnId), () -> {
+            List<String> availableDcGws = fibUtil.getL3VpnDcGateWays();
+            if (availableDcGws.contains(destinationIp)) {
+                availableDcGws.remove(destinationIp);
+            }
+            List<ListenableFuture<Void>> futures = new ArrayList<>();
+            futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operationalTx -> {
+                futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx -> {
+                    availableDcGws.forEach(dcGwIp -> {
+                        List<String> dcGws = Arrays.asList(dcGwIp, destinationIp);
+                        Collections.sort(dcGws);
+                        String nextHopKey = FibUtil.getGreLbGroupKey(dcGws);
+                        Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
+                        if (!optionalNextHops.isPresent()) {
+                            return;
+                        }
+                        Nexthops nexthops = optionalNextHops.get();
+                        final String groupId = nexthops.getGroupId();
+                        final long groupIdValue = Long.parseLong(groupId);
+                        Group group = MDSALUtil.buildGroup(groupIdValue, nextHopKey, GroupTypes.GroupSelect,
+                                MDSALUtil.buildBucketLists(Collections.emptyList()));
+                        LOG.trace("Removed LB group {} on dpn {}", group, dpnId);
+                        try {
+                            mdsalApiManager.removeGroup(configTx, dpnId, group);
+                        } catch (ExecutionException | InterruptedException e) {
+                            LOG.error("Group removal failed for group {} with exception", groupId, e);
+                        }
+                        removeNextHopPointer(nextHopKey);
+                        FibUtil.removeOrUpdateNextHopInfo(dpnId, nextHopKey, groupId, nexthops, operationalTx);
+                    });
+                    synchronized (getDcGateWaySyncKey(destinationIp)) {
+                        FibUtil.removeL3vpnDcGateWay(destinationIp, operationalTx);
+                    }
+                }));
+            }));
+            return futures;
+        }, RETRY_COUNT);
+    }
+
+    /**
+     * This method is invoked when the tunnel status is deleted.
+     * All the buckets which point to given destination will be marked down.
+     */
+    public void updateDcGwLoadBalancingGroup(Uint64 dpnId, String destinationIp,
+            boolean isTunnelUp, Class<? extends TunnelTypeBase> tunnelType) {
+        jobCoordinator.enqueueJob(getJobKey(dpnId), () -> {
+            List<String> availableDcGws = fibUtil.getL3VpnDcGateWays();
+            if (availableDcGws.contains(destinationIp)) {
+                availableDcGws.remove(destinationIp);
+            }
+            List<ListenableFuture<Void>> futures = new ArrayList<>();
+            futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx -> {
+                availableDcGws.forEach(dcGwIp -> {
+                    List<String> dcGws = Arrays.asList(dcGwIp, destinationIp);
+                    Collections.sort(dcGws);
+                    String nextHopKey = FibUtil.getGreLbGroupKey(dcGws);
+                    int bucketId = dcGws.indexOf(destinationIp);
                     Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
                     if (!optionalNextHops.isPresent()) {
                         return;
@@ -1136,60 +1238,21 @@ public class NexthopManager implements AutoCloseable {
                     Nexthops nexthops = optionalNextHops.get();
                     final String groupId = nexthops.getGroupId();
                     final long groupIdValue = Long.parseLong(groupId);
-                    if (noOfDcGws > 1) {
-                        mdsalApiManager.removeBucket(confTx, dpnId, groupIdValue, bucketId);
-                    } else {
-                        LOG.trace("Removed LB group {} on dpn {}", groupIdValue, dpnId);
-                        mdsalApiManager.removeGroup(confTx, dpnId, groupIdValue);
-                        removeNextHopPointer(nextHopKey);
-                    }
-                    // When the DC-GW is removed from configuration.
-                    if (noOfDcGws != availableDcGws.size()) {
-                        FibUtil.removeOrUpdateNextHopInfo(dpnId, nextHopKey, groupId, nexthops, operTx);
+                    Bucket bucket = buildBucketForDcGwLbGroup(destinationIp, dpnId, bucketId, tunnelType, isTunnelUp);
+                    LOG.trace("updated bucket {} to group {} on dpn {}.", bucket, groupId, dpnId);
+                    try {
+                        mdsalApiManager.addBucket(configTx, dpnId, groupIdValue, bucket);
+                    } catch (ExecutionException | InterruptedException e) {
+                        LOG.error("Bucket addition failed for bucket {} with exception", bucketId, e);
                     }
-                }
-                FibUtil.removeDpnIdToNextHopInfo(destinationIp, dpnId, operTx);
-            }), LOG, "Error removing or updating load-balancing group");
-        }), LOG, "Error removing or updating load-balancing group");
+                });
+            }));
+            return futures;
+        }, RETRY_COUNT);
     }
 
-    /**
-     * This method is invoked when the tunnel status is updated.
-     * The bucket is directly removed/added based on the operational status of the tunnel.
-     */
-    public void updateDcGwLoadBalancingGroup(List<String> availableDcGws,
-            BigInteger dpnId, String destinationIp, boolean isTunnelUp, Class<? extends TunnelTypeBase> tunnelType) {
-        Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
-        ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
-            // TODO : Place the logic to construct all possible DC-GW combination here.
-            int bucketId = availableDcGws.indexOf(destinationIp);
-            Optional<DpnLbNexthops> dpnLbNextHops = fibUtil.getDpnLbNexthops(dpnId, destinationIp);
-            if (!dpnLbNextHops.isPresent()) {
-                return;
-            }
-            List<String> nextHopKeys = nullToEmpty(dpnLbNextHops.get().getNexthopKey());
-            for (String nextHopKey : nextHopKeys) {
-                Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
-                if (!optionalNextHops.isPresent()) {
-                    return;
-                }
-                Nexthops nexthops = optionalNextHops.get();
-                final String groupId = nexthops.getGroupId();
-                final long groupIdValue = Long.parseLong(groupId);
-                if (isTunnelUp) {
-                    Bucket bucket = buildBucketForDcGwLbGroup(destinationIp, dpnId, bucketId, tunnelType);
-                    LOG.trace("Added bucket {} to group {} on dpn {}.", bucket, groupId, dpnId);
-                    mdsalApiManager.addBucket(confTx, dpnId, groupIdValue, bucket);
-                } else {
-                    LOG.trace("Removed bucketId {} from group {} on dpn {}.", bucketId, groupId, dpnId);
-                    mdsalApiManager.removeBucket(confTx, dpnId, groupIdValue, bucketId);
-                }
-            }
-        }), LOG, "Error updating load-balancing group");
-    }
-
-    private Bucket buildBucketForDcGwLbGroup(String ipAddress, BigInteger dpnId, int index,
-                                             Class<? extends TunnelTypeBase> tunnelType) {
+    private Bucket buildBucketForDcGwLbGroup(String ipAddress, Uint64 dpnId,
+            int index, Class<? extends TunnelTypeBase> tunnelType, boolean isTunnelUp) {
         List<Action> listAction = new ArrayList<>();
         // ActionKey 0 goes to mpls label.
         int actionKey = 1;
@@ -1205,20 +1268,24 @@ public class NexthopManager implements AutoCloseable {
             // clear off actions if there is no egress actions.
             listAction = Collections.emptyList();
         }
+        long watchPort = MDSALUtil.WATCH_PORT;
+        if (!isTunnelUp) {
+            watchPort = 0xFFFFFFFEL;
+        }
         //OVS expects a non-zero weight value for load balancing to happen in select groups
         return MDSALUtil.buildBucket(listAction, SELECT_GROUP_WEIGHT, index,
-                MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP);
+                watchPort, MDSALUtil.WATCH_GROUP);
     }
 
-    public void programDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId, String destinationIp,
+    public void programDcGwLoadBalancingGroup(Uint64 dpnId, String destinationIp,
                                               int addRemoveOrUpdate, boolean isTunnelUp,
                                               Class<? extends TunnelTypeBase> tunnelType) {
         if (NwConstants.ADD_FLOW == addRemoveOrUpdate) {
-            createDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp, tunnelType);
+            createDcGwLoadBalancingGroup(dpnId, destinationIp, tunnelType);
         } else if (NwConstants.DEL_FLOW == addRemoveOrUpdate) {
-            removeOrUpdateDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp);
+            removeDcGwLoadBalancingGroup(dpnId, destinationIp);
         } else if (NwConstants.MOD_FLOW == addRemoveOrUpdate) {
-            updateDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp, isTunnelUp, tunnelType);
+            updateDcGwLoadBalancingGroup(dpnId, destinationIp, isTunnelUp, tunnelType);
         }
     }
 }