fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/NexthopManager.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static java.util.stream.Collectors.toList;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  14
  15 import com.google.common.base.Preconditions;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import java.util.ArrayList;
  18 import java.util.Arrays;
  19 import java.util.Collections;
  20 import java.util.HashMap;
  21 import java.util.List;
  22 import java.util.Locale;
  23 import java.util.Map;
  24 import java.util.Objects;
  25 import java.util.Optional;
  26 import java.util.concurrent.CopyOnWriteArrayList;
  27 import java.util.concurrent.ExecutionException;
  28 import java.util.concurrent.Future;
  29 import java.util.concurrent.TimeUnit;
  30 import java.util.concurrent.TimeoutException;
  31 import javax.annotation.PreDestroy;
  32 import javax.inject.Inject;
  33 import javax.inject.Singleton;
  34 import org.eclipse.jdt.annotation.Nullable;
  35 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  36 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  37 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  38 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  39 import org.opendaylight.genius.itm.globals.ITMConstants;
  40 import org.opendaylight.genius.mdsalutil.ActionInfo;
  41 import org.opendaylight.genius.mdsalutil.BucketInfo;
  42 import org.opendaylight.genius.mdsalutil.GroupEntity;
  43 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  44 import org.opendaylight.genius.mdsalutil.NwConstants;
  45 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  46 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  47 import org.opendaylight.genius.mdsalutil.actions.ActionOutput;
  48 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
  49 import org.opendaylight.genius.mdsalutil.actions.ActionPushVlan;
  50 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionRegMove;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetSource;
  54 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  55 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldVlanVid;
  56 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  57 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  58 import org.opendaylight.mdsal.binding.api.DataBroker;
  59 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  60 import org.opendaylight.mdsal.common.api.ReadFailedException;
  61 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  62 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  63 import org.opendaylight.netvirt.fibmanager.api.L3VPNTransportTypes;
  64 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  65 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev170119.L2vlan;
  66 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev170119.Tunnel;
  67 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfaceType;
  69 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.OutputActionCase;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.PushVlanActionCase;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.SetFieldCase;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInput;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInputBuilder;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutput;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInput;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInputBuilder;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolOutput;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInput;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInputBuilder;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdOutput;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceInputBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceOutput;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelOperStatus;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelsState;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelListKey;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rev160406.DcGatewayIpList;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelInputBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelOutput;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameInputBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameOutput;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.lockmanager.rev160413.LockManagerService;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupInput;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupInputBuilder;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupOutput;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.SalGroupService;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupId;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupRef;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.Buckets;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeRef;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeBase;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeFlat;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeVlan;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.L3nexthop;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthops;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthopsKey;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopKey;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacencies;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacenciesBuilder;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacenciesKey;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpn;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpnBuilder;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.lb.nexthops.Nexthops;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.add.group.input.buckets.bucket.action.action.NxActionResubmitRpcAddGroupCase;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.nodes.node.table.flow.instructions.instruction.instruction.apply.actions._case.apply.actions.action.action.NxActionRegLoadNodesNodeTableFlowApplyActionsCase;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.nx.action.reg.load.grouping.NxRegLoad;
 138 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 139 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 140 import org.opendaylight.yangtools.yang.common.RpcResult;
 141 import org.opendaylight.yangtools.yang.common.Uint32;
 142 import org.opendaylight.yangtools.yang.common.Uint64;
 143 import org.slf4j.Logger;
 144 import org.slf4j.LoggerFactory;
 145
 146 @Singleton
 147 public class NexthopManager implements AutoCloseable {
 148     private static final Logger LOG = LoggerFactory.getLogger(NexthopManager.class);
 149     private static final String NEXTHOP_ID_POOL_NAME = "nextHopPointerPool";
 150     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 151     //  We set the total wait time for lock to be obtained at 9 seconds since GC pauses can be upto 8 seconds
 152     //in scale setups.
 153     private static final long WAIT_TIME_TO_ACQUIRE_LOCK = 9000L;
 154     private static final int SELECT_GROUP_WEIGHT = 1;
 155     private static final int RETRY_COUNT = 6;
 156
 157     private final DataBroker dataBroker;
 158     private final ManagedNewTransactionRunner txRunner;
 159     private final IMdsalApiManager mdsalApiManager;
 160     private final OdlInterfaceRpcService odlInterfaceRpcService;
 161     private final ItmRpcService itmManager;
 162     private final IdManagerService idManager;
 163     private final IElanService elanService;
 164     private final LockManagerService lockManager;
 165     private final SalGroupService salGroupService;
 166     private final JobCoordinator jobCoordinator;
 167     private final FibUtil fibUtil;
 168     private final IInterfaceManager interfaceManager;
 169     private volatile L3VPNTransportTypes configuredTransportTypeL3VPN = L3VPNTransportTypes.Invalid;
 170
 171     /**
 172      * Provides nexthop functions.
 173      * Creates group ID pool
 174      *
 175      * @param dataBroker       - dataBroker reference
 176      * @param mdsalApiManager  - mdsalApiManager reference
 177      * @param idManager        - idManager reference
 178      * @param odlInterfaceRpcService - odlInterfaceRpcService reference
 179      * @param itmManager       - itmManager reference
 180      */
 181     @Inject
 182     public NexthopManager(final DataBroker dataBroker,
 183                           final IMdsalApiManager mdsalApiManager,
 184                           final IdManagerService idManager,
 185                           final OdlInterfaceRpcService odlInterfaceRpcService,
 186                           final ItmRpcService itmManager,
 187                           final LockManagerService lockManager,
 188                           final IElanService elanService,
 189                           final SalGroupService salGroupService,
 190                           final JobCoordinator jobCoordinator,
 191                           final FibUtil fibUtil,
 192                           final IInterfaceManager interfaceManager) {
 193         this.dataBroker = dataBroker;
 194         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 195         this.mdsalApiManager = mdsalApiManager;
 196         this.idManager = idManager;
 197         this.odlInterfaceRpcService = odlInterfaceRpcService;
 198         this.itmManager = itmManager;
 199         this.elanService = elanService;
 200         this.salGroupService = salGroupService;
 201         this.jobCoordinator = jobCoordinator;
 202         this.fibUtil = fibUtil;
 203         this.lockManager = lockManager;
 204         this.interfaceManager = interfaceManager;
 205         createIdPool();
 206     }
 207
 208     private void createIdPool() {
 209         CreateIdPoolInput createPool = new CreateIdPoolInputBuilder()
 210             .setPoolName(NEXTHOP_ID_POOL_NAME)
 211             .setLow(150000L)
 212             .setHigh(175000L)
 213             .build();
 214         try {
 215             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPool);
 216             if (result != null && result.get().isSuccessful()) {
 217                 LOG.info("Created IdPool for NextHopPointerPool");
 218             }
 219         } catch (InterruptedException | ExecutionException e) {
 220             LOG.error("Failed to create idPool for NextHopPointerPool", e);
 221         }
 222     }
 223
 224     private String getNextHopKey(Uint32 vpnId, String ipAddress) {
 225         return "nexthop." + vpnId + ipAddress;
 226     }
 227
 228     String getRemoteSelectGroupKey(Uint32 vpnId, String ipAddress) {
 229         return "remote.ecmp.nexthop." + vpnId + ipAddress;
 230     }
 231
 232     String getLocalSelectGroupKey(Uint32 vpnId, String ipAddress) {
 233         return "local.ecmp.nexthop." + vpnId + ipAddress;
 234     }
 235
 236     public ItmRpcService getItmManager() {
 237         return itmManager;
 238     }
 239
 240     protected long createNextHopPointer(String nexthopKey) {
 241         AllocateIdInput getIdInput = new AllocateIdInputBuilder()
 242             .setPoolName(NEXTHOP_ID_POOL_NAME).setIdKey(nexthopKey)
 243             .build();
 244         //TODO: Proper error handling once IdManager code is complete
 245         try {
 246             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
 247             RpcResult<AllocateIdOutput> rpcResult = result.get();
 248             return rpcResult.getResult().getIdValue().toJava();
 249         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 250             // FIXME: NPEs should not be caught but rather their root cause should be eliminated
 251             LOG.trace("Failed to allocate {}", getIdInput, e);
 252         }
 253         return 0;
 254     }
 255
 256     protected void removeNextHopPointer(String nexthopKey) {
 257         ReleaseIdInput idInput = new ReleaseIdInputBuilder()
 258             .setPoolName(NEXTHOP_ID_POOL_NAME)
 259             .setIdKey(nexthopKey).build();
 260         try {
 261             RpcResult<ReleaseIdOutput> rpcResult = idManager.releaseId(idInput).get();
 262             if (!rpcResult.isSuccessful()) {
 263                 LOG.error("RPC Call to Get Unique Id for nexthopKey {} returned with Errors {}",
 264                         nexthopKey, rpcResult.getErrors());
 265             }
 266         } catch (InterruptedException | ExecutionException e) {
 267             LOG.warn("Exception when getting Unique Id for key {}", nexthopKey, e);
 268         }
 269     }
 270
 271     protected List<ActionInfo> getEgressActionsForInterface(final String ifName, int actionKey,
 272                                                             boolean isTunnelInterface,
 273                                                             Uint32 vpnId, String destIpPrefix) {
 274         List<Action> actions;
 275         try {
 276             if (isTunnelInterface && interfaceManager.isItmDirectTunnelsEnabled()) {
 277                 RpcResult<GetEgressActionsForTunnelOutput> rpcResult =
 278                         itmManager.getEgressActionsForTunnel(new GetEgressActionsForTunnelInputBuilder()
 279                                 .setIntfName(ifName).build()).get();
 280                 if (!rpcResult.isSuccessful()) {
 281                     LOG.error("RPC Call to Get egress tunnel actions for interface {} returned with Errors {}",
 282                             ifName, rpcResult.getErrors());
 283                     return Collections.emptyList();
 284                 } else {
 285                     actions = rpcResult.getResult().nonnullAction();
 286                 }
 287             } else {
 288                 RpcResult<GetEgressActionsForInterfaceOutput> rpcResult = odlInterfaceRpcService
 289                         .getEgressActionsForInterface(new GetEgressActionsForInterfaceInputBuilder()
 290                                 .setIntfName(ifName).build()).get();
 291                 if (!rpcResult.isSuccessful()) {
 292                     LOG.error("RPC Call to Get egress vm actions for interface {} vpnId {} ipPrefix {} returned with "
 293                                     + "Errors {}", ifName, vpnId, destIpPrefix, rpcResult.getErrors());
 294                     return Collections.emptyList();
 295                 } else {
 296                     actions = rpcResult.getResult().nonnullAction();
 297                 }
 298             }
 299             List<ActionInfo> listActionInfo = new ArrayList<>();
 300             for (Action action : actions) {
 301                 actionKey = action.key().getOrder() + actionKey;
 302                 org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action
 303                     actionClass = action.getAction();
 304                 if (actionClass instanceof OutputActionCase) {
 305                     listActionInfo.add(new ActionOutput(actionKey,
 306                         ((OutputActionCase) actionClass).getOutputAction().getOutputNodeConnector()));
 307                 } else if (actionClass instanceof PushVlanActionCase) {
 308                     listActionInfo.add(new ActionPushVlan(actionKey));
 309                 } else if (actionClass instanceof SetFieldCase) {
 310                     if (((SetFieldCase) actionClass).getSetField().getVlanMatch() != null) {
 311                         int vlanVid = ((SetFieldCase) actionClass).getSetField().getVlanMatch()
 312                             .getVlanId().getVlanId().getValue().toJava();
 313                         listActionInfo.add(new ActionSetFieldVlanVid(actionKey, vlanVid));
 314                     }
 315                 } else if (actionClass instanceof NxActionResubmitRpcAddGroupCase) {
 316                     Short tableId = ((NxActionResubmitRpcAddGroupCase) actionClass).getNxResubmit().getTable().toJava();
 317                     listActionInfo.add(new ActionNxResubmit(actionKey, tableId));
 318                 } else if (actionClass instanceof NxActionRegLoadNodesNodeTableFlowApplyActionsCase) {
 319                     NxRegLoad nxRegLoad =
 320                         ((NxActionRegLoadNodesNodeTableFlowApplyActionsCase) actionClass).getNxRegLoad();
 321                     listActionInfo.add(new ActionRegLoad(actionKey, NxmNxReg6.class,
 322                         nxRegLoad.getDst().getStart().toJava(), nxRegLoad.getDst().getEnd().toJava(),
 323                         nxRegLoad.getValue().longValue()));
 324                 }
 325             }
 326             return listActionInfo;
 327         } catch (InterruptedException | ExecutionException | NullPointerException e) {
 328             LOG.error("Exception when egress actions for interface {} isTunnel {} vpnId {} ipPrefix {}", ifName,
 329                     isTunnelInterface, vpnId, destIpPrefix, e);
 330         }
 331         LOG.warn("Exception when egress actions for interface {}", ifName);
 332         return Collections.emptyList();
 333     }
 334
 335     @Nullable
 336     protected String getTunnelInterfaceName(Uint64 srcDpId, Uint64 dstDpId) {
 337         Class<? extends TunnelTypeBase> tunType = getReqTunType(getReqTransType().toUpperCase(Locale.getDefault()));
 338         Future<RpcResult<GetTunnelInterfaceNameOutput>> result;
 339         try {
 340             result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 341                 .setSourceDpid(srcDpId)
 342                 .setDestinationDpid(dstDpId)
 343                 .setTunnelType(tunType)
 344                 .build());
 345             RpcResult<GetTunnelInterfaceNameOutput> rpcResult = result.get();
 346             if (!rpcResult.isSuccessful()) {
 347                 LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
 348             } else {
 349                 return rpcResult.getResult().getInterfaceName();
 350             }
 351         } catch (InterruptedException | ExecutionException e) {
 352             LOG.warn("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstDpId, e);
 353         }
 354         return null;
 355     }
 356
 357     @Nullable
 358     protected String getTunnelInterfaceName(Uint64 srcDpId, org.opendaylight.yang.gen.v1.urn.ietf.params
 359         .xml.ns.yang.ietf.inet.types.rev130715.IpAddress dstIp, Class<? extends TunnelTypeBase> tunnelType) {
 360         Future<RpcResult<GetInternalOrExternalInterfaceNameOutput>> result;
 361         try {
 362             LOG.debug("Trying to fetch tunnel interface name for source dpn {} destIp {} tunType {}", srcDpId,
 363                     dstIp.stringValue(), tunnelType.getName());
 364             result = itmManager.getInternalOrExternalInterfaceName(new GetInternalOrExternalInterfaceNameInputBuilder()
 365                 .setSourceDpid(srcDpId)
 366                 .setDestinationIp(dstIp)
 367                 .setTunnelType(tunnelType)
 368                 .build());
 369             RpcResult<GetInternalOrExternalInterfaceNameOutput> rpcResult = result.get();
 370             if (!rpcResult.isSuccessful()) {
 371                 LOG.warn("RPC Call to getTunnelInterfaceName returned with Errors {}", rpcResult.getErrors());
 372             } else {
 373                 return rpcResult.getResult().getInterfaceName();
 374             }
 375         } catch (InterruptedException | ExecutionException e) {
 376             LOG.error("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstIp, e);
 377         }
 378         return null;
 379     }
 380
 381
 382     public long getLocalNextHopGroup(Uint32 vpnId,
 383             String ipNextHopAddress) {
 384         long groupId = createNextHopPointer(getNextHopKey(vpnId, ipNextHopAddress));
 385         if (groupId == FibConstants.INVALID_GROUP_ID) {
 386             LOG.error("Unable to allocate groupId for vpnId {} , prefix {}", vpnId, ipNextHopAddress);
 387         }
 388         return groupId;
 389     }
 390
 391     public long getLocalSelectGroup(Uint32 vpnId,
 392             String ipNextHopAddress) {
 393         long groupId = createNextHopPointer(getLocalSelectGroupKey(vpnId, ipNextHopAddress));
 394         if (groupId == FibConstants.INVALID_GROUP_ID) {
 395             LOG.error("Unable to allocate groupId for vpnId {} , prefix {}", vpnId, ipNextHopAddress);
 396         }
 397         return groupId;
 398     }
 399
 400     public long createLocalNextHop(Uint32 vpnId, Uint64 dpnId, String ifName,
 401                                    String primaryIpAddress, String currDestIpPrefix,
 402                                    String gwMacAddress, Uint32 parentVpnId) {
 403         //For VPN Imported routes, getting VPN Instance name using parentVpnId
 404         String vpnName = parentVpnId != null ? fibUtil.getVpnNameFromId(parentVpnId) : fibUtil.getVpnNameFromId(vpnId);
 405         if (vpnName == null) {
 406             return 0;
 407         }
 408         String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, primaryIpAddress);
 409
 410         long groupId = createNextHopPointer(getNextHopKey(vpnId, primaryIpAddress));
 411         if (groupId == 0) {
 412             LOG.error("Unable to allocate groupId for vpnId {} , IntfName {}, primaryIpAddress {} curIpPrefix {}",
 413                     vpnId, ifName, primaryIpAddress, currDestIpPrefix);
 414             return groupId;
 415         }
 416         String nextHopLockStr = vpnId + primaryIpAddress;
 417         String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, currDestIpPrefix);
 418         jobCoordinator.enqueueJob(jobKey, () -> {
 419             try {
 420                 if (FibUtil.lockCluster(lockManager, nextHopLockStr, WAIT_TIME_TO_ACQUIRE_LOCK)) {
 421                     VpnNexthop nexthop = getVpnNexthop(vpnId, primaryIpAddress);
 422                     LOG.trace("nexthop: {} retrieved for vpnId {}, prefix {}, ifName {} on dpn {}", nexthop, vpnId,
 423                             primaryIpAddress, ifName, dpnId);
 424                     if (nexthop == null) {
 425                         String encMacAddress = macAddress == null
 426                                 ? fibUtil.getMacAddressFromPrefix(ifName, vpnName, primaryIpAddress) : macAddress;
 427                         List<ActionInfo> listActionInfo = new ArrayList<>();
 428                         int actionKey = 0;
 429                         // MAC re-write
 430                         if (encMacAddress != null) {
 431                             if (gwMacAddress != null) {
 432                                 LOG.trace("The Local NextHop Group Source Mac {} for VpnInterface {} on VPN {}",
 433                                         gwMacAddress, ifName, vpnId);
 434                                 listActionInfo.add(new ActionSetFieldEthernetSource(actionKey++,
 435                                         new MacAddress(gwMacAddress)));
 436                             }
 437                             listActionInfo.add(new ActionSetFieldEthernetDestination(actionKey++,
 438                                     new MacAddress(encMacAddress)));
 439                             // listActionInfo.add(0, new ActionPopMpls());
 440                         } else {
 441                             LOG.error("mac address for new local nexthop group {} is null for vpnId {}, prefix {}, "
 442                                     + "ifName {} on dpn {}", groupId, vpnId, primaryIpAddress, ifName, dpnId);
 443                         }
 444                         List<ActionInfo> nhActionInfoList = getEgressActionsForInterface(ifName, actionKey, false,
 445                                 vpnId, currDestIpPrefix);
 446                         if (nhActionInfoList.isEmpty()) {
 447                             LOG.error("createLocalNextHop: Skipping, Empty list of egress actions received for "
 448                                     + "interface {} on dpn {} for vpn {} prefix {}", ifName, dpnId, vpnId,
 449                                     currDestIpPrefix);
 450                         }
 451                         listActionInfo.addAll(nhActionInfoList);
 452                         BucketInfo bucket = new BucketInfo(listActionInfo);
 453                         List<BucketInfo> listBucketInfo = new ArrayList<>();
 454                         listBucketInfo.add(bucket);
 455                         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, groupId, primaryIpAddress,
 456                                 GroupTypes.GroupAll, listBucketInfo);
 457                         LOG.trace("Install LNH Group: id {}, mac address {}, interface {} for prefix {}", groupId,
 458                                 encMacAddress, ifName, primaryIpAddress);
 459                         //Try to install group directly on the DPN bypassing the FRM, in order to avoid waiting for the
 460                         // group to get installed before programming the flows
 461                         installGroupOnDpn(groupId, dpnId, primaryIpAddress, listBucketInfo,
 462                                 getNextHopKey(vpnId, primaryIpAddress), GroupTypes.GroupAll);
 463                         // install Group
 464                         mdsalApiManager.syncInstallGroup(groupEntity);
 465                         // update MD-SAL DS
 466                         addVpnNexthopToDS(dpnId, vpnId, primaryIpAddress, currDestIpPrefix, groupId);
 467
 468                     } else {
 469                         // Ignore adding new prefix , if it already exists
 470                         List<IpAdjacencies> prefixesList = nexthop.getIpAdjacencies();
 471                         IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currDestIpPrefix).build();
 472                         if (prefixesList != null && prefixesList.contains(prefix)) {
 473                             LOG.trace("Prefix {} is already present in l3nextHop {} ", currDestIpPrefix, nexthop);
 474                         } else {
 475                             IpAdjacenciesBuilder ipPrefixesBuilder =
 476                                     new IpAdjacenciesBuilder().withKey(new IpAdjacenciesKey(currDestIpPrefix));
 477                             LOG.trace("Updating prefix {} to vpnNextHop {} Operational DS", currDestIpPrefix, nexthop);
 478                             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
 479                                     getVpnNextHopIpPrefixIdentifier(vpnId, primaryIpAddress, currDestIpPrefix),
 480                                     ipPrefixesBuilder.build());
 481                         }
 482                     }
 483                 }
 484             } finally {
 485                 FibUtil.unlockCluster(lockManager, nextHopLockStr);
 486             }
 487             return Collections.emptyList();
 488         });
 489         return groupId;
 490     }
 491
 492     private void installGroupOnDpn(long groupId, Uint64 dpnId, String groupName, List<BucketInfo> bucketsInfo,
 493                                      String nextHopKey, GroupTypes groupType) {
 494         NodeRef nodeRef = FibUtil.buildNodeRef(dpnId);
 495         Buckets buckets = FibUtil.buildBuckets(bucketsInfo);
 496         GroupRef groupRef = new GroupRef(FibUtil.buildGroupInstanceIdentifier(groupId, dpnId));
 497         AddGroupInput input = new AddGroupInputBuilder().setNode(nodeRef).setGroupId(new GroupId(groupId))
 498                 .setBuckets(buckets).setGroupRef(groupRef).setGroupType(groupType)
 499                 .setGroupName(groupName).build();
 500         Future<RpcResult<AddGroupOutput>> groupStats = salGroupService.addGroup(input);
 501         RpcResult<AddGroupOutput> rpcResult = null;
 502         try {
 503             rpcResult = groupStats.get(WAIT_TIME_FOR_SYNC_INSTALL, TimeUnit.MILLISECONDS);
 504             if (rpcResult != null && rpcResult.isSuccessful()) {
 505                 LOG.info("installGroupOnDpn: Group {} with key {} has been successfully installed directly on dpn {}.",
 506                         groupId, nextHopKey, dpnId);
 507             } else {
 508                 LOG.error("installGroupOnDpn: Unable to install group {} with key {} directly on dpn {} due to {}.",
 509                         groupId, nextHopKey, dpnId, rpcResult != null ? rpcResult.getErrors() : null);
 510             }
 511         } catch (InterruptedException | ExecutionException e) {
 512             LOG.error("installGroupOnDpn: Error while installing group {} directly on dpn {}", groupId, dpnId);
 513         } catch (TimeoutException e) {
 514             LOG.error("installGroupOnDpn: Group {} installation on dpn {} timed out.", groupId, dpnId);
 515         }
 516     }
 517
 518     protected void addVpnNexthopToDS(Uint64 dpnId, Uint32 vpnId, String primaryIpAddr,
 519                                      String currIpAddr, long egressPointer) {
 520         InstanceIdentifierBuilder<VpnNexthops> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
 521             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId));
 522
 523         List<IpAdjacencies> ipPrefixesList = new ArrayList<>();
 524         IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currIpAddr).build();
 525         ipPrefixesList.add(prefix);
 526         // Add nexthop to vpn node
 527         VpnNexthop nh = new VpnNexthopBuilder()
 528             .withKey(new VpnNexthopKey(primaryIpAddr))
 529             .setDpnId(dpnId)
 530             .setIpAdjacencies(ipPrefixesList)
 531             .setEgressPointer(egressPointer).build();
 532
 533         InstanceIdentifier<VpnNexthop> id1 = idBuilder
 534             .child(VpnNexthop.class, new VpnNexthopKey(primaryIpAddr)).build();
 535         LOG.trace("Adding vpnnextHop {} to Operational DS", nh);
 536         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id1, nh);
 537
 538     }
 539
 540     protected InstanceIdentifier<IpAdjacencies> getVpnNextHopIpPrefixIdentifier(Uint32 vpnId, String primaryIpAddress,
 541                                                                                 String ipPrefix) {
 542         InstanceIdentifier<IpAdjacencies> id = InstanceIdentifier.builder(L3nexthop.class)
 543                 .child(VpnNexthops.class, new VpnNexthopsKey(vpnId))
 544                 .child(VpnNexthop.class, new VpnNexthopKey(primaryIpAddress))
 545                 .child(IpAdjacencies.class, new IpAdjacenciesKey(ipPrefix)).build();
 546         return id;
 547     }
 548
 549     @Nullable
 550     protected VpnNexthop getVpnNexthop(Uint32 vpnId, String ipAddress) {
 551
 552         // check if vpn node is there
 553         InstanceIdentifierBuilder<VpnNexthops> idBuilder =
 554             InstanceIdentifier.builder(L3nexthop.class).child(VpnNexthops.class,
 555                 new VpnNexthopsKey(vpnId));
 556         InstanceIdentifier<VpnNexthops> id = idBuilder.build();
 557         Optional<VpnNexthops> vpnNexthops;
 558         try {
 559             vpnNexthops = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 560                     LogicalDatastoreType.OPERATIONAL, id);
 561         } catch (ExecutionException | InterruptedException e) {
 562             LOG.error("getVpnNexthop: Exception while reading VpnNexthops DS for the address {} vpn {}", ipAddress,
 563                     vpnId, e);
 564             return null;
 565         }
 566         if (vpnNexthops.isPresent()) {
 567             // get nexthops list for vpn
 568             List<VpnNexthop> nexthops = vpnNexthops.get().nonnullVpnNexthop();
 569             for (VpnNexthop nexthop : nexthops) {
 570                 if (Objects.equals(nexthop.getIpAddress(), ipAddress)) {
 571                     // return nexthop
 572                     LOG.trace("VpnNextHop : {}", nexthop);
 573                     return nexthop;
 574                 }
 575             }
 576             // return null if not found
 577         }
 578         return null;
 579     }
 580
 581     @Nullable
 582     public AdjacencyResult getRemoteNextHopPointer(Uint64 remoteDpnId, Uint32 vpnId, String prefixIp,
 583             @Nullable String nextHopIp, Class<? extends TunnelTypeBase> tunnelType) {
 584         String egressIfName = null;
 585         LOG.trace("getRemoteNextHopPointer: input [remoteDpnId {}, vpnId {}, prefixIp {}, nextHopIp {} ]", remoteDpnId,
 586             vpnId, prefixIp, nextHopIp);
 587
 588         Class<? extends InterfaceType> egressIfType;
 589         ElanInstance elanInstance = getElanInstanceForPrefix(vpnId, prefixIp);
 590         if (elanInstance != null) {
 591             egressIfType = getInterfaceType(elanInstance);
 592         } else {
 593             LOG.warn("Failed to determine network type for prefixIp {} using tunnel", prefixIp);
 594             egressIfType = Tunnel.class;
 595         }
 596
 597         if (Tunnel.class.equals(egressIfType)) {
 598             egressIfName = getTunnelRemoteNextHopPointer(remoteDpnId, nextHopIp, tunnelType);
 599         } else {
 600             egressIfName = getExtPortRemoteNextHopPointer(remoteDpnId, elanInstance);
 601         }
 602
 603         LOG.trace("NextHop pointer for prefixIp {} vpnId {} dpnId {} is {}", prefixIp, vpnId, remoteDpnId,
 604             egressIfName);
 605         return egressIfName != null ? new AdjacencyResult(egressIfName, egressIfType, nextHopIp,
 606                 prefixIp) : null;
 607     }
 608
 609     private void removeVpnNexthopFromDS(Uint32 vpnId, String ipPrefix) {
 610
 611         InstanceIdentifierBuilder<VpnNexthop> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
 612             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId))
 613             .child(VpnNexthop.class, new VpnNexthopKey(ipPrefix));
 614         InstanceIdentifier<VpnNexthop> id = idBuilder.build();
 615         // remove from DS
 616         LOG.trace("Removing vpn next hop from datastore : {}", id);
 617         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 618     }
 619
 620     public void removeLocalNextHop(Uint64 dpnId, Uint32 vpnId, String primaryIpAddress, String currDestIpPrefix) {
 621         String nextHopLockStr = vpnId + primaryIpAddress;
 622         try {
 623             if (FibUtil.lockCluster(lockManager, nextHopLockStr, WAIT_TIME_TO_ACQUIRE_LOCK)) {
 624                 VpnNexthop nh = getVpnNexthop(vpnId, primaryIpAddress);
 625                 if (nh != null) {
 626                     List<IpAdjacencies> prefixesList = new ArrayList<>(nh.nonnullIpAdjacencies());
 627                     IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currDestIpPrefix).build();
 628                     prefixesList.remove(prefix);
 629                     if (prefixesList.isEmpty()) { //remove the group only if there are no more flows using this group
 630                         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, nh.getEgressPointer().toJava(),
 631                                 primaryIpAddress, GroupTypes.GroupAll, Collections.emptyList());
 632                         // remove Group ...
 633                         mdsalApiManager.removeGroup(groupEntity);
 634                         //update MD-SAL DS
 635                         removeVpnNexthopFromDS(vpnId, primaryIpAddress);
 636                         //release groupId
 637                         removeNextHopPointer(getNextHopKey(vpnId, primaryIpAddress));
 638                         LOG.debug("Local Next hop {} for {} {} on dpn {} successfully deleted",
 639                                 nh.getEgressPointer(), vpnId, primaryIpAddress, dpnId);
 640                     } else {
 641                         //remove the currIpPrefx from IpPrefixList of the vpnNexthop
 642                         LOG.trace("Removing the prefix {} from vpnNextHop {} Operational DS", currDestIpPrefix, nh);
 643                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
 644                                 getVpnNextHopIpPrefixIdentifier(vpnId, primaryIpAddress, currDestIpPrefix));
 645                     }
 646                 } else {
 647                     //throw error
 648                     LOG.error("Local NextHop for VpnId {} curIpPrefix {} on dpn {} primaryIpAddress {} not deleted",
 649                             vpnId, currDestIpPrefix, dpnId, primaryIpAddress);
 650                 }
 651             }
 652         } finally {
 653             FibUtil.unlockCluster(lockManager, nextHopLockStr);
 654         }
 655     }
 656
 657     public void setConfTransType(String service, String transportType) {
 658
 659         if (!service.equalsIgnoreCase("L3VPN")) {
 660             LOG.error("Incorrect service {} provided for setting the transport type.", service);
 661             return;
 662         }
 663
 664         L3VPNTransportTypes transType = L3VPNTransportTypes.validateTransportType(transportType
 665                 .toUpperCase(Locale.getDefault()));
 666
 667         if (transType != L3VPNTransportTypes.Invalid) {
 668             configuredTransportTypeL3VPN = transType;
 669         }
 670     }
 671
 672     public void writeConfTransTypeConfigDS() {
 673         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, getConfTransportTypeIdentifier(),
 674             createConfTransportType(configuredTransportTypeL3VPN.getTransportType()));
 675     }
 676
 677     public L3VPNTransportTypes getConfiguredTransportTypeL3VPN() {
 678         return this.configuredTransportTypeL3VPN;
 679     }
 680
 681     public String getReqTransType() {
 682         if (configuredTransportTypeL3VPN == L3VPNTransportTypes.Invalid) {
 683             /*
 684              * Restart scenario, Read from the ConfigDS.
 685              * if the value is Unset, cache value as VxLAN.
 686              */
 687             LOG.trace("configureTransportType is not yet set.");
 688             Optional<ConfTransportTypeL3vpn> configuredTransTypeFromConfig;
 689             try {
 690                 configuredTransTypeFromConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 691                         LogicalDatastoreType.CONFIGURATION, getConfTransportTypeIdentifier());
 692             } catch (ExecutionException | InterruptedException e) {
 693                 LOG.error("getReqTransType: Exception while reading ConfTransportTypeL3vpn DS", e);
 694                 return null;
 695             }
 696             if (configuredTransTypeFromConfig.isPresent()) {
 697                 if (TunnelTypeGre.class.equals(configuredTransTypeFromConfig.get().getTransportType())) {
 698                     configuredTransportTypeL3VPN = L3VPNTransportTypes.GRE;
 699                 } else {
 700                     configuredTransportTypeL3VPN = L3VPNTransportTypes.VxLAN;
 701                 }
 702                 LOG.trace("configuredTransportType set from config DS to {}",
 703                     getConfiguredTransportTypeL3VPN().getTransportType());
 704             } else {
 705                 setConfTransType("L3VPN", L3VPNTransportTypes.VxLAN.getTransportType());
 706                 LOG.trace("configuredTransportType is not set in the Config DS. VxLAN as default will be used.");
 707             }
 708         } else {
 709             LOG.trace("configuredTransportType is set as {}", getConfiguredTransportTypeL3VPN().getTransportType());
 710         }
 711         return getConfiguredTransportTypeL3VPN().getTransportType();
 712     }
 713
 714     public InstanceIdentifier<ConfTransportTypeL3vpn> getConfTransportTypeIdentifier() {
 715         return InstanceIdentifier.builder(ConfTransportTypeL3vpn.class).build();
 716     }
 717
 718     private ConfTransportTypeL3vpn createConfTransportType(String type) {
 719         ConfTransportTypeL3vpn confTransType;
 720         switch (type) {
 721             case ITMConstants.TUNNEL_TYPE_GRE:
 722                 confTransType = new ConfTransportTypeL3vpnBuilder().setTransportType(TunnelTypeGre.class).build();
 723                 LOG.trace("Setting the confTransportType to GRE.");
 724                 break;
 725             case ITMConstants.TUNNEL_TYPE_VXLAN:
 726                 confTransType = new ConfTransportTypeL3vpnBuilder().setTransportType(TunnelTypeVxlan.class).build();
 727                 LOG.trace("Setting the confTransportType to VxLAN.");
 728                 break;
 729             default:
 730                 LOG.trace("Invalid transport type {} passed to Config DS ", type);
 731                 confTransType = null;
 732                 break;
 733         }
 734         return confTransType;
 735     }
 736
 737     public Class<? extends TunnelTypeBase> getReqTunType(String transportType) {
 738         switch (transportType) {
 739             case "VXLAN":
 740                 return TunnelTypeVxlan.class;
 741             case "GRE":
 742                 return TunnelTypeGre.class;
 743             default:
 744                 return TunnelTypeMplsOverGre.class;
 745         }
 746     }
 747
 748     public String getTransportTypeStr(String tunType) {
 749         if (tunType.equals(TunnelTypeVxlan.class.toString())) {
 750             return ITMConstants.TUNNEL_TYPE_VXLAN;
 751         } else if (tunType.equals(TunnelTypeGre.class.toString())) {
 752             return ITMConstants.TUNNEL_TYPE_GRE;
 753         } else if (tunType.equals(TunnelTypeMplsOverGre.class.toString())) {
 754             return ITMConstants.TUNNEL_TYPE_MPLSoGRE;
 755         } else {
 756             return ITMConstants.TUNNEL_TYPE_INVALID;
 757         }
 758     }
 759
 760     @Override
 761     @PreDestroy
 762     public void close() {
 763         LOG.info("{} close", getClass().getSimpleName());
 764     }
 765
 766     // TODO Clean up the exception handling
 767     @SuppressWarnings("checkstyle:IllegalCatch")
 768     @Nullable
 769     private String getTunnelRemoteNextHopPointer(Uint64 remoteDpnId, String nextHopIp,
 770                                                  Class<? extends TunnelTypeBase> tunnelType) {
 771         if (nextHopIp != null && !nextHopIp.isEmpty()) {
 772             try {
 773                 // here use the config for tunnel type param
 774                 return getTunnelInterfaceName(remoteDpnId,
 775                     org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder
 776                         .getDefaultInstance(nextHopIp), tunnelType);
 777             } catch (Exception ex) {
 778                 LOG.error("Error while retrieving nexthop pointer for nexthop {} remoteDpn {}",
 779                         nextHopIp, remoteDpnId, ex);
 780             }
 781         }
 782
 783         return null;
 784     }
 785
 786     private String getExtPortRemoteNextHopPointer(Uint64 remoteDpnId, ElanInstance elanInstance) {
 787         return elanService.getExternalElanInterface(elanInstance.getElanInstanceName(), remoteDpnId);
 788     }
 789
 790     /**
 791      * Get the interface type associated with the type of ELAN used for routing
 792      * traffic to/from remote compute nodes.
 793      *
 794      * @param elanInstance The elan instance
 795      * @return L2vlan for flat/VLAN network type and Tunnel otherwise
 796      */
 797     private Class<? extends InterfaceType> getInterfaceType(ElanInstance elanInstance) {
 798         Class<? extends SegmentTypeBase> segmentType = elanInstance.getSegmentType();
 799         if (SegmentTypeFlat.class.equals(segmentType) || SegmentTypeVlan.class.equals(segmentType)) {
 800             return L2vlan.class;
 801         }
 802
 803         return Tunnel.class;
 804     }
 805
 806     private ElanInstance getElanInstanceForPrefix(Uint32 vpnId, String prefixIp) {
 807         ElanInstance elanInstance = null;
 808         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, prefixIp);
 809         if (prefix != null) {
 810             if (prefix.getNetworkId() != null) {
 811                 elanInstance = elanService.getElanInstance(prefix.getNetworkId().getValue());
 812             }
 813         }
 814
 815         return elanInstance;
 816     }
 817
 818     static class AdjacencyResult {
 819         private final String interfaceName;
 820         private final Class<? extends InterfaceType> interfaceType;
 821         private final String nextHopIp;
 822         private final String prefix;
 823
 824         AdjacencyResult(String interfaceName, Class<? extends InterfaceType> interfaceType, String nextHopIp,
 825                         String prefix) {
 826             this.interfaceName = interfaceName;
 827             this.interfaceType = interfaceType;
 828             this.nextHopIp = nextHopIp;
 829             this.prefix = prefix;
 830         }
 831
 832         public String getInterfaceName() {
 833             return interfaceName;
 834         }
 835
 836         public Class<? extends InterfaceType> getInterfaceType() {
 837             return interfaceType;
 838         }
 839
 840         public String getNextHopIp() {
 841             return nextHopIp;
 842         }
 843
 844         public String getPrefix() {
 845             return prefix;
 846         }
 847
 848         @Override
 849         public int hashCode() {
 850             final int prime = 31;
 851             int result = 1;
 852             result = prime * result + (interfaceName == null ? 0 : interfaceName.hashCode());
 853             return result;
 854         }
 855
 856         @Override
 857         public boolean equals(Object obj) {
 858             if (obj == null) {
 859                 return false;
 860             }
 861
 862             if (getClass() != obj.getClass()) {
 863                 return false;
 864             } else {
 865                 AdjacencyResult other = (AdjacencyResult) obj;
 866                 return interfaceName.equals(other.interfaceName);
 867             }
 868         }
 869     }
 870
 871     protected long setupLoadBalancingNextHop(Uint32 parentVpnId, Uint64 dpnId,
 872             String destPrefix, List<BucketInfo> localBucketInfo, List<BucketInfo> remoteBucketInfo) {
 873         long remoteGroupId = createNextHopPointer(getRemoteSelectGroupKey(parentVpnId, destPrefix));
 874         if (remoteGroupId == FibConstants.INVALID_GROUP_ID) {
 875             LOG.error("Unable to allocate/retrieve remote groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 876             return remoteGroupId;
 877         }
 878         long localGroupId =  createNextHopPointer(getLocalSelectGroupKey(parentVpnId, destPrefix));
 879         if (localGroupId == FibConstants.INVALID_GROUP_ID) {
 880             LOG.error("Unable to allocate/retrieve local groupId for vpnId {} , prefix {}",
 881                 parentVpnId, destPrefix);
 882             return remoteGroupId;
 883         }
 884         List<BucketInfo> combinedBucketInfo = new ArrayList<>();
 885         combinedBucketInfo.addAll(localBucketInfo);
 886         combinedBucketInfo.addAll(remoteBucketInfo);
 887         GroupEntity remoteGroupEntity = MDSALUtil.buildGroupEntity(
 888                 dpnId, remoteGroupId, destPrefix, GroupTypes.GroupSelect, combinedBucketInfo);
 889         GroupEntity localGroupEntity = MDSALUtil.buildGroupEntity(
 890                 dpnId, localGroupId, destPrefix, GroupTypes.GroupSelect, localBucketInfo);
 891         String jobKey = FibUtil.getCreateLocalNextHopJobKey(parentVpnId, dpnId, destPrefix);
 892         jobCoordinator.enqueueJob(jobKey, () -> {
 893             mdsalApiManager.syncInstallGroup(remoteGroupEntity);
 894             if (!localBucketInfo.isEmpty()) {
 895                 mdsalApiManager.syncInstallGroup(localGroupEntity);
 896             }
 897             if (LOG.isDebugEnabled()) {
 898                 LOG.debug("Finished installing GroupEntity with jobCoordinator key {} remoteGroupEntity.groupId {}"
 899                         + "localGroupEntity.groupId {}  groupEntity.groupType {}", jobKey,
 900                         remoteGroupEntity.getGroupId(), localGroupEntity.getGroupId(),
 901                         remoteGroupEntity.getGroupType());
 902             }
 903             // Delete local group(if exists) if there is no local info.
 904             // Local group has to be deleted if all VMs in a compute is deleted.
 905             if (localBucketInfo.isEmpty()) {
 906                 LOG.debug("Deleting local group {} since no local nhs present for "
 907                         + "prefix {}", localGroupEntity.getGroupId(), destPrefix);
 908                 mdsalApiManager.syncRemoveGroup(localGroupEntity);
 909             }
 910             return Collections.emptyList();
 911         });
 912         return remoteGroupId;
 913     }
 914
 915     protected void deleteLoadBalancingNextHop(Uint32 parentVpnId, Uint64 dpnId, String destPrefix) {
 916         long remoteGroupId = createNextHopPointer(getRemoteSelectGroupKey(parentVpnId, destPrefix));
 917         if (remoteGroupId == FibConstants.INVALID_GROUP_ID) {
 918             LOG.error("Unable to allocate/retrieve remote groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 919         }
 920         long localGroupId = createNextHopPointer(getLocalSelectGroupKey(parentVpnId, destPrefix));
 921         if (localGroupId == FibConstants.INVALID_GROUP_ID) {
 922             LOG.error("Unable to allocate/retrieve local groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 923         }
 924         GroupEntity remoteGroupEntity = MDSALUtil.buildGroupEntity(
 925                 dpnId, remoteGroupId, destPrefix, GroupTypes.GroupSelect, Collections.emptyList());
 926         GroupEntity localGroupEntity = MDSALUtil.buildGroupEntity(
 927                 dpnId, localGroupId, destPrefix, GroupTypes.GroupSelect, Collections.emptyList());
 928         String jobKey = FibUtil.getCreateLocalNextHopJobKey(parentVpnId, dpnId, destPrefix);
 929         jobCoordinator.enqueueJob(jobKey, () -> {
 930             mdsalApiManager.syncRemoveGroup(remoteGroupEntity);
 931             mdsalApiManager.syncRemoveGroup(localGroupEntity);
 932             if (LOG.isDebugEnabled()) {
 933                 LOG.debug("Finished removing GroupEntity with jobCoordinator key {} remoteGroupEntity.groupId {}"
 934                     + "localGroupEntity.groupId {}", jobKey, remoteGroupId, localGroupId);
 935             }
 936             return Collections.emptyList();
 937         });
 938     }
 939
 940     long createNextHopGroups(Uint32 vpnId, String rd, Uint64 dpnId, VrfEntry vrfEntry,
 941             @Nullable Routes routes, List<Routes> vpnExtraRoutes) {
 942         List<BucketInfo> localBucketInfo = new ArrayList<>();
 943         List<Routes> clonedVpnExtraRoutes  = new ArrayList<>(vpnExtraRoutes);
 944         if (clonedVpnExtraRoutes.contains(routes)) {
 945             localBucketInfo.addAll(getBucketsForLocalNexthop(vpnId, dpnId, vrfEntry, routes));
 946             clonedVpnExtraRoutes.remove(routes);
 947         }
 948         List<BucketInfo> remoteBucketInfo =
 949             new ArrayList<>(getBucketsForRemoteNexthop(vpnId, dpnId, vrfEntry, rd, clonedVpnExtraRoutes));
 950         return setupLoadBalancingNextHop(vpnId, dpnId,
 951             vrfEntry.getDestPrefix(), localBucketInfo, remoteBucketInfo);
 952     }
 953
 954     private List<BucketInfo> getBucketsForLocalNexthop(Uint32 vpnId, Uint64 dpnId,
 955             VrfEntry vrfEntry, Routes routes) {
 956         @Nullable List<String> nexthopIpList = routes.getNexthopIpList();
 957         if (LOG.isDebugEnabled()) {
 958             LOG.debug("NexthopManager.getBucketsForLocalNexthop invoked with vpnId {} dpnId {} "
 959                             + " vrfEntry.routePaths {}, routes.nexthopList {}", vpnId, dpnId, vrfEntry.getRoutePaths(),
 960                 nexthopIpList);
 961         }
 962         List<BucketInfo> listBucketInfo = new CopyOnWriteArrayList<>();
 963         if (nexthopIpList != null) {
 964             nexthopIpList.parallelStream().forEach(nextHopIp -> {
 965                 String localNextHopIP;
 966                 if (isIpv4Address(nextHopIp)) {
 967                     localNextHopIP = nextHopIp + NwConstants.IPV4PREFIX;
 968                 } else {
 969                     localNextHopIP = nextHopIp + NwConstants.IPV6PREFIX;
 970                 }
 971                 Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 972                 if (localNextHopInfo != null) {
 973                     long groupId = getLocalNextHopGroup(vpnId, localNextHopIP);
 974                     if (groupId == FibConstants.INVALID_GROUP_ID) {
 975                         LOG.error("Unable to allocate groupId for vpnId {} , prefix {} , interface {}", vpnId,
 976                             vrfEntry.getDestPrefix(), localNextHopInfo.getVpnInterfaceName());
 977                         return;
 978                     }
 979                     List<ActionInfo> actionsInfos =
 980                         Collections.singletonList(new ActionGroup(groupId));
 981                     BucketInfo bucket = new BucketInfo(actionsInfos);
 982                     bucket.setWeight(1);
 983                     listBucketInfo.add(bucket);
 984                 }
 985             });
 986         }
 987         LOG.trace("LOCAL: listbucket {}, vpnId {}, dpnId {}, routes {}", listBucketInfo, vpnId, dpnId, routes);
 988         return listBucketInfo;
 989     }
 990
 991     private List<BucketInfo> getBucketsForRemoteNexthop(Uint32 vpnId, Uint64 dpnId, VrfEntry vrfEntry, String rd,
 992             List<Routes> vpnExtraRoutes) {
 993         List<BucketInfo> listBucketInfo = new ArrayList<>();
 994         Map<String, List<ActionInfo>> egressActionMap = new HashMap<>();
 995         vpnExtraRoutes.stream().filter(vpnExtraRoute -> vpnExtraRoute.getNexthopIpList() != null).forEach(
 996             vpnExtraRoute -> vpnExtraRoute.getNexthopIpList().forEach(nextHopIp -> {
 997                 String nextHopPrefixIp;
 998                 if (isIpv4Address(nextHopIp)) {
 999                     nextHopPrefixIp = nextHopIp + NwConstants.IPV4PREFIX;
1000                 } else {
1001                     nextHopPrefixIp = nextHopIp + NwConstants.IPV6PREFIX;
1002                 }
1003                 List<String> tepIpAddresses = fibUtil.getNextHopAddresses(rd, nextHopPrefixIp);
1004                 if (tepIpAddresses.isEmpty()) {
1005                     return;
1006                 }
1007                 // There would be only one nexthop address for a VM ip which would be the tep Ip
1008                 String tepIp = tepIpAddresses.get(0);
1009                 AdjacencyResult adjacencyResult = getRemoteNextHopPointer(dpnId, vpnId,
1010                     vrfEntry.getDestPrefix(), tepIp, TunnelTypeVxlan.class);
1011                 if (adjacencyResult == null) {
1012                     return;
1013                 }
1014                 String egressInterface = adjacencyResult.getInterfaceName();
1015                 if (!FibUtil.isTunnelInterface(adjacencyResult)) {
1016                     return;
1017                 }
1018                 Class<? extends TunnelTypeBase> tunnelType =
1019                     VpnExtraRouteHelper.getTunnelType(itmManager, egressInterface);
1020                 StateTunnelList ifState = null;
1021                 try {
1022                     ifState = fibUtil.getTunnelState(egressInterface);
1023                     if (ifState == null || ifState.getOperState() != TunnelOperStatus.Up) {
1024                         LOG.trace("Tunnel is not up for interface {}", egressInterface);
1025                         return;
1026                     }
1027                 } catch (ReadFailedException e) {
1028                     LOG.error("getBucketsForRemoteNexthop: error in fetching tunnel state for interface {}",
1029                         egressInterface, e);
1030                     return;
1031                 }
1032                 if (!TunnelTypeVxlan.class.equals(tunnelType)) {
1033                     return;
1034                 }
1035                 Uint32 label = FibUtil.getLabelFromRoutePaths(vrfEntry).get();
1036                 Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, nextHopPrefixIp);
1037                 if (prefixInfo == null) {
1038                     LOG.error("No prefix info found for prefix {} in rd {} for VPN {}", nextHopPrefixIp, rd,
1039                         vpnId);
1040                     return;
1041                 }
1042                 Uint32 tunnelId;
1043                 if (FibUtil.isVxlanNetwork(prefixInfo.getNetworkType())) {
1044                     tunnelId = prefixInfo.getSegmentationId();
1045                 } else {
1046                     LOG.warn("Network is not of type VXLAN for prefix {}."
1047                         + "Going with default Lport Tag.", prefixInfo.toString());
1048                     tunnelId = label;
1049                 }
1050                 List<ActionInfo> actionInfos = new ArrayList<>();
1051                 actionInfos.add(new ActionSetFieldTunnelId(Uint64.valueOf(tunnelId.longValue())));
1052                 String ifName = prefixInfo.getVpnInterfaceName();
1053                 String vpnName = fibUtil.getVpnNameFromId(vpnId);
1054                 if (vpnName == null) {
1055                     return;
1056                 }
1057                 String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, nextHopPrefixIp);
1058                 actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
1059                     new MacAddress(macAddress)));
1060                 List<ActionInfo> egressActions;
1061                 if (egressActionMap.containsKey(egressInterface)) {
1062                     egressActions = egressActionMap.get(egressInterface);
1063                 } else {
1064                     egressActions = getEgressActionsForInterface(egressInterface, actionInfos.size(),
1065                             true, vpnId, vrfEntry.getDestPrefix());
1066                     if (egressActions.isEmpty()) {
1067                         LOG.error("Skipping getBucketsForRemoteNexthop: Empty list of egress actions received for "
1068                                         + "interface {} on dpn {} for vpn {} prefix {} nextHop {}", ifName, dpnId,
1069                                 vpnId, vrfEntry.getDestPrefix(), nextHopPrefixIp);
1070                     }
1071                     egressActionMap.put(egressInterface, egressActions);
1072                 }
1073                 if (egressActions.isEmpty()) {
1074                     LOG.error("Failed to retrieve egress action for prefix {} route-paths {}"
1075                             + " interface {}." + " Aborting remote FIB entry creation.",
1076                         vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), egressInterface);
1077                 }
1078                 actionInfos.addAll(egressActions);
1079                 BucketInfo bucket = new BucketInfo(actionInfos);
1080                 bucket.setWeight(1);
1081                 listBucketInfo.add(bucket);
1082             }));
1083         LOG.trace("LOCAL: listbucket {}, rd {}, dpnId {}, routes {}", listBucketInfo, rd, dpnId, vpnExtraRoutes);
1084         return listBucketInfo;
1085     }
1086
1087     public void createDcGwLoadBalancingGroup(Uint64 dpnId, String destinationIp,
1088                                              Class<? extends TunnelTypeBase> tunnelType) {
1089         jobCoordinator.enqueueJob(FibHelper.getJobKeyForDcGwLoadBalancingGroup(dpnId), () -> {
1090             List<ListenableFuture<Void>> futures = new ArrayList<>();
1091             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operationalTx -> {
1092                 synchronized (getDcGateWaySyncKey(destinationIp)) {
1093                     FibUtil.addL3vpnDcGateWay(destinationIp, operationalTx);
1094                 }
1095                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, configTx -> {
1096                     List<String> availableDcGws = getDcGwIps();
1097                     Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
1098                     int noOfDcGws = availableDcGws.size();
1099                     if (noOfDcGws == 1) {
1100                         LOG.trace("There are no enough DC GateWays {} present to program LB group", availableDcGws);
1101                         return;
1102                     }
1103                     if (availableDcGws.contains(destinationIp)) {
1104                         availableDcGws.remove(destinationIp);
1105                     }
1106                     availableDcGws.forEach(dcGwIp -> {
1107                         List<String> dcGws = Arrays.asList(dcGwIp, destinationIp);
1108                         Collections.sort(dcGws);
1109                         String groupIdKey = FibUtil.getGreLbGroupKey(dcGws);
1110                         Long groupId = createNextHopPointer(groupIdKey);
1111                         List<Bucket> listBucket = new ArrayList<>();
1112                         for (int index = 0; index < dcGws.size(); index++) {
1113                             if (isTunnelUp(dcGws.get(index), dpnId, tunnelType)) {
1114                                 listBucket.add(buildBucketForDcGwLbGroup(dcGws.get(index),
1115                                         dpnId, index, tunnelType, true));
1116                             }
1117                         }
1118                         Group group = MDSALUtil.buildGroup(groupId, groupIdKey, GroupTypes.GroupSelect,
1119                                         MDSALUtil.buildBucketLists(listBucket));
1120                         mdsalApiManager.addGroup(configTx, dpnId, group);
1121                         FibUtil.updateLbGroupInfo(dpnId, groupIdKey, groupId.toString(), operationalTx);
1122                         LOG.trace("LB group {} towards DC-GW installed on dpn {}. Group - {}",
1123                                 groupIdKey, dpnId, group);
1124                     });
1125                 }));
1126             }));
1127             return futures;
1128         }, RETRY_COUNT);
1129     }
1130
1131     private String getDcGateWaySyncKey(String destinationIp) {
1132         String mutex = new StringBuilder().append("L3vpncDcGateWay").append(destinationIp).toString();
1133         return mutex.intern();
1134     }
1135
1136     private List<String> getDcGwIps() {
1137         InstanceIdentifier<DcGatewayIpList> dcGatewayIpListid =
1138                 InstanceIdentifier.builder(DcGatewayIpList.class).build();
1139         DcGatewayIpList dcGatewayIpListConfig;
1140         try {
1141             dcGatewayIpListConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1142                     LogicalDatastoreType.CONFIGURATION, dcGatewayIpListid).orElse(null);
1143         } catch (ExecutionException | InterruptedException e) {
1144             LOG.error("getDcGwIps: Exception while reading DcGatewayIpList DS", e);
1145             return Collections.emptyList();
1146         }
1147         if (dcGatewayIpListConfig == null) {
1148             return Collections.emptyList();
1149         }
1150         return dcGatewayIpListConfig.getDcGatewayIp()
1151                 .stream()
1152                 .filter(dcGwIp -> dcGwIp.getTunnnelType().equals(TunnelTypeMplsOverGre.class))
1153                 .map(dcGwIp -> dcGwIp.getIpAddress().stringValue()).sorted()
1154                 .collect(toList());
1155     }
1156
1157     private boolean isTunnelUp(String dcGwIp, Uint64 dpnId, Class<? extends TunnelTypeBase> tunnelType) {
1158         String tunnelName = getTunnelRemoteNextHopPointer(dpnId, dcGwIp, tunnelType);
1159         if (tunnelName != null) {
1160             InstanceIdentifier<StateTunnelList> tunnelStateId =
1161                     InstanceIdentifier.builder(TunnelsState.class).child(
1162                             StateTunnelList.class, new StateTunnelListKey(tunnelName)).build();
1163             try {
1164                 return SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
1165                         tunnelStateId).map(StateTunnelList::getOperState)
1166                         .orElse(TunnelOperStatus.Down) == TunnelOperStatus.Up;
1167             } catch (ExecutionException | InterruptedException e) {
1168                 LOG.error("isTunnelUp: Exception while reading StateTunnelList DS for tunnel {} tunnelType {}",
1169                         tunnelName, tunnelType, e);
1170                 return false;
1171             }
1172         }
1173         return false;
1174     }
1175
1176     private List<Action> getEgressActions(String interfaceName, int actionKey) {
1177         List<Action> actions = Collections.emptyList();
1178         try {
1179             GetEgressActionsForInterfaceInputBuilder egressAction =
1180                     new GetEgressActionsForInterfaceInputBuilder().setIntfName(interfaceName).setActionKey(actionKey);
1181             Future<RpcResult<GetEgressActionsForInterfaceOutput>> result =
1182                     odlInterfaceRpcService.getEgressActionsForInterface(egressAction.build());
1183             RpcResult<GetEgressActionsForInterfaceOutput> rpcResult = result.get();
1184             if (!rpcResult.isSuccessful()) {
1185                 LOG.warn("RPC Call to Get egress actions for interface {} returned with Errors {}",
1186                         interfaceName, rpcResult.getErrors());
1187             } else {
1188                 actions = rpcResult.getResult().nonnullAction();
1189             }
1190         } catch (InterruptedException | ExecutionException e) {
1191             LOG.warn("Exception when egress actions for interface {}", interfaceName, e);
1192         }
1193         return actions;
1194     }
1195
1196     /**
1197      * This method is invoked when the neighbor is removed from DS.
1198      * All the LB groups which point to the given destination will be deleted.
1199      */
1200     public void removeDcGwLoadBalancingGroup(Uint64 dpnId,
1201             String destinationIp) {
1202         jobCoordinator.enqueueJob(FibHelper.getJobKeyForDcGwLoadBalancingGroup(dpnId), () -> {
1203             List<String> availableDcGws = fibUtil.getL3VpnDcGateWays();
1204             if (availableDcGws.contains(destinationIp)) {
1205                 availableDcGws.remove(destinationIp);
1206             }
1207             List<ListenableFuture<Void>> futures = new ArrayList<>();
1208             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operationalTx -> {
1209                 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx -> {
1210                     availableDcGws.forEach(dcGwIp -> {
1211                         List<String> dcGws = Arrays.asList(dcGwIp, destinationIp);
1212                         Collections.sort(dcGws);
1213                         String nextHopKey = FibUtil.getGreLbGroupKey(dcGws);
1214                         Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
1215                         if (!optionalNextHops.isPresent()) {
1216                             return;
1217                         }
1218                         Nexthops nexthops = optionalNextHops.get();
1219                         final String groupId = nexthops.getGroupId();
1220                         final long groupIdValue = Long.parseLong(groupId);
1221                         Group group = MDSALUtil.buildGroup(groupIdValue, nextHopKey, GroupTypes.GroupSelect,
1222                                 MDSALUtil.buildBucketLists(Collections.emptyList()));
1223                         LOG.trace("Removed LB group {} on dpn {}", group, dpnId);
1224                         try {
1225                             mdsalApiManager.removeGroup(configTx, dpnId, group);
1226                         } catch (ExecutionException | InterruptedException e) {
1227                             LOG.error("Group removal failed for group {} with exception", groupId, e);
1228                         }
1229                         removeNextHopPointer(nextHopKey);
1230                         FibUtil.removeOrUpdateNextHopInfo(dpnId, nextHopKey, groupId, nexthops, operationalTx);
1231                     });
1232                     synchronized (getDcGateWaySyncKey(destinationIp)) {
1233                         FibUtil.removeL3vpnDcGateWay(destinationIp, operationalTx);
1234                     }
1235                 }));
1236             }));
1237             return futures;
1238         }, RETRY_COUNT);
1239     }
1240
1241     /**
1242      * This method is invoked when the tunnel status is deleted.
1243      * All the buckets which point to given destination will be marked down.
1244      */
1245     public void updateDcGwLoadBalancingGroup(Uint64 dpnId, String destinationIp,
1246             boolean isTunnelUp, Class<? extends TunnelTypeBase> tunnelType) {
1247         jobCoordinator.enqueueJob(FibHelper.getJobKeyForDcGwLoadBalancingGroup(dpnId), () -> {
1248             List<String> availableDcGws = fibUtil.getL3VpnDcGateWays();
1249             if (availableDcGws.contains(destinationIp)) {
1250                 availableDcGws.remove(destinationIp);
1251             }
1252             List<ListenableFuture<Void>> futures = new ArrayList<>();
1253             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx -> {
1254                 availableDcGws.forEach(dcGwIp -> {
1255                     List<String> dcGws = Arrays.asList(dcGwIp, destinationIp);
1256                     Collections.sort(dcGws);
1257                     String nextHopKey = FibUtil.getGreLbGroupKey(dcGws);
1258                     int bucketId = dcGws.indexOf(destinationIp);
1259                     Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
1260                     if (!optionalNextHops.isPresent()) {
1261                         return;
1262                     }
1263                     Nexthops nexthops = optionalNextHops.get();
1264                     final String groupId = nexthops.getGroupId();
1265                     final long groupIdValue = Long.parseLong(groupId);
1266                     Bucket bucket = buildBucketForDcGwLbGroup(destinationIp, dpnId, bucketId, tunnelType, isTunnelUp);
1267                     LOG.trace("updated bucket {} to group {} on dpn {}.", bucket, groupId, dpnId);
1268                     try {
1269                         mdsalApiManager.addBucket(configTx, dpnId, groupIdValue, bucket);
1270                     } catch (ExecutionException | InterruptedException e) {
1271                         LOG.error("Bucket addition failed for bucket {} with exception", bucketId, e);
1272                     }
1273                 });
1274             }));
1275             return futures;
1276         }, RETRY_COUNT);
1277     }
1278
1279     private Bucket buildBucketForDcGwLbGroup(String ipAddress, Uint64 dpnId,
1280             int index, Class<? extends TunnelTypeBase> tunnelType, boolean isTunnelUp) {
1281         List<Action> listAction = new ArrayList<>();
1282         // ActionKey 0 goes to mpls label.
1283         int actionKey = 1;
1284         listAction.add(new ActionPushMpls().buildAction());
1285         listAction.add(new ActionRegMove(actionKey++, FibConstants.NXM_REG_MAPPING
1286                 .get(index), 0, 19).buildAction());
1287         String tunnelInterfaceName = getTunnelInterfaceName(dpnId, IpAddressBuilder.getDefaultInstance(ipAddress),
1288             tunnelType);
1289         List<Action> egressActions = getEgressActions(tunnelInterfaceName, actionKey++);
1290         if (!egressActions.isEmpty()) {
1291             listAction.addAll(getEgressActions(tunnelInterfaceName, actionKey++));
1292         } else {
1293             // clear off actions if there is no egress actions.
1294             listAction = Collections.emptyList();
1295         }
1296         long watchPort = MDSALUtil.WATCH_PORT;
1297         if (!isTunnelUp) {
1298             watchPort = 0xFFFFFFFEL;
1299         }
1300         //OVS expects a non-zero weight value for load balancing to happen in select groups
1301         return MDSALUtil.buildBucket(listAction, SELECT_GROUP_WEIGHT, index,
1302                 watchPort, MDSALUtil.WATCH_GROUP);
1303     }
1304
1305     public void programDcGwLoadBalancingGroup(Uint64 dpnId, String destinationIp,
1306                                               int addRemoveOrUpdate, boolean isTunnelUp,
1307                                               Class<? extends TunnelTypeBase> tunnelType) {
1308         if (NwConstants.ADD_FLOW == addRemoveOrUpdate) {
1309             createDcGwLoadBalancingGroup(dpnId, destinationIp, tunnelType);
1310         } else if (NwConstants.DEL_FLOW == addRemoveOrUpdate) {
1311             removeDcGwLoadBalancingGroup(dpnId, destinationIp);
1312         } else if (NwConstants.MOD_FLOW == addRemoveOrUpdate) {
1313             updateDcGwLoadBalancingGroup(dpnId, destinationIp, isTunnelUp, tunnelType);
1314         }
1315     }
1316 }