fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/NexthopManager.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  11 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  12 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  13
  14 import com.google.common.base.Optional;
  15 import com.google.common.base.Preconditions;
  16 import java.math.BigInteger;
  17 import java.util.ArrayList;
  18 import java.util.Collections;
  19 import java.util.HashMap;
  20 import java.util.List;
  21 import java.util.Locale;
  22 import java.util.Map;
  23 import java.util.Objects;
  24 import java.util.concurrent.CopyOnWriteArrayList;
  25 import java.util.concurrent.ExecutionException;
  26 import java.util.concurrent.Future;
  27 import javax.annotation.Nullable;
  28 import javax.annotation.PreDestroy;
  29 import javax.inject.Inject;
  30 import javax.inject.Singleton;
  31 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  32 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  33 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  34 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  35 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  36 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  37 import org.opendaylight.genius.itm.globals.ITMConstants;
  38 import org.opendaylight.genius.mdsalutil.ActionInfo;
  39 import org.opendaylight.genius.mdsalutil.BucketInfo;
  40 import org.opendaylight.genius.mdsalutil.GroupEntity;
  41 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  42 import org.opendaylight.genius.mdsalutil.NwConstants;
  43 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  44 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  45 import org.opendaylight.genius.mdsalutil.actions.ActionOutput;
  46 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
  47 import org.opendaylight.genius.mdsalutil.actions.ActionPushVlan;
  48 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  49 import org.opendaylight.genius.mdsalutil.actions.ActionRegMove;
  50 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetSource;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldVlanVid;
  54 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  55 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  56 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  57 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  58 import org.opendaylight.netvirt.fibmanager.api.L3VPNTransportTypes;
  59 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  60 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev170119.L2vlan;
  61 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev170119.Tunnel;
  62 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
  63 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfaceType;
  64 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.OutputActionCase;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.PushVlanActionCase;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.SetFieldCase;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInput;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInputBuilder;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutput;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInput;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInputBuilder;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolOutput;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInput;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInputBuilder;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdOutput;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceInputBuilder;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceOutput;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelOperStatus;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelsState;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelListKey;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelInputBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelOutput;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameInputBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameOutput;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.lockmanager.rev160413.LockManagerService;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupInput;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupInputBuilder;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupOutput;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.SalGroupService;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupId;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupRef;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.Buckets;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeRef;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeBase;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeFlat;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeVlan;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.L3nexthop;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthops;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthopsKey;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopKey;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacencies;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacenciesBuilder;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacenciesKey;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpn;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpnBuilder;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.dpid.l3vpn.lb.nexthops.DpnLbNexthops;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.lb.nexthops.Nexthops;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.add.group.input.buckets.bucket.action.action.NxActionResubmitRpcAddGroupCase;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.nodes.node.table.flow.instructions.instruction.instruction.apply.actions._case.apply.actions.action.action.NxActionRegLoadNodesNodeTableFlowApplyActionsCase;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.nx.action.reg.load.grouping.NxRegLoad;
 133 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 134 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 135 import org.opendaylight.yangtools.yang.common.RpcResult;
 136 import org.slf4j.Logger;
 137 import org.slf4j.LoggerFactory;
 138
 139 @Singleton
 140 public class NexthopManager implements AutoCloseable {
 141     private static final Logger LOG = LoggerFactory.getLogger(NexthopManager.class);
 142     private static final String NEXTHOP_ID_POOL_NAME = "nextHopPointerPool";
 143     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 144     private static final long WAIT_TIME_TO_ACQUIRE_LOCK = 3000L;
 145     private static final int SELECT_GROUP_WEIGHT = 1;
 146
 147     private final DataBroker dataBroker;
 148     private final ManagedNewTransactionRunner txRunner;
 149     private final IMdsalApiManager mdsalApiManager;
 150     private final OdlInterfaceRpcService odlInterfaceRpcService;
 151     private final ItmRpcService itmManager;
 152     private final IdManagerService idManager;
 153     private final IElanService elanService;
 154     private final LockManagerService lockManager;
 155     private final SalGroupService salGroupService;
 156     private final JobCoordinator jobCoordinator;
 157     private final FibUtil fibUtil;
 158     private final IInterfaceManager interfaceManager;
 159     private volatile L3VPNTransportTypes configuredTransportTypeL3VPN = L3VPNTransportTypes.Invalid;
 160
 161     /**
 162      * Provides nexthop functions.
 163      * Creates group ID pool
 164      *
 165      * @param dataBroker       - dataBroker reference
 166      * @param mdsalApiManager  - mdsalApiManager reference
 167      * @param idManager        - idManager reference
 168      * @param odlInterfaceRpcService - odlInterfaceRpcService reference
 169      * @param itmManager       - itmManager reference
 170      */
 171     @Inject
 172     public NexthopManager(final DataBroker dataBroker,
 173                           final IMdsalApiManager mdsalApiManager,
 174                           final IdManagerService idManager,
 175                           final OdlInterfaceRpcService odlInterfaceRpcService,
 176                           final ItmRpcService itmManager,
 177                           final LockManagerService lockManager,
 178                           final IElanService elanService,
 179                           final SalGroupService salGroupService,
 180                           final JobCoordinator jobCoordinator,
 181                           final FibUtil fibUtil,
 182                           final IInterfaceManager interfaceManager) {
 183         this.dataBroker = dataBroker;
 184         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 185         this.mdsalApiManager = mdsalApiManager;
 186         this.idManager = idManager;
 187         this.odlInterfaceRpcService = odlInterfaceRpcService;
 188         this.itmManager = itmManager;
 189         this.elanService = elanService;
 190         this.salGroupService = salGroupService;
 191         this.jobCoordinator = jobCoordinator;
 192         this.fibUtil = fibUtil;
 193         this.lockManager = lockManager;
 194         this.interfaceManager = interfaceManager;
 195         createIdPool();
 196     }
 197
 198     private void createIdPool() {
 199         CreateIdPoolInput createPool = new CreateIdPoolInputBuilder()
 200             .setPoolName(NEXTHOP_ID_POOL_NAME)
 201             .setLow(150000L)
 202             .setHigh(175000L)
 203             .build();
 204         try {
 205             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPool);
 206             if (result != null && result.get().isSuccessful()) {
 207                 LOG.info("Created IdPool for NextHopPointerPool");
 208             }
 209         } catch (InterruptedException | ExecutionException e) {
 210             LOG.error("Failed to create idPool for NextHopPointerPool", e);
 211         }
 212     }
 213
 214     private String getNextHopKey(long vpnId, String ipAddress) {
 215         return "nexthop." + vpnId + ipAddress;
 216     }
 217
 218     String getRemoteSelectGroupKey(long vpnId, String ipAddress) {
 219         return "remote.ecmp.nexthop." + vpnId + ipAddress;
 220     }
 221
 222     String getLocalSelectGroupKey(long vpnId, String ipAddress) {
 223         return "local.ecmp.nexthop." + vpnId + ipAddress;
 224     }
 225
 226     public ItmRpcService getItmManager() {
 227         return itmManager;
 228     }
 229
 230     protected long createNextHopPointer(String nexthopKey) {
 231         AllocateIdInput getIdInput = new AllocateIdInputBuilder()
 232             .setPoolName(NEXTHOP_ID_POOL_NAME).setIdKey(nexthopKey)
 233             .build();
 234         //TODO: Proper error handling once IdManager code is complete
 235         try {
 236             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
 237             RpcResult<AllocateIdOutput> rpcResult = result.get();
 238             return rpcResult.getResult().getIdValue();
 239         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 240             LOG.trace("", e);
 241         }
 242         return 0;
 243     }
 244
 245     protected void removeNextHopPointer(String nexthopKey) {
 246         ReleaseIdInput idInput = new ReleaseIdInputBuilder()
 247             .setPoolName(NEXTHOP_ID_POOL_NAME)
 248             .setIdKey(nexthopKey).build();
 249         try {
 250             RpcResult<ReleaseIdOutput> rpcResult = idManager.releaseId(idInput).get();
 251             if (!rpcResult.isSuccessful()) {
 252                 LOG.error("RPC Call to Get Unique Id for nexthopKey {} returned with Errors {}",
 253                         nexthopKey, rpcResult.getErrors());
 254             }
 255         } catch (InterruptedException | ExecutionException e) {
 256             LOG.warn("Exception when getting Unique Id for key {}", nexthopKey, e);
 257         }
 258     }
 259
 260     protected List<ActionInfo> getEgressActionsForInterface(final String ifName, int actionKey,
 261                                                             boolean isTunnelInterface) {
 262         List<Action> actions;
 263         try {
 264             if (isTunnelInterface && interfaceManager.isItmDirectTunnelsEnabled()) {
 265                 RpcResult<GetEgressActionsForTunnelOutput> rpcResult =
 266                         itmManager.getEgressActionsForTunnel(new GetEgressActionsForTunnelInputBuilder()
 267                                 .setIntfName(ifName).build()).get();
 268                 if (!rpcResult.isSuccessful()) {
 269                     LOG.error("RPC Call to Get egress tunnel actions for interface {} returned with Errors {}",
 270                             ifName, rpcResult.getErrors());
 271                     return Collections.emptyList();
 272                 } else {
 273                     actions = rpcResult.getResult().nonnullAction();
 274                 }
 275             } else {
 276                 RpcResult<GetEgressActionsForInterfaceOutput> rpcResult = odlInterfaceRpcService
 277                         .getEgressActionsForInterface(new GetEgressActionsForInterfaceInputBuilder()
 278                                 .setIntfName(ifName).build()).get();
 279                 if (!rpcResult.isSuccessful()) {
 280                     LOG.error("RPC Call to Get egress vm actions for interface {} returned with Errors {}",
 281                             ifName, rpcResult.getErrors());
 282                     return Collections.emptyList();
 283                 } else {
 284                     actions = rpcResult.getResult().nonnullAction();
 285                 }
 286             }
 287             List<ActionInfo> listActionInfo = new ArrayList<>();
 288             for (Action action : actions) {
 289                 actionKey = action.key().getOrder() + actionKey;
 290                 org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action
 291                     actionClass = action.getAction();
 292                 if (actionClass instanceof OutputActionCase) {
 293                     listActionInfo.add(new ActionOutput(actionKey,
 294                         ((OutputActionCase) actionClass).getOutputAction().getOutputNodeConnector()));
 295                 } else if (actionClass instanceof PushVlanActionCase) {
 296                     listActionInfo.add(new ActionPushVlan(actionKey));
 297                 } else if (actionClass instanceof SetFieldCase) {
 298                     if (((SetFieldCase) actionClass).getSetField().getVlanMatch() != null) {
 299                         int vlanVid = ((SetFieldCase) actionClass).getSetField().getVlanMatch()
 300                             .getVlanId().getVlanId().getValue();
 301                         listActionInfo.add(new ActionSetFieldVlanVid(actionKey, vlanVid));
 302                     }
 303                 } else if (actionClass instanceof NxActionResubmitRpcAddGroupCase) {
 304                     Short tableId = ((NxActionResubmitRpcAddGroupCase) actionClass).getNxResubmit().getTable();
 305                     listActionInfo.add(new ActionNxResubmit(actionKey, tableId));
 306                 } else if (actionClass instanceof NxActionRegLoadNodesNodeTableFlowApplyActionsCase) {
 307                     NxRegLoad nxRegLoad =
 308                         ((NxActionRegLoadNodesNodeTableFlowApplyActionsCase) actionClass).getNxRegLoad();
 309                     listActionInfo.add(new ActionRegLoad(actionKey, NxmNxReg6.class,
 310                         nxRegLoad.getDst().getStart(), nxRegLoad.getDst().getEnd(),
 311                         nxRegLoad.getValue().longValue()));
 312                 }
 313             }
 314             return listActionInfo;
 315         } catch (InterruptedException | ExecutionException e) {
 316             LOG.warn("Exception when egress actions for interface {}", ifName, e);
 317         }
 318         LOG.warn("Exception when egress actions for interface {}", ifName);
 319         return Collections.emptyList();
 320     }
 321
 322     @Nullable
 323     protected String getTunnelInterfaceName(BigInteger srcDpId, BigInteger dstDpId) {
 324         Class<? extends TunnelTypeBase> tunType = getReqTunType(getReqTransType().toUpperCase(Locale.getDefault()));
 325         Future<RpcResult<GetTunnelInterfaceNameOutput>> result;
 326         try {
 327             result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 328                 .setSourceDpid(srcDpId)
 329                 .setDestinationDpid(dstDpId)
 330                 .setTunnelType(tunType)
 331                 .build());
 332             RpcResult<GetTunnelInterfaceNameOutput> rpcResult = result.get();
 333             if (!rpcResult.isSuccessful()) {
 334                 LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
 335             } else {
 336                 return rpcResult.getResult().getInterfaceName();
 337             }
 338         } catch (InterruptedException | ExecutionException e) {
 339             LOG.warn("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstDpId, e);
 340         }
 341         return null;
 342     }
 343
 344     @Nullable
 345     protected String getTunnelInterfaceName(BigInteger srcDpId, org.opendaylight.yang.gen.v1.urn.ietf.params
 346         .xml.ns.yang.ietf.inet.types.rev130715.IpAddress dstIp, Class<? extends TunnelTypeBase> tunnelType) {
 347         Future<RpcResult<GetInternalOrExternalInterfaceNameOutput>> result;
 348         try {
 349             LOG.debug("Trying to fetch tunnel interface name for source dpn {} destIp {} tunType {}", srcDpId,
 350                     dstIp.stringValue(), tunnelType.getName());
 351             result = itmManager.getInternalOrExternalInterfaceName(new GetInternalOrExternalInterfaceNameInputBuilder()
 352                 .setSourceDpid(srcDpId)
 353                 .setDestinationIp(dstIp)
 354                 .setTunnelType(tunnelType)
 355                 .build());
 356             RpcResult<GetInternalOrExternalInterfaceNameOutput> rpcResult = result.get();
 357             if (!rpcResult.isSuccessful()) {
 358                 LOG.warn("RPC Call to getTunnelInterfaceName returned with Errors {}", rpcResult.getErrors());
 359             } else {
 360                 return rpcResult.getResult().getInterfaceName();
 361             }
 362         } catch (InterruptedException | ExecutionException e) {
 363             LOG.warn("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstIp, e);
 364         }
 365         return null;
 366     }
 367
 368
 369     public long getLocalNextHopGroup(long vpnId,
 370             String ipNextHopAddress) {
 371         long groupId = createNextHopPointer(getNextHopKey(vpnId, ipNextHopAddress));
 372         if (groupId == FibConstants.INVALID_GROUP_ID) {
 373             LOG.error("Unable to allocate groupId for vpnId {} , prefix {}", vpnId, ipNextHopAddress);
 374         }
 375         return groupId;
 376     }
 377
 378     public long getLocalSelectGroup(long vpnId,
 379             String ipNextHopAddress) {
 380         long groupId = createNextHopPointer(getLocalSelectGroupKey(vpnId, ipNextHopAddress));
 381         if (groupId == FibConstants.INVALID_GROUP_ID) {
 382             LOG.error("Unable to allocate groupId for vpnId {} , prefix {}", vpnId, ipNextHopAddress);
 383         }
 384         return groupId;
 385     }
 386
 387     public long createLocalNextHop(long vpnId, BigInteger dpnId, String ifName,
 388                                    String primaryIpAddress, String currDestIpPrefix,
 389                                    String gwMacAddress) {
 390         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 391         if (vpnName == null) {
 392             return 0;
 393         }
 394         String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, primaryIpAddress);
 395
 396         long groupId = createNextHopPointer(getNextHopKey(vpnId, primaryIpAddress));
 397         if (groupId == 0) {
 398             LOG.error("Unable to allocate groupId for vpnId {} , IntfName {}, primaryIpAddress {} curIpPrefix {}",
 399                     vpnId, ifName, primaryIpAddress, currDestIpPrefix);
 400             return groupId;
 401         }
 402         String nextHopLockStr = vpnId + primaryIpAddress;
 403         String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, currDestIpPrefix);
 404         jobCoordinator.enqueueJob(jobKey, () -> {
 405             try {
 406                 if (FibUtil.lockCluster(lockManager, nextHopLockStr, WAIT_TIME_TO_ACQUIRE_LOCK)) {
 407                     VpnNexthop nexthop = getVpnNexthop(vpnId, primaryIpAddress);
 408                     LOG.trace("nexthop: {} retrieved for vpnId {}, prefix {}, ifName {} on dpn {}", nexthop, vpnId,
 409                             primaryIpAddress, ifName, dpnId);
 410                     if (nexthop == null) {
 411                         String encMacAddress = macAddress == null
 412                                 ? fibUtil.getMacAddressFromPrefix(ifName, vpnName, primaryIpAddress) : macAddress;
 413                         List<BucketInfo> listBucketInfo = new ArrayList<>();
 414                         List<ActionInfo> listActionInfo = new ArrayList<>();
 415                         int actionKey = 0;
 416                         // MAC re-write
 417                         if (encMacAddress != null) {
 418                             if (gwMacAddress != null) {
 419                                 LOG.trace("The Local NextHop Group Source Mac {} for VpnInterface {} on VPN {}",
 420                                         gwMacAddress, ifName, vpnId);
 421                                 listActionInfo.add(new ActionSetFieldEthernetSource(actionKey++,
 422                                         new MacAddress(gwMacAddress)));
 423                             }
 424                             listActionInfo.add(new ActionSetFieldEthernetDestination(actionKey++,
 425                                     new MacAddress(encMacAddress)));
 426                             // listActionInfo.add(0, new ActionPopMpls());
 427                         } else {
 428                             // FIXME: Log message here.
 429                             LOG.debug("mac address for new local nexthop is null");
 430                         }
 431                         listActionInfo.addAll(getEgressActionsForInterface(ifName, actionKey, false));
 432                         BucketInfo bucket = new BucketInfo(listActionInfo);
 433
 434                         listBucketInfo.add(bucket);
 435                         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, groupId, primaryIpAddress,
 436                                 GroupTypes.GroupAll, listBucketInfo);
 437                         LOG.trace("Install LNH Group: id {}, mac address {}, interface {} for prefix {}", groupId,
 438                                 encMacAddress, ifName, primaryIpAddress);
 439                         //Try to install group directly on the DPN bypassing the FRM, in order to avoid waiting for the
 440                         // group to get installed before programming the flows
 441                         installGroupOnDpn(groupId, dpnId, primaryIpAddress, listBucketInfo,
 442                                 getNextHopKey(vpnId, primaryIpAddress), GroupTypes.GroupAll);
 443                         // install Group
 444                         mdsalApiManager.syncInstallGroup(groupEntity);
 445                         // update MD-SAL DS
 446                         addVpnNexthopToDS(dpnId, vpnId, primaryIpAddress, currDestIpPrefix, groupId);
 447
 448                     } else {
 449                         // Ignore adding new prefix , if it already exists
 450                         List<IpAdjacencies> prefixesList = nexthop.getIpAdjacencies();
 451                         IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currDestIpPrefix).build();
 452                         if (prefixesList != null && prefixesList.contains(prefix)) {
 453                             LOG.trace("Prefix {} is already present in l3nextHop {} ", currDestIpPrefix, nexthop);
 454                         } else {
 455                             IpAdjacenciesBuilder ipPrefixesBuilder =
 456                                     new IpAdjacenciesBuilder().withKey(new IpAdjacenciesKey(currDestIpPrefix));
 457                             LOG.trace("Updating prefix {} to vpnNextHop {} Operational DS", currDestIpPrefix, nexthop);
 458                             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
 459                                     getVpnNextHopIpPrefixIdentifier(vpnId, primaryIpAddress, currDestIpPrefix),
 460                                     ipPrefixesBuilder.build());
 461                         }
 462                     }
 463                 }
 464             } finally {
 465                 FibUtil.unlockCluster(lockManager, nextHopLockStr);
 466             }
 467             return Collections.emptyList();
 468         });
 469         return groupId;
 470     }
 471
 472     private void installGroupOnDpn(long groupId, BigInteger dpnId, String groupName, List<BucketInfo> bucketsInfo,
 473                                      String nextHopKey, GroupTypes groupType) {
 474         NodeRef nodeRef = FibUtil.buildNodeRef(dpnId);
 475         Buckets buckets = FibUtil.buildBuckets(bucketsInfo);
 476         GroupRef groupRef = new GroupRef(FibUtil.buildGroupInstanceIdentifier(groupId, dpnId));
 477         AddGroupInput input = new AddGroupInputBuilder().setNode(nodeRef).setGroupId(new GroupId(groupId))
 478                 .setBuckets(buckets).setGroupRef(groupRef).setGroupType(groupType)
 479                 .setGroupName(groupName).build();
 480         Future<RpcResult<AddGroupOutput>> groupStats = salGroupService.addGroup(input);
 481         RpcResult<AddGroupOutput> rpcResult = null;
 482         try {
 483             rpcResult = groupStats.get();
 484             if (rpcResult != null && rpcResult.isSuccessful()) {
 485                 LOG.info("Group {} with key {} has been successfully installed directly on dpn {}.", groupId,
 486                         nextHopKey, dpnId);
 487             } else {
 488                 LOG.error("Unable to install group {} with key {} directly on dpn {} due to {}.", groupId, nextHopKey,
 489                         dpnId, rpcResult != null ? rpcResult.getErrors() : null);
 490             }
 491         } catch (InterruptedException | ExecutionException e) {
 492             LOG.error("Error while installing group {} directly on dpn {}", groupId, dpnId);
 493         }
 494     }
 495
 496     protected void addVpnNexthopToDS(BigInteger dpnId, long vpnId, String primaryIpAddr,
 497                                      String currIpAddr, long egressPointer) {
 498         InstanceIdentifierBuilder<VpnNexthops> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
 499             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId));
 500
 501         List<IpAdjacencies> ipPrefixesList = new ArrayList<>();
 502         IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currIpAddr).build();
 503         ipPrefixesList.add(prefix);
 504         // Add nexthop to vpn node
 505         VpnNexthop nh = new VpnNexthopBuilder()
 506             .withKey(new VpnNexthopKey(primaryIpAddr))
 507             .setDpnId(dpnId)
 508             .setIpAdjacencies(ipPrefixesList)
 509             .setEgressPointer(egressPointer).build();
 510
 511         InstanceIdentifier<VpnNexthop> id1 = idBuilder
 512             .child(VpnNexthop.class, new VpnNexthopKey(primaryIpAddr)).build();
 513         LOG.trace("Adding vpnnextHop {} to Operational DS", nh);
 514         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id1, nh);
 515
 516     }
 517
 518     protected InstanceIdentifier<IpAdjacencies> getVpnNextHopIpPrefixIdentifier(long vpnId, String primaryIpAddress,
 519                                                                                 String ipPrefix) {
 520         InstanceIdentifier<IpAdjacencies> id = InstanceIdentifier.builder(L3nexthop.class)
 521                 .child(VpnNexthops.class, new VpnNexthopsKey(vpnId))
 522                 .child(VpnNexthop.class, new VpnNexthopKey(primaryIpAddress))
 523                 .child(IpAdjacencies.class, new IpAdjacenciesKey(ipPrefix)).build();
 524         return id;
 525     }
 526
 527     @Nullable
 528     protected VpnNexthop getVpnNexthop(long vpnId, String ipAddress) {
 529
 530         // check if vpn node is there
 531         InstanceIdentifierBuilder<VpnNexthops> idBuilder =
 532             InstanceIdentifier.builder(L3nexthop.class).child(VpnNexthops.class,
 533                 new VpnNexthopsKey(vpnId));
 534         InstanceIdentifier<VpnNexthops> id = idBuilder.build();
 535         Optional<VpnNexthops> vpnNexthops = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 536         if (vpnNexthops.isPresent()) {
 537             // get nexthops list for vpn
 538             List<VpnNexthop> nexthops = vpnNexthops.get().nonnullVpnNexthop();
 539             for (VpnNexthop nexthop : nexthops) {
 540                 if (Objects.equals(nexthop.getIpAddress(), ipAddress)) {
 541                     // return nexthop
 542                     LOG.trace("VpnNextHop : {}", nexthop);
 543                     return nexthop;
 544                 }
 545             }
 546             // return null if not found
 547         }
 548         return null;
 549     }
 550
 551     @Nullable
 552     public AdjacencyResult getRemoteNextHopPointer(BigInteger remoteDpnId, long vpnId, String prefixIp,
 553             @Nullable String nextHopIp, Class<? extends TunnelTypeBase> tunnelType) {
 554         String egressIfName = null;
 555         LOG.trace("getRemoteNextHopPointer: input [remoteDpnId {}, vpnId {}, prefixIp {}, nextHopIp {} ]", remoteDpnId,
 556             vpnId, prefixIp, nextHopIp);
 557
 558         Class<? extends InterfaceType> egressIfType;
 559         ElanInstance elanInstance = getElanInstanceForPrefix(vpnId, prefixIp);
 560         if (elanInstance != null) {
 561             egressIfType = getInterfaceType(elanInstance);
 562         } else {
 563             LOG.warn("Failed to determine network type for prefixIp {} using tunnel", prefixIp);
 564             egressIfType = Tunnel.class;
 565         }
 566
 567         if (Tunnel.class.equals(egressIfType)) {
 568             egressIfName = getTunnelRemoteNextHopPointer(remoteDpnId, nextHopIp, tunnelType);
 569         } else {
 570             egressIfName = getExtPortRemoteNextHopPointer(remoteDpnId, elanInstance);
 571         }
 572
 573         LOG.trace("NextHop pointer for prefixIp {} vpnId {} dpnId {} is {}", prefixIp, vpnId, remoteDpnId,
 574             egressIfName);
 575         return egressIfName != null ? new AdjacencyResult(egressIfName, egressIfType, nextHopIp,
 576                 prefixIp) : null;
 577     }
 578
 579     @Nullable
 580     public BigInteger getDpnForPrefix(long vpnId, String prefixIp) {
 581         VpnNexthop vpnNexthop = getVpnNexthop(vpnId, prefixIp);
 582         return vpnNexthop == null ? null : vpnNexthop.getDpnId();
 583     }
 584
 585     private void removeVpnNexthopFromDS(long vpnId, String ipPrefix) {
 586
 587         InstanceIdentifierBuilder<VpnNexthop> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
 588             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId))
 589             .child(VpnNexthop.class, new VpnNexthopKey(ipPrefix));
 590         InstanceIdentifier<VpnNexthop> id = idBuilder.build();
 591         // remove from DS
 592         LOG.trace("Removing vpn next hop from datastore : {}", id);
 593         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 594     }
 595
 596     public void removeLocalNextHop(BigInteger dpnId, Long vpnId, String primaryIpAddress, String currDestIpPrefix) {
 597         String nextHopLockStr = vpnId + primaryIpAddress;
 598         try {
 599             if (FibUtil.lockCluster(lockManager, nextHopLockStr, WAIT_TIME_TO_ACQUIRE_LOCK)) {
 600                 VpnNexthop nh = getVpnNexthop(vpnId, primaryIpAddress);
 601                 if (nh != null) {
 602                     List<IpAdjacencies> prefixesList = new ArrayList<>(nh.nonnullIpAdjacencies());
 603                     IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currDestIpPrefix).build();
 604                     prefixesList.remove(prefix);
 605                     if (prefixesList.isEmpty()) { //remove the group only if there are no more flows using this group
 606                         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, nh.getEgressPointer(),
 607                                 primaryIpAddress, GroupTypes.GroupAll, Collections.emptyList());
 608                         // remove Group ...
 609                         mdsalApiManager.removeGroup(groupEntity);
 610                         //update MD-SAL DS
 611                         removeVpnNexthopFromDS(vpnId, primaryIpAddress);
 612                         //release groupId
 613                         removeNextHopPointer(getNextHopKey(vpnId, primaryIpAddress));
 614                         LOG.debug("Local Next hop {} for {} {} on dpn {} successfully deleted",
 615                                 nh.getEgressPointer(), vpnId, primaryIpAddress, dpnId);
 616                     } else {
 617                         //remove the currIpPrefx from IpPrefixList of the vpnNexthop
 618                         LOG.trace("Removing the prefix {} from vpnNextHop {} Operational DS", currDestIpPrefix, nh);
 619                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
 620                                 getVpnNextHopIpPrefixIdentifier(vpnId, primaryIpAddress, currDestIpPrefix));
 621                     }
 622                 } else {
 623                     //throw error
 624                     LOG.error("Local NextHop for VpnId {} curIpPrefix {} on dpn {} primaryIpAddress {} not deleted",
 625                             vpnId, currDestIpPrefix, dpnId, primaryIpAddress);
 626                 }
 627             }
 628         } finally {
 629             FibUtil.unlockCluster(lockManager, nextHopLockStr);
 630         }
 631     }
 632
 633     public void setConfTransType(String service, String transportType) {
 634
 635         if (!service.equalsIgnoreCase("L3VPN")) {
 636             LOG.error("Incorrect service {} provided for setting the transport type.", service);
 637             return;
 638         }
 639
 640         L3VPNTransportTypes transType = L3VPNTransportTypes.validateTransportType(transportType
 641                 .toUpperCase(Locale.getDefault()));
 642
 643         if (transType != L3VPNTransportTypes.Invalid) {
 644             configuredTransportTypeL3VPN = transType;
 645         }
 646     }
 647
 648     public void writeConfTransTypeConfigDS() {
 649         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, getConfTransportTypeIdentifier(),
 650             createConfTransportType(configuredTransportTypeL3VPN.getTransportType()));
 651     }
 652
 653     public L3VPNTransportTypes getConfiguredTransportTypeL3VPN() {
 654         return this.configuredTransportTypeL3VPN;
 655     }
 656
 657     public String getReqTransType() {
 658         if (configuredTransportTypeL3VPN == L3VPNTransportTypes.Invalid) {
 659             /*
 660              * Restart scenario, Read from the ConfigDS.
 661              * if the value is Unset, cache value as VxLAN.
 662              */
 663             LOG.trace("configureTransportType is not yet set.");
 664             Optional<ConfTransportTypeL3vpn> configuredTransTypeFromConfig =
 665                 MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, getConfTransportTypeIdentifier());
 666
 667             if (configuredTransTypeFromConfig.isPresent()) {
 668                 if (TunnelTypeGre.class.equals(configuredTransTypeFromConfig.get().getTransportType())) {
 669                     configuredTransportTypeL3VPN = L3VPNTransportTypes.GRE;
 670                 } else {
 671                     configuredTransportTypeL3VPN = L3VPNTransportTypes.VxLAN;
 672                 }
 673                 LOG.trace("configuredTransportType set from config DS to {}",
 674                     getConfiguredTransportTypeL3VPN().getTransportType());
 675             } else {
 676                 setConfTransType("L3VPN", L3VPNTransportTypes.VxLAN.getTransportType());
 677                 LOG.trace("configuredTransportType is not set in the Config DS. VxLAN as default will be used.");
 678             }
 679         } else {
 680             LOG.trace("configuredTransportType is set as {}", getConfiguredTransportTypeL3VPN().getTransportType());
 681         }
 682         return getConfiguredTransportTypeL3VPN().getTransportType();
 683     }
 684
 685     public InstanceIdentifier<ConfTransportTypeL3vpn> getConfTransportTypeIdentifier() {
 686         return InstanceIdentifier.builder(ConfTransportTypeL3vpn.class).build();
 687     }
 688
 689     private ConfTransportTypeL3vpn createConfTransportType(String type) {
 690         ConfTransportTypeL3vpn confTransType;
 691         switch (type) {
 692             case ITMConstants.TUNNEL_TYPE_GRE:
 693                 confTransType = new ConfTransportTypeL3vpnBuilder().setTransportType(TunnelTypeGre.class).build();
 694                 LOG.trace("Setting the confTransportType to GRE.");
 695                 break;
 696             case ITMConstants.TUNNEL_TYPE_VXLAN:
 697                 confTransType = new ConfTransportTypeL3vpnBuilder().setTransportType(TunnelTypeVxlan.class).build();
 698                 LOG.trace("Setting the confTransportType to VxLAN.");
 699                 break;
 700             default:
 701                 LOG.trace("Invalid transport type {} passed to Config DS ", type);
 702                 confTransType = null;
 703                 break;
 704         }
 705         return confTransType;
 706     }
 707
 708     public Class<? extends TunnelTypeBase> getReqTunType(String transportType) {
 709         switch (transportType) {
 710             case "VXLAN":
 711                 return TunnelTypeVxlan.class;
 712             case "GRE":
 713                 return TunnelTypeGre.class;
 714             default:
 715                 return TunnelTypeMplsOverGre.class;
 716         }
 717     }
 718
 719     public String getTransportTypeStr(String tunType) {
 720         if (tunType.equals(TunnelTypeVxlan.class.toString())) {
 721             return ITMConstants.TUNNEL_TYPE_VXLAN;
 722         } else if (tunType.equals(TunnelTypeGre.class.toString())) {
 723             return ITMConstants.TUNNEL_TYPE_GRE;
 724         } else if (tunType.equals(TunnelTypeMplsOverGre.class.toString())) {
 725             return ITMConstants.TUNNEL_TYPE_MPLSoGRE;
 726         } else {
 727             return ITMConstants.TUNNEL_TYPE_INVALID;
 728         }
 729     }
 730
 731     @Override
 732     @PreDestroy
 733     public void close() {
 734         LOG.info("{} close", getClass().getSimpleName());
 735     }
 736
 737     // TODO Clean up the exception handling
 738     @SuppressWarnings("checkstyle:IllegalCatch")
 739     @Nullable
 740     private String getTunnelRemoteNextHopPointer(BigInteger remoteDpnId, String nextHopIp,
 741                                                  Class<? extends TunnelTypeBase> tunnelType) {
 742         if (nextHopIp != null && !nextHopIp.isEmpty()) {
 743             try {
 744                 // here use the config for tunnel type param
 745                 return getTunnelInterfaceName(remoteDpnId,
 746                     org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder
 747                         .getDefaultInstance(nextHopIp), tunnelType);
 748             } catch (Exception ex) {
 749                 LOG.error("Error while retrieving nexthop pointer for nexthop {} remoteDpn {}",
 750                         nextHopIp, remoteDpnId, ex);
 751             }
 752         }
 753
 754         return null;
 755     }
 756
 757     private String getExtPortRemoteNextHopPointer(BigInteger remoteDpnId, ElanInstance elanInstance) {
 758         return elanService.getExternalElanInterface(elanInstance.getElanInstanceName(), remoteDpnId);
 759     }
 760
 761     /**
 762      * Get the interface type associated with the type of ELAN used for routing
 763      * traffic to/from remote compute nodes.
 764      *
 765      * @param elanInstance The elan instance
 766      * @return L2vlan for flat/VLAN network type and Tunnel otherwise
 767      */
 768     private Class<? extends InterfaceType> getInterfaceType(ElanInstance elanInstance) {
 769         Class<? extends SegmentTypeBase> segmentType = elanInstance.getSegmentType();
 770         if (SegmentTypeFlat.class.equals(segmentType) || SegmentTypeVlan.class.equals(segmentType)) {
 771             return L2vlan.class;
 772         }
 773
 774         return Tunnel.class;
 775     }
 776
 777     private ElanInstance getElanInstanceForPrefix(long vpnId, String prefixIp) {
 778         ElanInstance elanInstance = null;
 779         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, prefixIp);
 780         if (prefix != null) {
 781             if (prefix.getNetworkId() != null) {
 782                 elanInstance = elanService.getElanInstance(prefix.getNetworkId().getValue());
 783             }
 784         }
 785
 786         return elanInstance;
 787     }
 788
 789     static class AdjacencyResult {
 790         private final String interfaceName;
 791         private final Class<? extends InterfaceType> interfaceType;
 792         private final String nextHopIp;
 793         private final String prefix;
 794
 795         AdjacencyResult(String interfaceName, Class<? extends InterfaceType> interfaceType, String nextHopIp,
 796                         String prefix) {
 797             this.interfaceName = interfaceName;
 798             this.interfaceType = interfaceType;
 799             this.nextHopIp = nextHopIp;
 800             this.prefix = prefix;
 801         }
 802
 803         public String getInterfaceName() {
 804             return interfaceName;
 805         }
 806
 807         public Class<? extends InterfaceType> getInterfaceType() {
 808             return interfaceType;
 809         }
 810
 811         public String getNextHopIp() {
 812             return nextHopIp;
 813         }
 814
 815         public String getPrefix() {
 816             return prefix;
 817         }
 818
 819         @Override
 820         public int hashCode() {
 821             final int prime = 31;
 822             int result = 1;
 823             result = prime * result + (interfaceName == null ? 0 : interfaceName.hashCode());
 824             return result;
 825         }
 826
 827         @Override
 828         public boolean equals(Object obj) {
 829             if (obj == null) {
 830                 return false;
 831             }
 832
 833             if (getClass() != obj.getClass()) {
 834                 return false;
 835             } else {
 836                 AdjacencyResult other = (AdjacencyResult) obj;
 837                 return interfaceName.equals(other.interfaceName);
 838             }
 839         }
 840     }
 841
 842     protected long setupLoadBalancingNextHop(Long parentVpnId, BigInteger dpnId,
 843             String destPrefix, List<BucketInfo> localBucketInfo, List<BucketInfo> remoteBucketInfo) {
 844         long remoteGroupId = createNextHopPointer(getRemoteSelectGroupKey(parentVpnId, destPrefix));
 845         if (remoteGroupId == FibConstants.INVALID_GROUP_ID) {
 846             LOG.error("Unable to allocate/retrieve remote groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 847             return remoteGroupId;
 848         }
 849         long localGroupId =  createNextHopPointer(getLocalSelectGroupKey(parentVpnId, destPrefix));
 850         if (localGroupId == FibConstants.INVALID_GROUP_ID) {
 851             LOG.error("Unable to allocate/retrieve local groupId for vpnId {} , prefix {}",
 852                 parentVpnId, destPrefix);
 853             return remoteGroupId;
 854         }
 855         List<BucketInfo> combinedBucketInfo = new ArrayList<>();
 856         combinedBucketInfo.addAll(localBucketInfo);
 857         combinedBucketInfo.addAll(remoteBucketInfo);
 858         GroupEntity remoteGroupEntity = MDSALUtil.buildGroupEntity(
 859                 dpnId, remoteGroupId, destPrefix, GroupTypes.GroupSelect, combinedBucketInfo);
 860         GroupEntity localGroupEntity = MDSALUtil.buildGroupEntity(
 861                 dpnId, localGroupId, destPrefix, GroupTypes.GroupSelect, localBucketInfo);
 862         String jobKey = FibUtil.getCreateLocalNextHopJobKey(parentVpnId, dpnId, destPrefix);
 863         jobCoordinator.enqueueJob(jobKey, () -> {
 864             mdsalApiManager.syncInstallGroup(remoteGroupEntity);
 865             if (!localBucketInfo.isEmpty()) {
 866                 mdsalApiManager.syncInstallGroup(localGroupEntity);
 867             }
 868             if (LOG.isDebugEnabled()) {
 869                 LOG.debug("Finished installing GroupEntity with jobCoordinator key {} remoteGroupEntity.groupId {}"
 870                         + "localGroupEntity.groupId {}  groupEntity.groupType {}", jobKey,
 871                         remoteGroupEntity.getGroupId(), localGroupEntity.getGroupId(),
 872                         remoteGroupEntity.getGroupType());
 873             }
 874             // Delete local group(if exists) if there is no local info.
 875             // Local group has to be deleted if all VMs in a compute is deleted.
 876             if (localBucketInfo.isEmpty()) {
 877                 LOG.debug("Deleting local group {} since no local nhs present for "
 878                         + "prefix {}", localGroupEntity.getGroupId(), destPrefix);
 879                 mdsalApiManager.syncRemoveGroup(localGroupEntity);
 880             }
 881             return Collections.emptyList();
 882         });
 883         return remoteGroupId;
 884     }
 885
 886     protected void deleteLoadBalancingNextHop(Long parentVpnId, BigInteger dpnId, String destPrefix) {
 887         long remoteGroupId = createNextHopPointer(getRemoteSelectGroupKey(parentVpnId, destPrefix));
 888         if (remoteGroupId == FibConstants.INVALID_GROUP_ID) {
 889             LOG.error("Unable to allocate/retrieve remote groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 890         }
 891         long localGroupId = createNextHopPointer(getLocalSelectGroupKey(parentVpnId, destPrefix));
 892         if (localGroupId == FibConstants.INVALID_GROUP_ID) {
 893             LOG.error("Unable to allocate/retrieve local groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 894         }
 895         GroupEntity remoteGroupEntity = MDSALUtil.buildGroupEntity(
 896                 dpnId, remoteGroupId, destPrefix, GroupTypes.GroupSelect, Collections.emptyList());
 897         GroupEntity localGroupEntity = MDSALUtil.buildGroupEntity(
 898                 dpnId, localGroupId, destPrefix, GroupTypes.GroupSelect, Collections.emptyList());
 899         String jobKey = FibUtil.getCreateLocalNextHopJobKey(parentVpnId, dpnId, destPrefix);
 900         jobCoordinator.enqueueJob(jobKey, () -> {
 901             mdsalApiManager.syncRemoveGroup(remoteGroupEntity);
 902             mdsalApiManager.syncRemoveGroup(localGroupEntity);
 903             if (LOG.isDebugEnabled()) {
 904                 LOG.debug("Finished removing GroupEntity with jobCoordinator key {} remoteGroupEntity.groupId {}"
 905                     + "localGroupEntity.groupId {}", jobKey, remoteGroupId, localGroupId);
 906             }
 907             return Collections.emptyList();
 908         });
 909     }
 910
 911     long createNextHopGroups(Long vpnId, String rd, BigInteger dpnId, VrfEntry vrfEntry,
 912             @Nullable Routes routes, List<Routes> vpnExtraRoutes) {
 913         List<BucketInfo> localBucketInfo = new ArrayList<>();
 914         List<Routes> clonedVpnExtraRoutes  = new ArrayList<>(vpnExtraRoutes);
 915         if (clonedVpnExtraRoutes.contains(routes)) {
 916             localBucketInfo.addAll(getBucketsForLocalNexthop(vpnId, dpnId, vrfEntry, routes));
 917             clonedVpnExtraRoutes.remove(routes);
 918         }
 919         List<BucketInfo> remoteBucketInfo =
 920             new ArrayList<>(getBucketsForRemoteNexthop(vpnId, dpnId, vrfEntry, rd, clonedVpnExtraRoutes));
 921         return setupLoadBalancingNextHop(vpnId, dpnId,
 922             vrfEntry.getDestPrefix(), localBucketInfo, remoteBucketInfo);
 923     }
 924
 925     private List<BucketInfo> getBucketsForLocalNexthop(Long vpnId, BigInteger dpnId,
 926             VrfEntry vrfEntry, Routes routes) {
 927         @Nullable List<String> nexthopIpList = routes.getNexthopIpList();
 928         if (LOG.isDebugEnabled()) {
 929             LOG.debug("NexthopManager.getBucketsForLocalNexthop invoked with vpnId {} dpnId {} "
 930                             + " vrfEntry.routePaths {}, routes.nexthopList {}", vpnId, dpnId, vrfEntry.getRoutePaths(),
 931                 nexthopIpList);
 932         }
 933         List<BucketInfo> listBucketInfo = new CopyOnWriteArrayList<>();
 934         if (nexthopIpList != null) {
 935             nexthopIpList.parallelStream().forEach(nextHopIp -> {
 936                 String localNextHopIP;
 937                 if (isIpv4Address(nextHopIp)) {
 938                     localNextHopIP = nextHopIp + NwConstants.IPV4PREFIX;
 939                 } else {
 940                     localNextHopIP = nextHopIp + NwConstants.IPV6PREFIX;
 941                 }
 942                 Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 943                 if (localNextHopInfo != null) {
 944                     long groupId = getLocalNextHopGroup(vpnId, localNextHopIP);
 945                     if (groupId == FibConstants.INVALID_GROUP_ID) {
 946                         LOG.error("Unable to allocate groupId for vpnId {} , prefix {} , interface {}", vpnId,
 947                             vrfEntry.getDestPrefix(), localNextHopInfo.getVpnInterfaceName());
 948                         return;
 949                     }
 950                     List<ActionInfo> actionsInfos =
 951                         Collections.singletonList(new ActionGroup(groupId));
 952                     BucketInfo bucket = new BucketInfo(actionsInfos);
 953                     bucket.setWeight(1);
 954                     listBucketInfo.add(bucket);
 955                 }
 956             });
 957         }
 958         LOG.trace("LOCAL: listbucket {}, vpnId {}, dpnId {}, routes {}", listBucketInfo, vpnId, dpnId, routes);
 959         return listBucketInfo;
 960     }
 961
 962     private List<BucketInfo> getBucketsForRemoteNexthop(Long vpnId, BigInteger dpnId, VrfEntry vrfEntry, String rd,
 963             List<Routes> vpnExtraRoutes) {
 964         List<BucketInfo> listBucketInfo = new ArrayList<>();
 965         Map<String, List<ActionInfo>> egressActionMap = new HashMap<>();
 966         vpnExtraRoutes.stream().filter(vpnExtraRoute -> vpnExtraRoute.getNexthopIpList() != null).forEach(
 967             vpnExtraRoute -> vpnExtraRoute.getNexthopIpList().forEach(nextHopIp -> {
 968                 String nextHopPrefixIp;
 969                 if (isIpv4Address(nextHopIp)) {
 970                     nextHopPrefixIp = nextHopIp + NwConstants.IPV4PREFIX;
 971                 } else {
 972                     nextHopPrefixIp = nextHopIp + NwConstants.IPV6PREFIX;
 973                 }
 974                 List<String> tepIpAddresses = fibUtil.getNextHopAddresses(rd, nextHopPrefixIp);
 975                 if (tepIpAddresses.isEmpty()) {
 976                     return;
 977                 }
 978                 // There would be only one nexthop address for a VM ip which would be the tep Ip
 979                 String tepIp = tepIpAddresses.get(0);
 980                 AdjacencyResult adjacencyResult = getRemoteNextHopPointer(dpnId, vpnId,
 981                     vrfEntry.getDestPrefix(), tepIp, TunnelTypeVxlan.class);
 982                 if (adjacencyResult == null) {
 983                     return;
 984                 }
 985                 String egressInterface = adjacencyResult.getInterfaceName();
 986                 if (!FibUtil.isTunnelInterface(adjacencyResult)) {
 987                     return;
 988                 }
 989                 Class<? extends TunnelTypeBase> tunnelType =
 990                     VpnExtraRouteHelper.getTunnelType(itmManager, egressInterface);
 991                 StateTunnelList ifState = null;
 992                 try {
 993                     ifState = fibUtil.getTunnelState(egressInterface);
 994                     if (ifState == null || ifState.getOperState() != TunnelOperStatus.Up) {
 995                         LOG.trace("Tunnel is not up for interface {}", egressInterface);
 996                         return;
 997                     }
 998                 } catch (ReadFailedException e) {
 999                     LOG.error("getBucketsForRemoteNexthop: error in fetching tunnel state for interface {}",
1000                         egressInterface, e);
1001                     return;
1002                 }
1003                 if (!TunnelTypeVxlan.class.equals(tunnelType)) {
1004                     return;
1005                 }
1006                 Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).get();
1007                 Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, nextHopPrefixIp);
1008                 if (prefixInfo == null) {
1009                     LOG.error("No prefix info found for prefix {} in rd {} for VPN {}", nextHopPrefixIp, rd,
1010                         vpnId);
1011                     return;
1012                 }
1013                 BigInteger tunnelId;
1014                 if (FibUtil.isVxlanNetwork(prefixInfo.getNetworkType())) {
1015                     tunnelId = BigInteger.valueOf(prefixInfo.getSegmentationId());
1016                 } else {
1017                     LOG.warn("Network is not of type VXLAN for prefix {}."
1018                         + "Going with default Lport Tag.", prefixInfo.toString());
1019                     tunnelId = BigInteger.valueOf(label);
1020                 }
1021                 List<ActionInfo> actionInfos = new ArrayList<>();
1022                 actionInfos.add(new ActionSetFieldTunnelId(tunnelId));
1023                 String ifName = prefixInfo.getVpnInterfaceName();
1024                 String vpnName = fibUtil.getVpnNameFromId(vpnId);
1025                 if (vpnName == null) {
1026                     return;
1027                 }
1028                 String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, nextHopPrefixIp);
1029                 actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
1030                     new MacAddress(macAddress)));
1031                 List<ActionInfo> egressActions;
1032                 if (egressActionMap.containsKey(egressInterface)) {
1033                     egressActions = egressActionMap.get(egressInterface);
1034                 } else {
1035                     egressActions = getEgressActionsForInterface(egressInterface, actionInfos.size(), true);
1036                     egressActionMap.put(egressInterface, egressActions);
1037                 }
1038                 if (egressActions.isEmpty()) {
1039                     LOG.error("Failed to retrieve egress action for prefix {} route-paths {}"
1040                             + " interface {}." + " Aborting remote FIB entry creation.",
1041                         vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), egressInterface);
1042                 }
1043                 actionInfos.addAll(egressActions);
1044                 BucketInfo bucket = new BucketInfo(actionInfos);
1045                 bucket.setWeight(1);
1046                 listBucketInfo.add(bucket);
1047             }));
1048         LOG.trace("LOCAL: listbucket {}, rd {}, dpnId {}, routes {}", listBucketInfo, rd, dpnId, vpnExtraRoutes);
1049         return listBucketInfo;
1050     }
1051
1052     public void createDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId, String destinationIp,
1053                                              Class<? extends TunnelTypeBase> tunnelType) {
1054         Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
1055         int noOfDcGws = availableDcGws.size();
1056         if (noOfDcGws == 1) {
1057             LOG.trace("There are no enough DC GateWays {} present to program LB group", availableDcGws);
1058             return;
1059         }
1060         // TODO : Place the logic to construct all possible DC-GW combination here.
1061         String groupIdKey = FibUtil.getGreLbGroupKey(availableDcGws);
1062         Long groupId = createNextHopPointer(groupIdKey);
1063         List<Bucket> listBucket = new ArrayList<>();
1064         for (int index = 0; index < noOfDcGws; index++) {
1065             if (isTunnelUp(availableDcGws.get(index), dpnId, tunnelType)) {
1066                 listBucket.add(buildBucketForDcGwLbGroup(availableDcGws.get(index), dpnId, index, tunnelType));
1067             }
1068         }
1069         Group group = MDSALUtil.buildGroup(groupId, groupIdKey, GroupTypes.GroupSelect,
1070                         MDSALUtil.buildBucketLists(listBucket));
1071         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1072             confTx -> mdsalApiManager.addGroup(confTx, dpnId, group)), LOG, "Error adding load-balancing group");
1073         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1074             operTx -> FibUtil.updateLbGroupInfo(dpnId, destinationIp, groupIdKey, groupId.toString(), operTx)), LOG,
1075             "Error updating load-balancing group info");
1076         LOG.trace("LB group {} towards DC-GW installed on dpn {}. Group - {}", groupIdKey, dpnId, group);
1077     }
1078
1079     private boolean isTunnelUp(String dcGwIp, BigInteger dpnId, Class<? extends TunnelTypeBase> tunnelType) {
1080         String tunnelName = getTunnelRemoteNextHopPointer(dpnId, dcGwIp, tunnelType);
1081         if (tunnelName != null) {
1082             InstanceIdentifier<StateTunnelList> tunnelStateId =
1083                     InstanceIdentifier.builder(TunnelsState.class).child(
1084                             StateTunnelList.class, new StateTunnelListKey(tunnelName)).build();
1085             return MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, tunnelStateId)
1086                     .toJavaUtil().map(StateTunnelList::getOperState)
1087                     .orElse(TunnelOperStatus.Down) == TunnelOperStatus.Up;
1088         }
1089         return false;
1090     }
1091
1092     private List<Action> getEgressActions(String interfaceName, int actionKey) {
1093         List<Action> actions = Collections.emptyList();
1094         try {
1095             GetEgressActionsForInterfaceInputBuilder egressAction =
1096                     new GetEgressActionsForInterfaceInputBuilder().setIntfName(interfaceName).setActionKey(actionKey);
1097             Future<RpcResult<GetEgressActionsForInterfaceOutput>> result =
1098                     odlInterfaceRpcService.getEgressActionsForInterface(egressAction.build());
1099             RpcResult<GetEgressActionsForInterfaceOutput> rpcResult = result.get();
1100             if (!rpcResult.isSuccessful()) {
1101                 LOG.warn("RPC Call to Get egress actions for interface {} returned with Errors {}",
1102                         interfaceName, rpcResult.getErrors());
1103             } else {
1104                 actions = rpcResult.getResult().nonnullAction();
1105             }
1106         } catch (InterruptedException | ExecutionException e) {
1107             LOG.warn("Exception when egress actions for interface {}", interfaceName, e);
1108         }
1109         return actions;
1110     }
1111
1112     /**
1113      * This method is invoked when the tunnel state is removed from DS.
1114      * If the there is just one DC-GW left in configuration then the LB groups can be deleted.
1115      * Otherwise, the groups are just updated.
1116      */
1117     public void removeOrUpdateDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId,
1118             String destinationIp) {
1119         Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
1120         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
1121             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
1122                 int noOfDcGws = availableDcGws.size();
1123                 // If availableDcGws does not contain the destination Ip it means this is a configuration delete.
1124                 if (!availableDcGws.contains(destinationIp)) {
1125                     availableDcGws.add(destinationIp);
1126                     Collections.sort(availableDcGws);
1127                 }
1128                 // TODO : Place the logic to construct all possible DC-GW combination here.
1129                 int bucketId = availableDcGws.indexOf(destinationIp);
1130                 Optional<DpnLbNexthops> dpnLbNextHops = fibUtil.getDpnLbNexthops(dpnId, destinationIp);
1131                 if (!dpnLbNextHops.isPresent()) {
1132                     return;
1133                 }
1134                 List<String> nextHopKeys = dpnLbNextHops.get().getNexthopKey();
1135                 if (nextHopKeys != null) {
1136                     for (String nextHopKey : nextHopKeys) {
1137                         Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
1138                         if (!optionalNextHops.isPresent()) {
1139                             return;
1140                         }
1141                         Nexthops nexthops = optionalNextHops.get();
1142                         final String groupId = nexthops.getGroupId();
1143                         final long groupIdValue = Long.parseLong(groupId);
1144                         if (noOfDcGws > 1) {
1145                             mdsalApiManager.removeBucket(confTx, dpnId, groupIdValue, bucketId);
1146                         } else {
1147                             LOG.trace("Removed LB group {} on dpn {}", groupIdValue, dpnId);
1148                             mdsalApiManager.removeGroup(confTx, dpnId, groupIdValue);
1149                             removeNextHopPointer(nextHopKey);
1150                         }
1151                         // When the DC-GW is removed from configuration.
1152                         if (noOfDcGws != availableDcGws.size()) {
1153                             FibUtil.removeOrUpdateNextHopInfo(dpnId, nextHopKey, groupId, nexthops, operTx);
1154                         }
1155                     }
1156                 }
1157                 FibUtil.removeDpnIdToNextHopInfo(destinationIp, dpnId, operTx);
1158             }), LOG, "Error removing or updating load-balancing group");
1159         }), LOG, "Error removing or updating load-balancing group");
1160     }
1161
1162     /**
1163      * This method is invoked when the tunnel status is updated.
1164      * The bucket is directly removed/added based on the operational status of the tunnel.
1165      */
1166     public void updateDcGwLoadBalancingGroup(List<String> availableDcGws,
1167             BigInteger dpnId, String destinationIp, boolean isTunnelUp, Class<? extends TunnelTypeBase> tunnelType) {
1168         Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
1169         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
1170             // TODO : Place the logic to construct all possible DC-GW combination here.
1171             int bucketId = availableDcGws.indexOf(destinationIp);
1172             Optional<DpnLbNexthops> dpnLbNextHops = fibUtil.getDpnLbNexthops(dpnId, destinationIp);
1173             if (!dpnLbNextHops.isPresent()) {
1174                 return;
1175             }
1176             List<String> nextHopKeys = dpnLbNextHops.get().getNexthopKey();
1177             if (nextHopKeys != null) {
1178                 for (String nextHopKey : nextHopKeys) {
1179                     Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
1180                     if (!optionalNextHops.isPresent()) {
1181                         return;
1182                     }
1183                     Nexthops nexthops = optionalNextHops.get();
1184                     final String groupId = nexthops.getGroupId();
1185                     final long groupIdValue = Long.parseLong(groupId);
1186                     if (isTunnelUp) {
1187                         Bucket bucket = buildBucketForDcGwLbGroup(destinationIp, dpnId, bucketId, tunnelType);
1188                         LOG.trace("Added bucket {} to group {} on dpn {}.", bucket, groupId, dpnId);
1189                         mdsalApiManager.addBucket(confTx, dpnId, groupIdValue, bucket);
1190                     } else {
1191                         LOG.trace("Removed bucketId {} from group {} on dpn {}.", bucketId, groupId, dpnId);
1192                         mdsalApiManager.removeBucket(confTx, dpnId, groupIdValue, bucketId);
1193                     }
1194                 }
1195             }
1196         }), LOG, "Error updating load-balancing group");
1197     }
1198
1199     private Bucket buildBucketForDcGwLbGroup(String ipAddress, BigInteger dpnId, int index,
1200                                              Class<? extends TunnelTypeBase> tunnelType) {
1201         List<Action> listAction = new ArrayList<>();
1202         // ActionKey 0 goes to mpls label.
1203         int actionKey = 1;
1204         listAction.add(new ActionPushMpls().buildAction());
1205         listAction.add(new ActionRegMove(actionKey++, FibConstants.NXM_REG_MAPPING
1206                 .get(index), 0, 19).buildAction());
1207         String tunnelInterfaceName = getTunnelInterfaceName(dpnId, IpAddressBuilder.getDefaultInstance(ipAddress),
1208             tunnelType);
1209         List<Action> egressActions = getEgressActions(tunnelInterfaceName, actionKey++);
1210         if (!egressActions.isEmpty()) {
1211             listAction.addAll(getEgressActions(tunnelInterfaceName, actionKey++));
1212         } else {
1213             // clear off actions if there is no egress actions.
1214             listAction = Collections.emptyList();
1215         }
1216         //OVS expects a non-zero weight value for load balancing to happen in select groups
1217         return MDSALUtil.buildBucket(listAction, SELECT_GROUP_WEIGHT, index,
1218                 MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP);
1219     }
1220
1221     public void programDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId, String destinationIp,
1222                                               int addRemoveOrUpdate, boolean isTunnelUp,
1223                                               Class<? extends TunnelTypeBase> tunnelType) {
1224         if (NwConstants.ADD_FLOW == addRemoveOrUpdate) {
1225             createDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp, tunnelType);
1226         } else if (NwConstants.DEL_FLOW == addRemoveOrUpdate) {
1227             removeOrUpdateDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp);
1228         } else if (NwConstants.MOD_FLOW == addRemoveOrUpdate) {
1229             updateDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp, isTunnelUp, tunnelType);
1230         }
1231     }
1232 }