fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/NexthopManager.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  11 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  12 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  13
  14 import com.google.common.base.Optional;
  15 import com.google.common.base.Preconditions;
  16 import java.math.BigInteger;
  17 import java.util.ArrayList;
  18 import java.util.Collections;
  19 import java.util.HashMap;
  20 import java.util.List;
  21 import java.util.Locale;
  22 import java.util.Map;
  23 import java.util.Objects;
  24 import java.util.concurrent.CopyOnWriteArrayList;
  25 import java.util.concurrent.ExecutionException;
  26 import java.util.concurrent.Future;
  27 import javax.annotation.PreDestroy;
  28 import javax.inject.Inject;
  29 import javax.inject.Singleton;
  30 import org.eclipse.jdt.annotation.Nullable;
  31 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  32 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  33 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  34 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  35 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  36 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  37 import org.opendaylight.genius.itm.globals.ITMConstants;
  38 import org.opendaylight.genius.mdsalutil.ActionInfo;
  39 import org.opendaylight.genius.mdsalutil.BucketInfo;
  40 import org.opendaylight.genius.mdsalutil.GroupEntity;
  41 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  42 import org.opendaylight.genius.mdsalutil.NwConstants;
  43 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  44 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  45 import org.opendaylight.genius.mdsalutil.actions.ActionOutput;
  46 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
  47 import org.opendaylight.genius.mdsalutil.actions.ActionPushVlan;
  48 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  49 import org.opendaylight.genius.mdsalutil.actions.ActionRegMove;
  50 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetSource;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldVlanVid;
  54 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  55 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  56 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  57 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  58 import org.opendaylight.netvirt.fibmanager.api.L3VPNTransportTypes;
  59 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  60 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev170119.L2vlan;
  61 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev170119.Tunnel;
  62 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
  63 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfaceType;
  64 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.OutputActionCase;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.PushVlanActionCase;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.SetFieldCase;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInput;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInputBuilder;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutput;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInput;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInputBuilder;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolOutput;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInput;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInputBuilder;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdOutput;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceInputBuilder;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceOutput;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelOperStatus;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelsState;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelListKey;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelInputBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelOutput;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameInputBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameOutput;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.lockmanager.rev160413.LockManagerService;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupInput;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupInputBuilder;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupOutput;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.SalGroupService;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupId;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupRef;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.Buckets;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeRef;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeBase;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeFlat;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeVlan;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.L3nexthop;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthops;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthopsKey;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopKey;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacencies;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacenciesBuilder;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacenciesKey;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpn;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpnBuilder;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.dpid.l3vpn.lb.nexthops.DpnLbNexthops;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.lb.nexthops.Nexthops;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.add.group.input.buckets.bucket.action.action.NxActionResubmitRpcAddGroupCase;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.nodes.node.table.flow.instructions.instruction.instruction.apply.actions._case.apply.actions.action.action.NxActionRegLoadNodesNodeTableFlowApplyActionsCase;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.nx.action.reg.load.grouping.NxRegLoad;
 133 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 134 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 135 import org.opendaylight.yangtools.yang.common.RpcResult;
 136 import org.slf4j.Logger;
 137 import org.slf4j.LoggerFactory;
 138
 139 @Singleton
 140 public class NexthopManager implements AutoCloseable {
 141     private static final Logger LOG = LoggerFactory.getLogger(NexthopManager.class);
 142     private static final String NEXTHOP_ID_POOL_NAME = "nextHopPointerPool";
 143     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 144     private static final long WAIT_TIME_TO_ACQUIRE_LOCK = 3000L;
 145     private static final int SELECT_GROUP_WEIGHT = 1;
 146
 147     private final DataBroker dataBroker;
 148     private final ManagedNewTransactionRunner txRunner;
 149     private final IMdsalApiManager mdsalApiManager;
 150     private final OdlInterfaceRpcService odlInterfaceRpcService;
 151     private final ItmRpcService itmManager;
 152     private final IdManagerService idManager;
 153     private final IElanService elanService;
 154     private final LockManagerService lockManager;
 155     private final SalGroupService salGroupService;
 156     private final JobCoordinator jobCoordinator;
 157     private final FibUtil fibUtil;
 158     private final IInterfaceManager interfaceManager;
 159     private volatile L3VPNTransportTypes configuredTransportTypeL3VPN = L3VPNTransportTypes.Invalid;
 160
 161     /**
 162      * Provides nexthop functions.
 163      * Creates group ID pool
 164      *
 165      * @param dataBroker       - dataBroker reference
 166      * @param mdsalApiManager  - mdsalApiManager reference
 167      * @param idManager        - idManager reference
 168      * @param odlInterfaceRpcService - odlInterfaceRpcService reference
 169      * @param itmManager       - itmManager reference
 170      */
 171     @Inject
 172     public NexthopManager(final DataBroker dataBroker,
 173                           final IMdsalApiManager mdsalApiManager,
 174                           final IdManagerService idManager,
 175                           final OdlInterfaceRpcService odlInterfaceRpcService,
 176                           final ItmRpcService itmManager,
 177                           final LockManagerService lockManager,
 178                           final IElanService elanService,
 179                           final SalGroupService salGroupService,
 180                           final JobCoordinator jobCoordinator,
 181                           final FibUtil fibUtil,
 182                           final IInterfaceManager interfaceManager) {
 183         this.dataBroker = dataBroker;
 184         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 185         this.mdsalApiManager = mdsalApiManager;
 186         this.idManager = idManager;
 187         this.odlInterfaceRpcService = odlInterfaceRpcService;
 188         this.itmManager = itmManager;
 189         this.elanService = elanService;
 190         this.salGroupService = salGroupService;
 191         this.jobCoordinator = jobCoordinator;
 192         this.fibUtil = fibUtil;
 193         this.lockManager = lockManager;
 194         this.interfaceManager = interfaceManager;
 195         createIdPool();
 196     }
 197
 198     private void createIdPool() {
 199         CreateIdPoolInput createPool = new CreateIdPoolInputBuilder()
 200             .setPoolName(NEXTHOP_ID_POOL_NAME)
 201             .setLow(150000L)
 202             .setHigh(175000L)
 203             .build();
 204         try {
 205             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPool);
 206             if (result != null && result.get().isSuccessful()) {
 207                 LOG.info("Created IdPool for NextHopPointerPool");
 208             }
 209         } catch (InterruptedException | ExecutionException e) {
 210             LOG.error("Failed to create idPool for NextHopPointerPool", e);
 211         }
 212     }
 213
 214     private String getNextHopKey(long vpnId, String ipAddress) {
 215         return "nexthop." + vpnId + ipAddress;
 216     }
 217
 218     String getRemoteSelectGroupKey(long vpnId, String ipAddress) {
 219         return "remote.ecmp.nexthop." + vpnId + ipAddress;
 220     }
 221
 222     String getLocalSelectGroupKey(long vpnId, String ipAddress) {
 223         return "local.ecmp.nexthop." + vpnId + ipAddress;
 224     }
 225
 226     public ItmRpcService getItmManager() {
 227         return itmManager;
 228     }
 229
 230     protected long createNextHopPointer(String nexthopKey) {
 231         AllocateIdInput getIdInput = new AllocateIdInputBuilder()
 232             .setPoolName(NEXTHOP_ID_POOL_NAME).setIdKey(nexthopKey)
 233             .build();
 234         //TODO: Proper error handling once IdManager code is complete
 235         try {
 236             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
 237             RpcResult<AllocateIdOutput> rpcResult = result.get();
 238             return rpcResult.getResult().getIdValue();
 239         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 240             // FIXME: NPEs should not be caught but rather their root cause should be eliminated
 241             LOG.trace("Failed to allocate {}", getIdInput, e);
 242         }
 243         return 0;
 244     }
 245
 246     protected void removeNextHopPointer(String nexthopKey) {
 247         ReleaseIdInput idInput = new ReleaseIdInputBuilder()
 248             .setPoolName(NEXTHOP_ID_POOL_NAME)
 249             .setIdKey(nexthopKey).build();
 250         try {
 251             RpcResult<ReleaseIdOutput> rpcResult = idManager.releaseId(idInput).get();
 252             if (!rpcResult.isSuccessful()) {
 253                 LOG.error("RPC Call to Get Unique Id for nexthopKey {} returned with Errors {}",
 254                         nexthopKey, rpcResult.getErrors());
 255             }
 256         } catch (InterruptedException | ExecutionException e) {
 257             LOG.warn("Exception when getting Unique Id for key {}", nexthopKey, e);
 258         }
 259     }
 260
 261     protected List<ActionInfo> getEgressActionsForInterface(final String ifName, int actionKey,
 262                                                             boolean isTunnelInterface,
 263                                                             long vpnId, String destIpPrefix) {
 264         List<Action> actions;
 265         try {
 266             if (isTunnelInterface && interfaceManager.isItmDirectTunnelsEnabled()) {
 267                 RpcResult<GetEgressActionsForTunnelOutput> rpcResult =
 268                         itmManager.getEgressActionsForTunnel(new GetEgressActionsForTunnelInputBuilder()
 269                                 .setIntfName(ifName).build()).get();
 270                 if (!rpcResult.isSuccessful()) {
 271                     LOG.error("RPC Call to Get egress tunnel actions for interface {} returned with Errors {}",
 272                             ifName, rpcResult.getErrors());
 273                     return Collections.emptyList();
 274                 } else {
 275                     actions = rpcResult.getResult().nonnullAction();
 276                 }
 277             } else {
 278                 RpcResult<GetEgressActionsForInterfaceOutput> rpcResult = odlInterfaceRpcService
 279                         .getEgressActionsForInterface(new GetEgressActionsForInterfaceInputBuilder()
 280                                 .setIntfName(ifName).build()).get();
 281                 if (!rpcResult.isSuccessful()) {
 282                     LOG.error("RPC Call to Get egress vm actions for interface {} vpnId {} ipPrefix {} returned with "
 283                                     + "Errors {}", ifName, vpnId, destIpPrefix, rpcResult.getErrors());
 284                     return Collections.emptyList();
 285                 } else {
 286                     actions = rpcResult.getResult().nonnullAction();
 287                 }
 288             }
 289             List<ActionInfo> listActionInfo = new ArrayList<>();
 290             for (Action action : actions) {
 291                 actionKey = action.key().getOrder() + actionKey;
 292                 org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action
 293                     actionClass = action.getAction();
 294                 if (actionClass instanceof OutputActionCase) {
 295                     listActionInfo.add(new ActionOutput(actionKey,
 296                         ((OutputActionCase) actionClass).getOutputAction().getOutputNodeConnector()));
 297                 } else if (actionClass instanceof PushVlanActionCase) {
 298                     listActionInfo.add(new ActionPushVlan(actionKey));
 299                 } else if (actionClass instanceof SetFieldCase) {
 300                     if (((SetFieldCase) actionClass).getSetField().getVlanMatch() != null) {
 301                         int vlanVid = ((SetFieldCase) actionClass).getSetField().getVlanMatch()
 302                             .getVlanId().getVlanId().getValue();
 303                         listActionInfo.add(new ActionSetFieldVlanVid(actionKey, vlanVid));
 304                     }
 305                 } else if (actionClass instanceof NxActionResubmitRpcAddGroupCase) {
 306                     Short tableId = ((NxActionResubmitRpcAddGroupCase) actionClass).getNxResubmit().getTable();
 307                     listActionInfo.add(new ActionNxResubmit(actionKey, tableId));
 308                 } else if (actionClass instanceof NxActionRegLoadNodesNodeTableFlowApplyActionsCase) {
 309                     NxRegLoad nxRegLoad =
 310                         ((NxActionRegLoadNodesNodeTableFlowApplyActionsCase) actionClass).getNxRegLoad();
 311                     listActionInfo.add(new ActionRegLoad(actionKey, NxmNxReg6.class,
 312                         nxRegLoad.getDst().getStart(), nxRegLoad.getDst().getEnd(),
 313                         nxRegLoad.getValue().longValue()));
 314                 }
 315             }
 316             return listActionInfo;
 317         } catch (InterruptedException | ExecutionException | NullPointerException e) {
 318             LOG.error("Exception when egress actions for interface {} isTunnel {} vpnId {} ipPrefix {}", ifName,
 319                     isTunnelInterface, vpnId, destIpPrefix, e);
 320         }
 321         LOG.warn("Exception when egress actions for interface {}", ifName);
 322         return Collections.emptyList();
 323     }
 324
 325     @Nullable
 326     protected String getTunnelInterfaceName(BigInteger srcDpId, BigInteger dstDpId) {
 327         Class<? extends TunnelTypeBase> tunType = getReqTunType(getReqTransType().toUpperCase(Locale.getDefault()));
 328         Future<RpcResult<GetTunnelInterfaceNameOutput>> result;
 329         try {
 330             result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 331                 .setSourceDpid(srcDpId)
 332                 .setDestinationDpid(dstDpId)
 333                 .setTunnelType(tunType)
 334                 .build());
 335             RpcResult<GetTunnelInterfaceNameOutput> rpcResult = result.get();
 336             if (!rpcResult.isSuccessful()) {
 337                 LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
 338             } else {
 339                 return rpcResult.getResult().getInterfaceName();
 340             }
 341         } catch (InterruptedException | ExecutionException e) {
 342             LOG.warn("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstDpId, e);
 343         }
 344         return null;
 345     }
 346
 347     @Nullable
 348     protected String getTunnelInterfaceName(BigInteger srcDpId, org.opendaylight.yang.gen.v1.urn.ietf.params
 349         .xml.ns.yang.ietf.inet.types.rev130715.IpAddress dstIp, Class<? extends TunnelTypeBase> tunnelType) {
 350         Future<RpcResult<GetInternalOrExternalInterfaceNameOutput>> result;
 351         try {
 352             LOG.debug("Trying to fetch tunnel interface name for source dpn {} destIp {} tunType {}", srcDpId,
 353                     dstIp.stringValue(), tunnelType.getName());
 354             result = itmManager.getInternalOrExternalInterfaceName(new GetInternalOrExternalInterfaceNameInputBuilder()
 355                 .setSourceDpid(srcDpId)
 356                 .setDestinationIp(dstIp)
 357                 .setTunnelType(tunnelType)
 358                 .build());
 359             RpcResult<GetInternalOrExternalInterfaceNameOutput> rpcResult = result.get();
 360             if (!rpcResult.isSuccessful()) {
 361                 LOG.warn("RPC Call to getTunnelInterfaceName returned with Errors {}", rpcResult.getErrors());
 362             } else {
 363                 return rpcResult.getResult().getInterfaceName();
 364             }
 365         } catch (InterruptedException | ExecutionException e) {
 366             LOG.error("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstIp, e);
 367         }
 368         return null;
 369     }
 370
 371
 372     public long getLocalNextHopGroup(long vpnId,
 373             String ipNextHopAddress) {
 374         long groupId = createNextHopPointer(getNextHopKey(vpnId, ipNextHopAddress));
 375         if (groupId == FibConstants.INVALID_GROUP_ID) {
 376             LOG.error("Unable to allocate groupId for vpnId {} , prefix {}", vpnId, ipNextHopAddress);
 377         }
 378         return groupId;
 379     }
 380
 381     public long getLocalSelectGroup(long vpnId,
 382             String ipNextHopAddress) {
 383         long groupId = createNextHopPointer(getLocalSelectGroupKey(vpnId, ipNextHopAddress));
 384         if (groupId == FibConstants.INVALID_GROUP_ID) {
 385             LOG.error("Unable to allocate groupId for vpnId {} , prefix {}", vpnId, ipNextHopAddress);
 386         }
 387         return groupId;
 388     }
 389
 390     public long createLocalNextHop(long vpnId, BigInteger dpnId, String ifName,
 391                                    String primaryIpAddress, String currDestIpPrefix,
 392                                    String gwMacAddress) {
 393         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 394         if (vpnName == null) {
 395             return 0;
 396         }
 397         String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, primaryIpAddress);
 398
 399         long groupId = createNextHopPointer(getNextHopKey(vpnId, primaryIpAddress));
 400         if (groupId == 0) {
 401             LOG.error("Unable to allocate groupId for vpnId {} , IntfName {}, primaryIpAddress {} curIpPrefix {}",
 402                     vpnId, ifName, primaryIpAddress, currDestIpPrefix);
 403             return groupId;
 404         }
 405         String nextHopLockStr = vpnId + primaryIpAddress;
 406         String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, currDestIpPrefix);
 407         jobCoordinator.enqueueJob(jobKey, () -> {
 408             try {
 409                 if (FibUtil.lockCluster(lockManager, nextHopLockStr, WAIT_TIME_TO_ACQUIRE_LOCK)) {
 410                     VpnNexthop nexthop = getVpnNexthop(vpnId, primaryIpAddress);
 411                     LOG.trace("nexthop: {} retrieved for vpnId {}, prefix {}, ifName {} on dpn {}", nexthop, vpnId,
 412                             primaryIpAddress, ifName, dpnId);
 413                     if (nexthop == null) {
 414                         String encMacAddress = macAddress == null
 415                                 ? fibUtil.getMacAddressFromPrefix(ifName, vpnName, primaryIpAddress) : macAddress;
 416                         List<ActionInfo> listActionInfo = new ArrayList<>();
 417                         int actionKey = 0;
 418                         // MAC re-write
 419                         if (encMacAddress != null) {
 420                             if (gwMacAddress != null) {
 421                                 LOG.trace("The Local NextHop Group Source Mac {} for VpnInterface {} on VPN {}",
 422                                         gwMacAddress, ifName, vpnId);
 423                                 listActionInfo.add(new ActionSetFieldEthernetSource(actionKey++,
 424                                         new MacAddress(gwMacAddress)));
 425                             }
 426                             listActionInfo.add(new ActionSetFieldEthernetDestination(actionKey++,
 427                                     new MacAddress(encMacAddress)));
 428                             // listActionInfo.add(0, new ActionPopMpls());
 429                         } else {
 430                             // FIXME: Log message here.
 431                             LOG.debug("mac address for new local nexthop is null");
 432                         }
 433                         List<ActionInfo> nhActionInfoList = getEgressActionsForInterface(ifName, actionKey, false,
 434                                 vpnId, currDestIpPrefix);
 435                         if (nhActionInfoList.isEmpty()) {
 436                             LOG.error("createLocalNextHop: Skipping, Empty list of egress actions received for "
 437                                     + "interface {} on dpn {} for vpn {} prefix {}", ifName, dpnId, vpnId,
 438                                     currDestIpPrefix);
 439                         }
 440                         listActionInfo.addAll(nhActionInfoList);
 441                         BucketInfo bucket = new BucketInfo(listActionInfo);
 442                         List<BucketInfo> listBucketInfo = new ArrayList<>();
 443                         listBucketInfo.add(bucket);
 444                         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, groupId, primaryIpAddress,
 445                                 GroupTypes.GroupAll, listBucketInfo);
 446                         LOG.trace("Install LNH Group: id {}, mac address {}, interface {} for prefix {}", groupId,
 447                                 encMacAddress, ifName, primaryIpAddress);
 448                         //Try to install group directly on the DPN bypassing the FRM, in order to avoid waiting for the
 449                         // group to get installed before programming the flows
 450                         installGroupOnDpn(groupId, dpnId, primaryIpAddress, listBucketInfo,
 451                                 getNextHopKey(vpnId, primaryIpAddress), GroupTypes.GroupAll);
 452                         // install Group
 453                         mdsalApiManager.syncInstallGroup(groupEntity);
 454                         // update MD-SAL DS
 455                         addVpnNexthopToDS(dpnId, vpnId, primaryIpAddress, currDestIpPrefix, groupId);
 456
 457                     } else {
 458                         // Ignore adding new prefix , if it already exists
 459                         List<IpAdjacencies> prefixesList = nexthop.getIpAdjacencies();
 460                         IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currDestIpPrefix).build();
 461                         if (prefixesList != null && prefixesList.contains(prefix)) {
 462                             LOG.trace("Prefix {} is already present in l3nextHop {} ", currDestIpPrefix, nexthop);
 463                         } else {
 464                             IpAdjacenciesBuilder ipPrefixesBuilder =
 465                                     new IpAdjacenciesBuilder().withKey(new IpAdjacenciesKey(currDestIpPrefix));
 466                             LOG.trace("Updating prefix {} to vpnNextHop {} Operational DS", currDestIpPrefix, nexthop);
 467                             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
 468                                     getVpnNextHopIpPrefixIdentifier(vpnId, primaryIpAddress, currDestIpPrefix),
 469                                     ipPrefixesBuilder.build());
 470                         }
 471                     }
 472                 }
 473             } finally {
 474                 FibUtil.unlockCluster(lockManager, nextHopLockStr);
 475             }
 476             return Collections.emptyList();
 477         });
 478         return groupId;
 479     }
 480
 481     private void installGroupOnDpn(long groupId, BigInteger dpnId, String groupName, List<BucketInfo> bucketsInfo,
 482                                      String nextHopKey, GroupTypes groupType) {
 483         NodeRef nodeRef = FibUtil.buildNodeRef(dpnId);
 484         Buckets buckets = FibUtil.buildBuckets(bucketsInfo);
 485         GroupRef groupRef = new GroupRef(FibUtil.buildGroupInstanceIdentifier(groupId, dpnId));
 486         AddGroupInput input = new AddGroupInputBuilder().setNode(nodeRef).setGroupId(new GroupId(groupId))
 487                 .setBuckets(buckets).setGroupRef(groupRef).setGroupType(groupType)
 488                 .setGroupName(groupName).build();
 489         Future<RpcResult<AddGroupOutput>> groupStats = salGroupService.addGroup(input);
 490         RpcResult<AddGroupOutput> rpcResult = null;
 491         try {
 492             rpcResult = groupStats.get();
 493             if (rpcResult != null && rpcResult.isSuccessful()) {
 494                 LOG.info("Group {} with key {} has been successfully installed directly on dpn {}.", groupId,
 495                         nextHopKey, dpnId);
 496             } else {
 497                 LOG.error("Unable to install group {} with key {} directly on dpn {} due to {}.", groupId, nextHopKey,
 498                         dpnId, rpcResult != null ? rpcResult.getErrors() : null);
 499             }
 500         } catch (InterruptedException | ExecutionException e) {
 501             LOG.error("Error while installing group {} directly on dpn {}", groupId, dpnId);
 502         }
 503     }
 504
 505     protected void addVpnNexthopToDS(BigInteger dpnId, long vpnId, String primaryIpAddr,
 506                                      String currIpAddr, long egressPointer) {
 507         InstanceIdentifierBuilder<VpnNexthops> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
 508             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId));
 509
 510         List<IpAdjacencies> ipPrefixesList = new ArrayList<>();
 511         IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currIpAddr).build();
 512         ipPrefixesList.add(prefix);
 513         // Add nexthop to vpn node
 514         VpnNexthop nh = new VpnNexthopBuilder()
 515             .withKey(new VpnNexthopKey(primaryIpAddr))
 516             .setDpnId(dpnId)
 517             .setIpAdjacencies(ipPrefixesList)
 518             .setEgressPointer(egressPointer).build();
 519
 520         InstanceIdentifier<VpnNexthop> id1 = idBuilder
 521             .child(VpnNexthop.class, new VpnNexthopKey(primaryIpAddr)).build();
 522         LOG.trace("Adding vpnnextHop {} to Operational DS", nh);
 523         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id1, nh);
 524
 525     }
 526
 527     protected InstanceIdentifier<IpAdjacencies> getVpnNextHopIpPrefixIdentifier(long vpnId, String primaryIpAddress,
 528                                                                                 String ipPrefix) {
 529         InstanceIdentifier<IpAdjacencies> id = InstanceIdentifier.builder(L3nexthop.class)
 530                 .child(VpnNexthops.class, new VpnNexthopsKey(vpnId))
 531                 .child(VpnNexthop.class, new VpnNexthopKey(primaryIpAddress))
 532                 .child(IpAdjacencies.class, new IpAdjacenciesKey(ipPrefix)).build();
 533         return id;
 534     }
 535
 536     @Nullable
 537     protected VpnNexthop getVpnNexthop(long vpnId, String ipAddress) {
 538
 539         // check if vpn node is there
 540         InstanceIdentifierBuilder<VpnNexthops> idBuilder =
 541             InstanceIdentifier.builder(L3nexthop.class).child(VpnNexthops.class,
 542                 new VpnNexthopsKey(vpnId));
 543         InstanceIdentifier<VpnNexthops> id = idBuilder.build();
 544         Optional<VpnNexthops> vpnNexthops = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 545         if (vpnNexthops.isPresent()) {
 546             // get nexthops list for vpn
 547             List<VpnNexthop> nexthops = vpnNexthops.get().nonnullVpnNexthop();
 548             for (VpnNexthop nexthop : nexthops) {
 549                 if (Objects.equals(nexthop.getIpAddress(), ipAddress)) {
 550                     // return nexthop
 551                     LOG.trace("VpnNextHop : {}", nexthop);
 552                     return nexthop;
 553                 }
 554             }
 555             // return null if not found
 556         }
 557         return null;
 558     }
 559
 560     @Nullable
 561     public AdjacencyResult getRemoteNextHopPointer(BigInteger remoteDpnId, long vpnId, String prefixIp,
 562             @Nullable String nextHopIp, Class<? extends TunnelTypeBase> tunnelType) {
 563         String egressIfName = null;
 564         LOG.trace("getRemoteNextHopPointer: input [remoteDpnId {}, vpnId {}, prefixIp {}, nextHopIp {} ]", remoteDpnId,
 565             vpnId, prefixIp, nextHopIp);
 566
 567         Class<? extends InterfaceType> egressIfType;
 568         ElanInstance elanInstance = getElanInstanceForPrefix(vpnId, prefixIp);
 569         if (elanInstance != null) {
 570             egressIfType = getInterfaceType(elanInstance);
 571         } else {
 572             LOG.warn("Failed to determine network type for prefixIp {} using tunnel", prefixIp);
 573             egressIfType = Tunnel.class;
 574         }
 575
 576         if (Tunnel.class.equals(egressIfType)) {
 577             egressIfName = getTunnelRemoteNextHopPointer(remoteDpnId, nextHopIp, tunnelType);
 578         } else {
 579             egressIfName = getExtPortRemoteNextHopPointer(remoteDpnId, elanInstance);
 580         }
 581
 582         LOG.trace("NextHop pointer for prefixIp {} vpnId {} dpnId {} is {}", prefixIp, vpnId, remoteDpnId,
 583             egressIfName);
 584         return egressIfName != null ? new AdjacencyResult(egressIfName, egressIfType, nextHopIp,
 585                 prefixIp) : null;
 586     }
 587
 588     private void removeVpnNexthopFromDS(long vpnId, String ipPrefix) {
 589
 590         InstanceIdentifierBuilder<VpnNexthop> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
 591             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId))
 592             .child(VpnNexthop.class, new VpnNexthopKey(ipPrefix));
 593         InstanceIdentifier<VpnNexthop> id = idBuilder.build();
 594         // remove from DS
 595         LOG.trace("Removing vpn next hop from datastore : {}", id);
 596         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 597     }
 598
 599     public void removeLocalNextHop(BigInteger dpnId, Long vpnId, String primaryIpAddress, String currDestIpPrefix) {
 600         String nextHopLockStr = vpnId + primaryIpAddress;
 601         try {
 602             if (FibUtil.lockCluster(lockManager, nextHopLockStr, WAIT_TIME_TO_ACQUIRE_LOCK)) {
 603                 VpnNexthop nh = getVpnNexthop(vpnId, primaryIpAddress);
 604                 if (nh != null) {
 605                     List<IpAdjacencies> prefixesList = new ArrayList<>(nh.nonnullIpAdjacencies());
 606                     IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currDestIpPrefix).build();
 607                     prefixesList.remove(prefix);
 608                     if (prefixesList.isEmpty()) { //remove the group only if there are no more flows using this group
 609                         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, nh.getEgressPointer(),
 610                                 primaryIpAddress, GroupTypes.GroupAll, Collections.emptyList());
 611                         // remove Group ...
 612                         mdsalApiManager.removeGroup(groupEntity);
 613                         //update MD-SAL DS
 614                         removeVpnNexthopFromDS(vpnId, primaryIpAddress);
 615                         //release groupId
 616                         removeNextHopPointer(getNextHopKey(vpnId, primaryIpAddress));
 617                         LOG.debug("Local Next hop {} for {} {} on dpn {} successfully deleted",
 618                                 nh.getEgressPointer(), vpnId, primaryIpAddress, dpnId);
 619                     } else {
 620                         //remove the currIpPrefx from IpPrefixList of the vpnNexthop
 621                         LOG.trace("Removing the prefix {} from vpnNextHop {} Operational DS", currDestIpPrefix, nh);
 622                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
 623                                 getVpnNextHopIpPrefixIdentifier(vpnId, primaryIpAddress, currDestIpPrefix));
 624                     }
 625                 } else {
 626                     //throw error
 627                     LOG.error("Local NextHop for VpnId {} curIpPrefix {} on dpn {} primaryIpAddress {} not deleted",
 628                             vpnId, currDestIpPrefix, dpnId, primaryIpAddress);
 629                 }
 630             }
 631         } finally {
 632             FibUtil.unlockCluster(lockManager, nextHopLockStr);
 633         }
 634     }
 635
 636     public void setConfTransType(String service, String transportType) {
 637
 638         if (!service.equalsIgnoreCase("L3VPN")) {
 639             LOG.error("Incorrect service {} provided for setting the transport type.", service);
 640             return;
 641         }
 642
 643         L3VPNTransportTypes transType = L3VPNTransportTypes.validateTransportType(transportType
 644                 .toUpperCase(Locale.getDefault()));
 645
 646         if (transType != L3VPNTransportTypes.Invalid) {
 647             configuredTransportTypeL3VPN = transType;
 648         }
 649     }
 650
 651     public void writeConfTransTypeConfigDS() {
 652         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, getConfTransportTypeIdentifier(),
 653             createConfTransportType(configuredTransportTypeL3VPN.getTransportType()));
 654     }
 655
 656     public L3VPNTransportTypes getConfiguredTransportTypeL3VPN() {
 657         return this.configuredTransportTypeL3VPN;
 658     }
 659
 660     public String getReqTransType() {
 661         if (configuredTransportTypeL3VPN == L3VPNTransportTypes.Invalid) {
 662             /*
 663              * Restart scenario, Read from the ConfigDS.
 664              * if the value is Unset, cache value as VxLAN.
 665              */
 666             LOG.trace("configureTransportType is not yet set.");
 667             Optional<ConfTransportTypeL3vpn> configuredTransTypeFromConfig =
 668                 MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, getConfTransportTypeIdentifier());
 669
 670             if (configuredTransTypeFromConfig.isPresent()) {
 671                 if (TunnelTypeGre.class.equals(configuredTransTypeFromConfig.get().getTransportType())) {
 672                     configuredTransportTypeL3VPN = L3VPNTransportTypes.GRE;
 673                 } else {
 674                     configuredTransportTypeL3VPN = L3VPNTransportTypes.VxLAN;
 675                 }
 676                 LOG.trace("configuredTransportType set from config DS to {}",
 677                     getConfiguredTransportTypeL3VPN().getTransportType());
 678             } else {
 679                 setConfTransType("L3VPN", L3VPNTransportTypes.VxLAN.getTransportType());
 680                 LOG.trace("configuredTransportType is not set in the Config DS. VxLAN as default will be used.");
 681             }
 682         } else {
 683             LOG.trace("configuredTransportType is set as {}", getConfiguredTransportTypeL3VPN().getTransportType());
 684         }
 685         return getConfiguredTransportTypeL3VPN().getTransportType();
 686     }
 687
 688     public InstanceIdentifier<ConfTransportTypeL3vpn> getConfTransportTypeIdentifier() {
 689         return InstanceIdentifier.builder(ConfTransportTypeL3vpn.class).build();
 690     }
 691
 692     private ConfTransportTypeL3vpn createConfTransportType(String type) {
 693         ConfTransportTypeL3vpn confTransType;
 694         switch (type) {
 695             case ITMConstants.TUNNEL_TYPE_GRE:
 696                 confTransType = new ConfTransportTypeL3vpnBuilder().setTransportType(TunnelTypeGre.class).build();
 697                 LOG.trace("Setting the confTransportType to GRE.");
 698                 break;
 699             case ITMConstants.TUNNEL_TYPE_VXLAN:
 700                 confTransType = new ConfTransportTypeL3vpnBuilder().setTransportType(TunnelTypeVxlan.class).build();
 701                 LOG.trace("Setting the confTransportType to VxLAN.");
 702                 break;
 703             default:
 704                 LOG.trace("Invalid transport type {} passed to Config DS ", type);
 705                 confTransType = null;
 706                 break;
 707         }
 708         return confTransType;
 709     }
 710
 711     public Class<? extends TunnelTypeBase> getReqTunType(String transportType) {
 712         switch (transportType) {
 713             case "VXLAN":
 714                 return TunnelTypeVxlan.class;
 715             case "GRE":
 716                 return TunnelTypeGre.class;
 717             default:
 718                 return TunnelTypeMplsOverGre.class;
 719         }
 720     }
 721
 722     public String getTransportTypeStr(String tunType) {
 723         if (tunType.equals(TunnelTypeVxlan.class.toString())) {
 724             return ITMConstants.TUNNEL_TYPE_VXLAN;
 725         } else if (tunType.equals(TunnelTypeGre.class.toString())) {
 726             return ITMConstants.TUNNEL_TYPE_GRE;
 727         } else if (tunType.equals(TunnelTypeMplsOverGre.class.toString())) {
 728             return ITMConstants.TUNNEL_TYPE_MPLSoGRE;
 729         } else {
 730             return ITMConstants.TUNNEL_TYPE_INVALID;
 731         }
 732     }
 733
 734     @Override
 735     @PreDestroy
 736     public void close() {
 737         LOG.info("{} close", getClass().getSimpleName());
 738     }
 739
 740     // TODO Clean up the exception handling
 741     @SuppressWarnings("checkstyle:IllegalCatch")
 742     @Nullable
 743     private String getTunnelRemoteNextHopPointer(BigInteger remoteDpnId, String nextHopIp,
 744                                                  Class<? extends TunnelTypeBase> tunnelType) {
 745         if (nextHopIp != null && !nextHopIp.isEmpty()) {
 746             try {
 747                 // here use the config for tunnel type param
 748                 return getTunnelInterfaceName(remoteDpnId,
 749                     org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder
 750                         .getDefaultInstance(nextHopIp), tunnelType);
 751             } catch (Exception ex) {
 752                 LOG.error("Error while retrieving nexthop pointer for nexthop {} remoteDpn {}",
 753                         nextHopIp, remoteDpnId, ex);
 754             }
 755         }
 756
 757         return null;
 758     }
 759
 760     private String getExtPortRemoteNextHopPointer(BigInteger remoteDpnId, ElanInstance elanInstance) {
 761         return elanService.getExternalElanInterface(elanInstance.getElanInstanceName(), remoteDpnId);
 762     }
 763
 764     /**
 765      * Get the interface type associated with the type of ELAN used for routing
 766      * traffic to/from remote compute nodes.
 767      *
 768      * @param elanInstance The elan instance
 769      * @return L2vlan for flat/VLAN network type and Tunnel otherwise
 770      */
 771     private Class<? extends InterfaceType> getInterfaceType(ElanInstance elanInstance) {
 772         Class<? extends SegmentTypeBase> segmentType = elanInstance.getSegmentType();
 773         if (SegmentTypeFlat.class.equals(segmentType) || SegmentTypeVlan.class.equals(segmentType)) {
 774             return L2vlan.class;
 775         }
 776
 777         return Tunnel.class;
 778     }
 779
 780     private ElanInstance getElanInstanceForPrefix(long vpnId, String prefixIp) {
 781         ElanInstance elanInstance = null;
 782         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, prefixIp);
 783         if (prefix != null) {
 784             if (prefix.getNetworkId() != null) {
 785                 elanInstance = elanService.getElanInstance(prefix.getNetworkId().getValue());
 786             }
 787         }
 788
 789         return elanInstance;
 790     }
 791
 792     static class AdjacencyResult {
 793         private final String interfaceName;
 794         private final Class<? extends InterfaceType> interfaceType;
 795         private final String nextHopIp;
 796         private final String prefix;
 797
 798         AdjacencyResult(String interfaceName, Class<? extends InterfaceType> interfaceType, String nextHopIp,
 799                         String prefix) {
 800             this.interfaceName = interfaceName;
 801             this.interfaceType = interfaceType;
 802             this.nextHopIp = nextHopIp;
 803             this.prefix = prefix;
 804         }
 805
 806         public String getInterfaceName() {
 807             return interfaceName;
 808         }
 809
 810         public Class<? extends InterfaceType> getInterfaceType() {
 811             return interfaceType;
 812         }
 813
 814         public String getNextHopIp() {
 815             return nextHopIp;
 816         }
 817
 818         public String getPrefix() {
 819             return prefix;
 820         }
 821
 822         @Override
 823         public int hashCode() {
 824             final int prime = 31;
 825             int result = 1;
 826             result = prime * result + (interfaceName == null ? 0 : interfaceName.hashCode());
 827             return result;
 828         }
 829
 830         @Override
 831         public boolean equals(Object obj) {
 832             if (obj == null) {
 833                 return false;
 834             }
 835
 836             if (getClass() != obj.getClass()) {
 837                 return false;
 838             } else {
 839                 AdjacencyResult other = (AdjacencyResult) obj;
 840                 return interfaceName.equals(other.interfaceName);
 841             }
 842         }
 843     }
 844
 845     protected long setupLoadBalancingNextHop(Long parentVpnId, BigInteger dpnId,
 846             String destPrefix, List<BucketInfo> localBucketInfo, List<BucketInfo> remoteBucketInfo) {
 847         long remoteGroupId = createNextHopPointer(getRemoteSelectGroupKey(parentVpnId, destPrefix));
 848         if (remoteGroupId == FibConstants.INVALID_GROUP_ID) {
 849             LOG.error("Unable to allocate/retrieve remote groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 850             return remoteGroupId;
 851         }
 852         long localGroupId =  createNextHopPointer(getLocalSelectGroupKey(parentVpnId, destPrefix));
 853         if (localGroupId == FibConstants.INVALID_GROUP_ID) {
 854             LOG.error("Unable to allocate/retrieve local groupId for vpnId {} , prefix {}",
 855                 parentVpnId, destPrefix);
 856             return remoteGroupId;
 857         }
 858         List<BucketInfo> combinedBucketInfo = new ArrayList<>();
 859         combinedBucketInfo.addAll(localBucketInfo);
 860         combinedBucketInfo.addAll(remoteBucketInfo);
 861         GroupEntity remoteGroupEntity = MDSALUtil.buildGroupEntity(
 862                 dpnId, remoteGroupId, destPrefix, GroupTypes.GroupSelect, combinedBucketInfo);
 863         GroupEntity localGroupEntity = MDSALUtil.buildGroupEntity(
 864                 dpnId, localGroupId, destPrefix, GroupTypes.GroupSelect, localBucketInfo);
 865         String jobKey = FibUtil.getCreateLocalNextHopJobKey(parentVpnId, dpnId, destPrefix);
 866         jobCoordinator.enqueueJob(jobKey, () -> {
 867             mdsalApiManager.syncInstallGroup(remoteGroupEntity);
 868             if (!localBucketInfo.isEmpty()) {
 869                 mdsalApiManager.syncInstallGroup(localGroupEntity);
 870             }
 871             if (LOG.isDebugEnabled()) {
 872                 LOG.debug("Finished installing GroupEntity with jobCoordinator key {} remoteGroupEntity.groupId {}"
 873                         + "localGroupEntity.groupId {}  groupEntity.groupType {}", jobKey,
 874                         remoteGroupEntity.getGroupId(), localGroupEntity.getGroupId(),
 875                         remoteGroupEntity.getGroupType());
 876             }
 877             // Delete local group(if exists) if there is no local info.
 878             // Local group has to be deleted if all VMs in a compute is deleted.
 879             if (localBucketInfo.isEmpty()) {
 880                 LOG.debug("Deleting local group {} since no local nhs present for "
 881                         + "prefix {}", localGroupEntity.getGroupId(), destPrefix);
 882                 mdsalApiManager.syncRemoveGroup(localGroupEntity);
 883             }
 884             return Collections.emptyList();
 885         });
 886         return remoteGroupId;
 887     }
 888
 889     protected void deleteLoadBalancingNextHop(Long parentVpnId, BigInteger dpnId, String destPrefix) {
 890         long remoteGroupId = createNextHopPointer(getRemoteSelectGroupKey(parentVpnId, destPrefix));
 891         if (remoteGroupId == FibConstants.INVALID_GROUP_ID) {
 892             LOG.error("Unable to allocate/retrieve remote groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 893         }
 894         long localGroupId = createNextHopPointer(getLocalSelectGroupKey(parentVpnId, destPrefix));
 895         if (localGroupId == FibConstants.INVALID_GROUP_ID) {
 896             LOG.error("Unable to allocate/retrieve local groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 897         }
 898         GroupEntity remoteGroupEntity = MDSALUtil.buildGroupEntity(
 899                 dpnId, remoteGroupId, destPrefix, GroupTypes.GroupSelect, Collections.emptyList());
 900         GroupEntity localGroupEntity = MDSALUtil.buildGroupEntity(
 901                 dpnId, localGroupId, destPrefix, GroupTypes.GroupSelect, Collections.emptyList());
 902         String jobKey = FibUtil.getCreateLocalNextHopJobKey(parentVpnId, dpnId, destPrefix);
 903         jobCoordinator.enqueueJob(jobKey, () -> {
 904             mdsalApiManager.syncRemoveGroup(remoteGroupEntity);
 905             mdsalApiManager.syncRemoveGroup(localGroupEntity);
 906             if (LOG.isDebugEnabled()) {
 907                 LOG.debug("Finished removing GroupEntity with jobCoordinator key {} remoteGroupEntity.groupId {}"
 908                     + "localGroupEntity.groupId {}", jobKey, remoteGroupId, localGroupId);
 909             }
 910             return Collections.emptyList();
 911         });
 912     }
 913
 914     long createNextHopGroups(Long vpnId, String rd, BigInteger dpnId, VrfEntry vrfEntry,
 915             @Nullable Routes routes, List<Routes> vpnExtraRoutes) {
 916         List<BucketInfo> localBucketInfo = new ArrayList<>();
 917         List<Routes> clonedVpnExtraRoutes  = new ArrayList<>(vpnExtraRoutes);
 918         if (clonedVpnExtraRoutes.contains(routes)) {
 919             localBucketInfo.addAll(getBucketsForLocalNexthop(vpnId, dpnId, vrfEntry, routes));
 920             clonedVpnExtraRoutes.remove(routes);
 921         }
 922         List<BucketInfo> remoteBucketInfo =
 923             new ArrayList<>(getBucketsForRemoteNexthop(vpnId, dpnId, vrfEntry, rd, clonedVpnExtraRoutes));
 924         return setupLoadBalancingNextHop(vpnId, dpnId,
 925             vrfEntry.getDestPrefix(), localBucketInfo, remoteBucketInfo);
 926     }
 927
 928     private List<BucketInfo> getBucketsForLocalNexthop(Long vpnId, BigInteger dpnId,
 929             VrfEntry vrfEntry, Routes routes) {
 930         @Nullable List<String> nexthopIpList = routes.getNexthopIpList();
 931         if (LOG.isDebugEnabled()) {
 932             LOG.debug("NexthopManager.getBucketsForLocalNexthop invoked with vpnId {} dpnId {} "
 933                             + " vrfEntry.routePaths {}, routes.nexthopList {}", vpnId, dpnId, vrfEntry.getRoutePaths(),
 934                 nexthopIpList);
 935         }
 936         List<BucketInfo> listBucketInfo = new CopyOnWriteArrayList<>();
 937         if (nexthopIpList != null) {
 938             nexthopIpList.parallelStream().forEach(nextHopIp -> {
 939                 String localNextHopIP;
 940                 if (isIpv4Address(nextHopIp)) {
 941                     localNextHopIP = nextHopIp + NwConstants.IPV4PREFIX;
 942                 } else {
 943                     localNextHopIP = nextHopIp + NwConstants.IPV6PREFIX;
 944                 }
 945                 Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 946                 if (localNextHopInfo != null) {
 947                     long groupId = getLocalNextHopGroup(vpnId, localNextHopIP);
 948                     if (groupId == FibConstants.INVALID_GROUP_ID) {
 949                         LOG.error("Unable to allocate groupId for vpnId {} , prefix {} , interface {}", vpnId,
 950                             vrfEntry.getDestPrefix(), localNextHopInfo.getVpnInterfaceName());
 951                         return;
 952                     }
 953                     List<ActionInfo> actionsInfos =
 954                         Collections.singletonList(new ActionGroup(groupId));
 955                     BucketInfo bucket = new BucketInfo(actionsInfos);
 956                     bucket.setWeight(1);
 957                     listBucketInfo.add(bucket);
 958                 }
 959             });
 960         }
 961         LOG.trace("LOCAL: listbucket {}, vpnId {}, dpnId {}, routes {}", listBucketInfo, vpnId, dpnId, routes);
 962         return listBucketInfo;
 963     }
 964
 965     private List<BucketInfo> getBucketsForRemoteNexthop(Long vpnId, BigInteger dpnId, VrfEntry vrfEntry, String rd,
 966             List<Routes> vpnExtraRoutes) {
 967         List<BucketInfo> listBucketInfo = new ArrayList<>();
 968         Map<String, List<ActionInfo>> egressActionMap = new HashMap<>();
 969         vpnExtraRoutes.stream().filter(vpnExtraRoute -> vpnExtraRoute.getNexthopIpList() != null).forEach(
 970             vpnExtraRoute -> vpnExtraRoute.getNexthopIpList().forEach(nextHopIp -> {
 971                 String nextHopPrefixIp;
 972                 if (isIpv4Address(nextHopIp)) {
 973                     nextHopPrefixIp = nextHopIp + NwConstants.IPV4PREFIX;
 974                 } else {
 975                     nextHopPrefixIp = nextHopIp + NwConstants.IPV6PREFIX;
 976                 }
 977                 List<String> tepIpAddresses = fibUtil.getNextHopAddresses(rd, nextHopPrefixIp);
 978                 if (tepIpAddresses.isEmpty()) {
 979                     return;
 980                 }
 981                 // There would be only one nexthop address for a VM ip which would be the tep Ip
 982                 String tepIp = tepIpAddresses.get(0);
 983                 AdjacencyResult adjacencyResult = getRemoteNextHopPointer(dpnId, vpnId,
 984                     vrfEntry.getDestPrefix(), tepIp, TunnelTypeVxlan.class);
 985                 if (adjacencyResult == null) {
 986                     return;
 987                 }
 988                 String egressInterface = adjacencyResult.getInterfaceName();
 989                 if (!FibUtil.isTunnelInterface(adjacencyResult)) {
 990                     return;
 991                 }
 992                 Class<? extends TunnelTypeBase> tunnelType =
 993                     VpnExtraRouteHelper.getTunnelType(itmManager, egressInterface);
 994                 StateTunnelList ifState = null;
 995                 try {
 996                     ifState = fibUtil.getTunnelState(egressInterface);
 997                     if (ifState == null || ifState.getOperState() != TunnelOperStatus.Up) {
 998                         LOG.trace("Tunnel is not up for interface {}", egressInterface);
 999                         return;
1000                     }
1001                 } catch (ReadFailedException e) {
1002                     LOG.error("getBucketsForRemoteNexthop: error in fetching tunnel state for interface {}",
1003                         egressInterface, e);
1004                     return;
1005                 }
1006                 if (!TunnelTypeVxlan.class.equals(tunnelType)) {
1007                     return;
1008                 }
1009                 Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).get();
1010                 Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, nextHopPrefixIp);
1011                 if (prefixInfo == null) {
1012                     LOG.error("No prefix info found for prefix {} in rd {} for VPN {}", nextHopPrefixIp, rd,
1013                         vpnId);
1014                     return;
1015                 }
1016                 BigInteger tunnelId;
1017                 if (FibUtil.isVxlanNetwork(prefixInfo.getNetworkType())) {
1018                     tunnelId = BigInteger.valueOf(prefixInfo.getSegmentationId());
1019                 } else {
1020                     LOG.warn("Network is not of type VXLAN for prefix {}."
1021                         + "Going with default Lport Tag.", prefixInfo.toString());
1022                     tunnelId = BigInteger.valueOf(label);
1023                 }
1024                 List<ActionInfo> actionInfos = new ArrayList<>();
1025                 actionInfos.add(new ActionSetFieldTunnelId(tunnelId));
1026                 String ifName = prefixInfo.getVpnInterfaceName();
1027                 String vpnName = fibUtil.getVpnNameFromId(vpnId);
1028                 if (vpnName == null) {
1029                     return;
1030                 }
1031                 String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, nextHopPrefixIp);
1032                 actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
1033                     new MacAddress(macAddress)));
1034                 List<ActionInfo> egressActions;
1035                 if (egressActionMap.containsKey(egressInterface)) {
1036                     egressActions = egressActionMap.get(egressInterface);
1037                 } else {
1038                     egressActions = getEgressActionsForInterface(egressInterface, actionInfos.size(),
1039                             true, vpnId, vrfEntry.getDestPrefix());
1040                     if (egressActions.isEmpty()) {
1041                         LOG.error("Skipping getBucketsForRemoteNexthop: Empty list of egress actions received for "
1042                                         + "interface {} on dpn {} for vpn {} prefix {} nextHop {}", ifName, dpnId,
1043                                 vpnId, vrfEntry.getDestPrefix(), nextHopPrefixIp);
1044                     }
1045                     egressActionMap.put(egressInterface, egressActions);
1046                 }
1047                 if (egressActions.isEmpty()) {
1048                     LOG.error("Failed to retrieve egress action for prefix {} route-paths {}"
1049                             + " interface {}." + " Aborting remote FIB entry creation.",
1050                         vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), egressInterface);
1051                 }
1052                 actionInfos.addAll(egressActions);
1053                 BucketInfo bucket = new BucketInfo(actionInfos);
1054                 bucket.setWeight(1);
1055                 listBucketInfo.add(bucket);
1056             }));
1057         LOG.trace("LOCAL: listbucket {}, rd {}, dpnId {}, routes {}", listBucketInfo, rd, dpnId, vpnExtraRoutes);
1058         return listBucketInfo;
1059     }
1060
1061     public void createDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId, String destinationIp,
1062                                              Class<? extends TunnelTypeBase> tunnelType) {
1063         Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
1064         int noOfDcGws = availableDcGws.size();
1065         if (noOfDcGws == 1) {
1066             LOG.trace("There are no enough DC GateWays {} present to program LB group", availableDcGws);
1067             return;
1068         }
1069         // TODO : Place the logic to construct all possible DC-GW combination here.
1070         String groupIdKey = FibUtil.getGreLbGroupKey(availableDcGws);
1071         Long groupId = createNextHopPointer(groupIdKey);
1072         List<Bucket> listBucket = new ArrayList<>();
1073         for (int index = 0; index < noOfDcGws; index++) {
1074             if (isTunnelUp(availableDcGws.get(index), dpnId, tunnelType)) {
1075                 listBucket.add(buildBucketForDcGwLbGroup(availableDcGws.get(index), dpnId, index, tunnelType));
1076             }
1077         }
1078         Group group = MDSALUtil.buildGroup(groupId, groupIdKey, GroupTypes.GroupSelect,
1079                         MDSALUtil.buildBucketLists(listBucket));
1080         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1081             confTx -> mdsalApiManager.addGroup(confTx, dpnId, group)), LOG, "Error adding load-balancing group");
1082         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1083             operTx -> FibUtil.updateLbGroupInfo(dpnId, destinationIp, groupIdKey, groupId.toString(), operTx)), LOG,
1084             "Error updating load-balancing group info");
1085         LOG.trace("LB group {} towards DC-GW installed on dpn {}. Group - {}", groupIdKey, dpnId, group);
1086     }
1087
1088     private boolean isTunnelUp(String dcGwIp, BigInteger dpnId, Class<? extends TunnelTypeBase> tunnelType) {
1089         String tunnelName = getTunnelRemoteNextHopPointer(dpnId, dcGwIp, tunnelType);
1090         if (tunnelName != null) {
1091             InstanceIdentifier<StateTunnelList> tunnelStateId =
1092                     InstanceIdentifier.builder(TunnelsState.class).child(
1093                             StateTunnelList.class, new StateTunnelListKey(tunnelName)).build();
1094             return MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, tunnelStateId)
1095                     .toJavaUtil().map(StateTunnelList::getOperState)
1096                     .orElse(TunnelOperStatus.Down) == TunnelOperStatus.Up;
1097         }
1098         return false;
1099     }
1100
1101     private List<Action> getEgressActions(String interfaceName, int actionKey) {
1102         List<Action> actions = Collections.emptyList();
1103         try {
1104             GetEgressActionsForInterfaceInputBuilder egressAction =
1105                     new GetEgressActionsForInterfaceInputBuilder().setIntfName(interfaceName).setActionKey(actionKey);
1106             Future<RpcResult<GetEgressActionsForInterfaceOutput>> result =
1107                     odlInterfaceRpcService.getEgressActionsForInterface(egressAction.build());
1108             RpcResult<GetEgressActionsForInterfaceOutput> rpcResult = result.get();
1109             if (!rpcResult.isSuccessful()) {
1110                 LOG.warn("RPC Call to Get egress actions for interface {} returned with Errors {}",
1111                         interfaceName, rpcResult.getErrors());
1112             } else {
1113                 actions = rpcResult.getResult().nonnullAction();
1114             }
1115         } catch (InterruptedException | ExecutionException e) {
1116             LOG.warn("Exception when egress actions for interface {}", interfaceName, e);
1117         }
1118         return actions;
1119     }
1120
1121     /**
1122      * This method is invoked when the tunnel state is removed from DS.
1123      * If the there is just one DC-GW left in configuration then the LB groups can be deleted.
1124      * Otherwise, the groups are just updated.
1125      */
1126     public void removeOrUpdateDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId,
1127             String destinationIp) {
1128         Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
1129         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
1130             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
1131                 int noOfDcGws = availableDcGws.size();
1132                 // If availableDcGws does not contain the destination Ip it means this is a configuration delete.
1133                 if (!availableDcGws.contains(destinationIp)) {
1134                     availableDcGws.add(destinationIp);
1135                     Collections.sort(availableDcGws);
1136                 }
1137                 // TODO : Place the logic to construct all possible DC-GW combination here.
1138                 int bucketId = availableDcGws.indexOf(destinationIp);
1139                 Optional<DpnLbNexthops> dpnLbNextHops = fibUtil.getDpnLbNexthops(dpnId, destinationIp);
1140                 if (!dpnLbNextHops.isPresent()) {
1141                     return;
1142                 }
1143                 List<String> nextHopKeys = dpnLbNextHops.get().getNexthopKey();
1144                 if (nextHopKeys != null) {
1145                     for (String nextHopKey : nextHopKeys) {
1146                         Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
1147                         if (!optionalNextHops.isPresent()) {
1148                             return;
1149                         }
1150                         Nexthops nexthops = optionalNextHops.get();
1151                         final String groupId = nexthops.getGroupId();
1152                         final long groupIdValue = Long.parseLong(groupId);
1153                         if (noOfDcGws > 1) {
1154                             mdsalApiManager.removeBucket(confTx, dpnId, groupIdValue, bucketId);
1155                         } else {
1156                             LOG.trace("Removed LB group {} on dpn {}", groupIdValue, dpnId);
1157                             mdsalApiManager.removeGroup(confTx, dpnId, groupIdValue);
1158                             removeNextHopPointer(nextHopKey);
1159                         }
1160                         // When the DC-GW is removed from configuration.
1161                         if (noOfDcGws != availableDcGws.size()) {
1162                             FibUtil.removeOrUpdateNextHopInfo(dpnId, nextHopKey, groupId, nexthops, operTx);
1163                         }
1164                     }
1165                 }
1166                 FibUtil.removeDpnIdToNextHopInfo(destinationIp, dpnId, operTx);
1167             }), LOG, "Error removing or updating load-balancing group");
1168         }), LOG, "Error removing or updating load-balancing group");
1169     }
1170
1171     /**
1172      * This method is invoked when the tunnel status is updated.
1173      * The bucket is directly removed/added based on the operational status of the tunnel.
1174      */
1175     public void updateDcGwLoadBalancingGroup(List<String> availableDcGws,
1176             BigInteger dpnId, String destinationIp, boolean isTunnelUp, Class<? extends TunnelTypeBase> tunnelType) {
1177         Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
1178         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
1179             // TODO : Place the logic to construct all possible DC-GW combination here.
1180             int bucketId = availableDcGws.indexOf(destinationIp);
1181             Optional<DpnLbNexthops> dpnLbNextHops = fibUtil.getDpnLbNexthops(dpnId, destinationIp);
1182             if (!dpnLbNextHops.isPresent()) {
1183                 return;
1184             }
1185             List<String> nextHopKeys = dpnLbNextHops.get().getNexthopKey();
1186             if (nextHopKeys != null) {
1187                 for (String nextHopKey : nextHopKeys) {
1188                     Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
1189                     if (!optionalNextHops.isPresent()) {
1190                         return;
1191                     }
1192                     Nexthops nexthops = optionalNextHops.get();
1193                     final String groupId = nexthops.getGroupId();
1194                     final long groupIdValue = Long.parseLong(groupId);
1195                     if (isTunnelUp) {
1196                         Bucket bucket = buildBucketForDcGwLbGroup(destinationIp, dpnId, bucketId, tunnelType);
1197                         LOG.trace("Added bucket {} to group {} on dpn {}.", bucket, groupId, dpnId);
1198                         mdsalApiManager.addBucket(confTx, dpnId, groupIdValue, bucket);
1199                     } else {
1200                         LOG.trace("Removed bucketId {} from group {} on dpn {}.", bucketId, groupId, dpnId);
1201                         mdsalApiManager.removeBucket(confTx, dpnId, groupIdValue, bucketId);
1202                     }
1203                 }
1204             }
1205         }), LOG, "Error updating load-balancing group");
1206     }
1207
1208     private Bucket buildBucketForDcGwLbGroup(String ipAddress, BigInteger dpnId, int index,
1209                                              Class<? extends TunnelTypeBase> tunnelType) {
1210         List<Action> listAction = new ArrayList<>();
1211         // ActionKey 0 goes to mpls label.
1212         int actionKey = 1;
1213         listAction.add(new ActionPushMpls().buildAction());
1214         listAction.add(new ActionRegMove(actionKey++, FibConstants.NXM_REG_MAPPING
1215                 .get(index), 0, 19).buildAction());
1216         String tunnelInterfaceName = getTunnelInterfaceName(dpnId, IpAddressBuilder.getDefaultInstance(ipAddress),
1217             tunnelType);
1218         List<Action> egressActions = getEgressActions(tunnelInterfaceName, actionKey++);
1219         if (!egressActions.isEmpty()) {
1220             listAction.addAll(getEgressActions(tunnelInterfaceName, actionKey++));
1221         } else {
1222             // clear off actions if there is no egress actions.
1223             listAction = Collections.emptyList();
1224         }
1225         //OVS expects a non-zero weight value for load balancing to happen in select groups
1226         return MDSALUtil.buildBucket(listAction, SELECT_GROUP_WEIGHT, index,
1227                 MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP);
1228     }
1229
1230     public void programDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId, String destinationIp,
1231                                               int addRemoveOrUpdate, boolean isTunnelUp,
1232                                               Class<? extends TunnelTypeBase> tunnelType) {
1233         if (NwConstants.ADD_FLOW == addRemoveOrUpdate) {
1234             createDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp, tunnelType);
1235         } else if (NwConstants.DEL_FLOW == addRemoveOrUpdate) {
1236             removeOrUpdateDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp);
1237         } else if (NwConstants.MOD_FLOW == addRemoveOrUpdate) {
1238             updateDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp, isTunnelUp, tunnelType);
1239         }
1240     }
1241 }