fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/NexthopManager.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  11 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  12 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  13
  14 import com.google.common.base.Optional;
  15 import com.google.common.base.Preconditions;
  16 import java.math.BigInteger;
  17 import java.util.ArrayList;
  18 import java.util.Collections;
  19 import java.util.HashMap;
  20 import java.util.List;
  21 import java.util.Locale;
  22 import java.util.Map;
  23 import java.util.concurrent.CopyOnWriteArrayList;
  24 import java.util.concurrent.ExecutionException;
  25 import java.util.concurrent.Future;
  26 import javax.annotation.PreDestroy;
  27 import javax.inject.Inject;
  28 import javax.inject.Singleton;
  29 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  30 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  31 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  32 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  33 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  34 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  35 import org.opendaylight.genius.itm.globals.ITMConstants;
  36 import org.opendaylight.genius.mdsalutil.ActionInfo;
  37 import org.opendaylight.genius.mdsalutil.BucketInfo;
  38 import org.opendaylight.genius.mdsalutil.GroupEntity;
  39 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  40 import org.opendaylight.genius.mdsalutil.NwConstants;
  41 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  42 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  43 import org.opendaylight.genius.mdsalutil.actions.ActionOutput;
  44 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
  45 import org.opendaylight.genius.mdsalutil.actions.ActionPushVlan;
  46 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  47 import org.opendaylight.genius.mdsalutil.actions.ActionRegMove;
  48 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  49 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetSource;
  50 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldVlanVid;
  52 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  53 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  54 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  55 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  56 import org.opendaylight.netvirt.fibmanager.api.L3VPNTransportTypes;
  57 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  58 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev140508.L2vlan;
  59 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev140508.Tunnel;
  60 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
  61 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfaceType;
  62 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.OutputActionCase;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.PushVlanActionCase;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.SetFieldCase;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInput;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInputBuilder;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutput;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInput;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInputBuilder;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolOutput;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInput;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInputBuilder;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdOutput;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceInputBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceOutput;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelOperStatus;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelsState;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelListKey;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelInputBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelOutput;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameInputBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameOutput;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.lockmanager.rev160413.LockManagerService;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupInput;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupInputBuilder;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupOutput;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.SalGroupService;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupId;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupRef;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.Buckets;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeRef;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeBase;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeFlat;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeVlan;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.L3nexthop;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthops;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthopsKey;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopKey;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacencies;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacenciesBuilder;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacenciesKey;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpn;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpnBuilder;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.dpid.l3vpn.lb.nexthops.DpnLbNexthops;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.lb.nexthops.Nexthops;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.add.group.input.buckets.bucket.action.action.NxActionResubmitRpcAddGroupCase;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.nodes.node.table.flow.instructions.instruction.instruction.apply.actions._case.apply.actions.action.action.NxActionRegLoadNodesNodeTableFlowApplyActionsCase;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.nx.action.reg.load.grouping.NxRegLoad;
 131 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 132 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 133 import org.opendaylight.yangtools.yang.common.RpcResult;
 134 import org.slf4j.Logger;
 135 import org.slf4j.LoggerFactory;
 136
 137 @Singleton
 138 public class NexthopManager implements AutoCloseable {
 139     private static final Logger LOG = LoggerFactory.getLogger(NexthopManager.class);
 140     private static final String NEXTHOP_ID_POOL_NAME = "nextHopPointerPool";
 141     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 142     private static final long WAIT_TIME_TO_ACQUIRE_LOCK = 3000L;
 143     private static final int SELECT_GROUP_WEIGHT = 1;
 144
 145     private final DataBroker dataBroker;
 146     private final ManagedNewTransactionRunner txRunner;
 147     private final IMdsalApiManager mdsalApiManager;
 148     private final OdlInterfaceRpcService odlInterfaceRpcService;
 149     private final ItmRpcService itmManager;
 150     private final IdManagerService idManager;
 151     private final IElanService elanService;
 152     private final LockManagerService lockManager;
 153     private final SalGroupService salGroupService;
 154     private final JobCoordinator jobCoordinator;
 155     private final FibUtil fibUtil;
 156     private final IInterfaceManager interfaceManager;
 157     private volatile L3VPNTransportTypes configuredTransportTypeL3VPN = L3VPNTransportTypes.Invalid;
 158
 159     /**
 160      * Provides nexthop functions.
 161      * Creates group ID pool
 162      *
 163      * @param dataBroker       - dataBroker reference
 164      * @param mdsalApiManager  - mdsalApiManager reference
 165      * @param idManager        - idManager reference
 166      * @param odlInterfaceRpcService - odlInterfaceRpcService reference
 167      * @param itmManager       - itmManager reference
 168      */
 169     @Inject
 170     public NexthopManager(final DataBroker dataBroker,
 171                           final IMdsalApiManager mdsalApiManager,
 172                           final IdManagerService idManager,
 173                           final OdlInterfaceRpcService odlInterfaceRpcService,
 174                           final ItmRpcService itmManager,
 175                           final LockManagerService lockManager,
 176                           final IElanService elanService,
 177                           final SalGroupService salGroupService,
 178                           final JobCoordinator jobCoordinator,
 179                           final FibUtil fibUtil,
 180                           final IInterfaceManager interfaceManager) {
 181         this.dataBroker = dataBroker;
 182         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 183         this.mdsalApiManager = mdsalApiManager;
 184         this.idManager = idManager;
 185         this.odlInterfaceRpcService = odlInterfaceRpcService;
 186         this.itmManager = itmManager;
 187         this.elanService = elanService;
 188         this.salGroupService = salGroupService;
 189         this.jobCoordinator = jobCoordinator;
 190         this.fibUtil = fibUtil;
 191         this.lockManager = lockManager;
 192         this.interfaceManager = interfaceManager;
 193         createIdPool();
 194     }
 195
 196     private void createIdPool() {
 197         CreateIdPoolInput createPool = new CreateIdPoolInputBuilder()
 198             .setPoolName(NEXTHOP_ID_POOL_NAME)
 199             .setLow(150000L)
 200             .setHigh(175000L)
 201             .build();
 202         try {
 203             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPool);
 204             if (result != null && result.get().isSuccessful()) {
 205                 LOG.info("Created IdPool for NextHopPointerPool");
 206             }
 207         } catch (InterruptedException | ExecutionException e) {
 208             LOG.error("Failed to create idPool for NextHopPointerPool", e);
 209         }
 210     }
 211
 212     private String getNextHopKey(long vpnId, String ipAddress) {
 213         return "nexthop." + vpnId + ipAddress;
 214     }
 215
 216     String getRemoteSelectGroupKey(long vpnId, String ipAddress) {
 217         return "remote.ecmp.nexthop." + vpnId + ipAddress;
 218     }
 219
 220     String getLocalSelectGroupKey(long vpnId, String ipAddress) {
 221         return "local.ecmp.nexthop." + vpnId + ipAddress;
 222     }
 223
 224     public ItmRpcService getItmManager() {
 225         return itmManager;
 226     }
 227
 228     protected long createNextHopPointer(String nexthopKey) {
 229         AllocateIdInput getIdInput = new AllocateIdInputBuilder()
 230             .setPoolName(NEXTHOP_ID_POOL_NAME).setIdKey(nexthopKey)
 231             .build();
 232         //TODO: Proper error handling once IdManager code is complete
 233         try {
 234             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
 235             RpcResult<AllocateIdOutput> rpcResult = result.get();
 236             return rpcResult.getResult().getIdValue();
 237         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 238             LOG.trace("", e);
 239         }
 240         return 0;
 241     }
 242
 243     protected void removeNextHopPointer(String nexthopKey) {
 244         ReleaseIdInput idInput = new ReleaseIdInputBuilder()
 245             .setPoolName(NEXTHOP_ID_POOL_NAME)
 246             .setIdKey(nexthopKey).build();
 247         try {
 248             RpcResult<ReleaseIdOutput> rpcResult = idManager.releaseId(idInput).get();
 249             if (!rpcResult.isSuccessful()) {
 250                 LOG.error("RPC Call to Get Unique Id for nexthopKey {} returned with Errors {}",
 251                         nexthopKey, rpcResult.getErrors());
 252             }
 253         } catch (InterruptedException | ExecutionException e) {
 254             LOG.warn("Exception when getting Unique Id for key {}", nexthopKey, e);
 255         }
 256     }
 257
 258     protected List<ActionInfo> getEgressActionsForInterface(final String ifName, int actionKey,
 259                                                             boolean isTunnelInterface) {
 260         List<Action> actions;
 261         try {
 262             if (isTunnelInterface && interfaceManager.isItmDirectTunnelsEnabled()) {
 263                 RpcResult<GetEgressActionsForTunnelOutput> rpcResult =
 264                         itmManager.getEgressActionsForTunnel(new GetEgressActionsForTunnelInputBuilder()
 265                                 .setIntfName(ifName).build()).get();
 266                 if (!rpcResult.isSuccessful()) {
 267                     LOG.error("RPC Call to Get egress tunnel actions for interface {} returned with Errors {}",
 268                             ifName, rpcResult.getErrors());
 269                     return Collections.emptyList();
 270                 } else {
 271                     actions = rpcResult.getResult().getAction();
 272                 }
 273             } else {
 274                 RpcResult<GetEgressActionsForInterfaceOutput> rpcResult = odlInterfaceRpcService
 275                         .getEgressActionsForInterface(new GetEgressActionsForInterfaceInputBuilder()
 276                                 .setIntfName(ifName).build()).get();
 277                 if (!rpcResult.isSuccessful()) {
 278                     LOG.error("RPC Call to Get egress vm actions for interface {} returned with Errors {}",
 279                             ifName, rpcResult.getErrors());
 280                     return Collections.emptyList();
 281                 } else {
 282                     actions = rpcResult.getResult().getAction();
 283                 }
 284             }
 285             List<ActionInfo> listActionInfo = new ArrayList<>();
 286             for (Action action : actions) {
 287                 actionKey = action.key().getOrder() + actionKey;
 288                 org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action
 289                     actionClass = action.getAction();
 290                 if (actionClass instanceof OutputActionCase) {
 291                     listActionInfo.add(new ActionOutput(actionKey,
 292                         ((OutputActionCase) actionClass).getOutputAction().getOutputNodeConnector()));
 293                 } else if (actionClass instanceof PushVlanActionCase) {
 294                     listActionInfo.add(new ActionPushVlan(actionKey));
 295                 } else if (actionClass instanceof SetFieldCase) {
 296                     if (((SetFieldCase) actionClass).getSetField().getVlanMatch() != null) {
 297                         int vlanVid = ((SetFieldCase) actionClass).getSetField().getVlanMatch()
 298                             .getVlanId().getVlanId().getValue();
 299                         listActionInfo.add(new ActionSetFieldVlanVid(actionKey, vlanVid));
 300                     }
 301                 } else if (actionClass instanceof NxActionResubmitRpcAddGroupCase) {
 302                     Short tableId = ((NxActionResubmitRpcAddGroupCase) actionClass).getNxResubmit().getTable();
 303                     listActionInfo.add(new ActionNxResubmit(actionKey, tableId));
 304                 } else if (actionClass instanceof NxActionRegLoadNodesNodeTableFlowApplyActionsCase) {
 305                     NxRegLoad nxRegLoad =
 306                         ((NxActionRegLoadNodesNodeTableFlowApplyActionsCase) actionClass).getNxRegLoad();
 307                     listActionInfo.add(new ActionRegLoad(actionKey, NxmNxReg6.class,
 308                         nxRegLoad.getDst().getStart(), nxRegLoad.getDst().getEnd(),
 309                         nxRegLoad.getValue().longValue()));
 310                 }
 311             }
 312             return listActionInfo;
 313         } catch (InterruptedException | ExecutionException e) {
 314             LOG.warn("Exception when egress actions for interface {}", ifName, e);
 315         }
 316         LOG.warn("Exception when egress actions for interface {}", ifName);
 317         return Collections.emptyList();
 318     }
 319
 320     protected String getTunnelInterfaceName(BigInteger srcDpId, BigInteger dstDpId) {
 321         Class<? extends TunnelTypeBase> tunType = getReqTunType(getReqTransType().toUpperCase(Locale.getDefault()));
 322         Future<RpcResult<GetTunnelInterfaceNameOutput>> result;
 323         try {
 324             result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 325                 .setSourceDpid(srcDpId)
 326                 .setDestinationDpid(dstDpId)
 327                 .setTunnelType(tunType)
 328                 .build());
 329             RpcResult<GetTunnelInterfaceNameOutput> rpcResult = result.get();
 330             if (!rpcResult.isSuccessful()) {
 331                 LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
 332             } else {
 333                 return rpcResult.getResult().getInterfaceName();
 334             }
 335         } catch (InterruptedException | ExecutionException e) {
 336             LOG.warn("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstDpId, e);
 337         }
 338         return null;
 339     }
 340
 341     protected String getTunnelInterfaceName(BigInteger srcDpId, org.opendaylight.yang.gen.v1.urn.ietf.params
 342         .xml.ns.yang.ietf.inet.types.rev130715.IpAddress dstIp, Class<? extends TunnelTypeBase> tunnelType) {
 343         Future<RpcResult<GetInternalOrExternalInterfaceNameOutput>> result;
 344         try {
 345             LOG.debug("Trying to fetch tunnel interface name for source dpn {} destIp {} tunType {}", srcDpId,
 346                     dstIp.stringValue(), tunnelType.getName());
 347             result = itmManager.getInternalOrExternalInterfaceName(new GetInternalOrExternalInterfaceNameInputBuilder()
 348                 .setSourceDpid(srcDpId)
 349                 .setDestinationIp(dstIp)
 350                 .setTunnelType(tunnelType)
 351                 .build());
 352             RpcResult<GetInternalOrExternalInterfaceNameOutput> rpcResult = result.get();
 353             if (!rpcResult.isSuccessful()) {
 354                 LOG.warn("RPC Call to getTunnelInterfaceName returned with Errors {}", rpcResult.getErrors());
 355             } else {
 356                 return rpcResult.getResult().getInterfaceName();
 357             }
 358         } catch (InterruptedException | ExecutionException e) {
 359             LOG.warn("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstIp, e);
 360         }
 361         return null;
 362     }
 363
 364
 365     public long getLocalNextHopGroup(long vpnId,
 366             String ipNextHopAddress) {
 367         long groupId = createNextHopPointer(getNextHopKey(vpnId, ipNextHopAddress));
 368         if (groupId == FibConstants.INVALID_GROUP_ID) {
 369             LOG.error("Unable to allocate groupId for vpnId {} , prefix {}", vpnId, ipNextHopAddress);
 370         }
 371         return groupId;
 372     }
 373
 374     public long getLocalSelectGroup(long vpnId,
 375             String ipNextHopAddress) {
 376         long groupId = createNextHopPointer(getLocalSelectGroupKey(vpnId, ipNextHopAddress));
 377         if (groupId == FibConstants.INVALID_GROUP_ID) {
 378             LOG.error("Unable to allocate groupId for vpnId {} , prefix {}", vpnId, ipNextHopAddress);
 379         }
 380         return groupId;
 381     }
 382
 383     public long createLocalNextHop(long vpnId, BigInteger dpnId, String ifName,
 384                                    String primaryIpAddress, String currDestIpPrefix,
 385                                    String gwMacAddress) {
 386         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 387         if (vpnName == null) {
 388             return 0;
 389         }
 390         String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, primaryIpAddress);
 391
 392         long groupId = createNextHopPointer(getNextHopKey(vpnId, primaryIpAddress));
 393         if (groupId == 0) {
 394             LOG.error("Unable to allocate groupId for vpnId {} , IntfName {}, primaryIpAddress {} curIpPrefix {}",
 395                     vpnId, ifName, primaryIpAddress, currDestIpPrefix);
 396             return groupId;
 397         }
 398         String nextHopLockStr = vpnId + primaryIpAddress;
 399         String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, currDestIpPrefix);
 400         jobCoordinator.enqueueJob(jobKey, () -> {
 401             try {
 402                 if (FibUtil.lockCluster(lockManager, nextHopLockStr, WAIT_TIME_TO_ACQUIRE_LOCK)) {
 403                     VpnNexthop nexthop = getVpnNexthop(vpnId, primaryIpAddress);
 404                     LOG.trace("nexthop: {} retrieved for vpnId {}, prefix {}, ifName {} on dpn {}", nexthop, vpnId,
 405                             primaryIpAddress, ifName, dpnId);
 406                     if (nexthop == null) {
 407                         String encMacAddress = macAddress == null
 408                                 ? fibUtil.getMacAddressFromPrefix(ifName, vpnName, primaryIpAddress) : macAddress;
 409                         List<BucketInfo> listBucketInfo = new ArrayList<>();
 410                         List<ActionInfo> listActionInfo = new ArrayList<>();
 411                         int actionKey = 0;
 412                         // MAC re-write
 413                         if (encMacAddress != null) {
 414                             if (gwMacAddress != null) {
 415                                 LOG.trace("The Local NextHop Group Source Mac {} for VpnInterface {} on VPN {}",
 416                                         gwMacAddress, ifName, vpnId);
 417                                 listActionInfo.add(new ActionSetFieldEthernetSource(actionKey++,
 418                                         new MacAddress(gwMacAddress)));
 419                             }
 420                             listActionInfo.add(new ActionSetFieldEthernetDestination(actionKey++,
 421                                     new MacAddress(encMacAddress)));
 422                             // listActionInfo.add(0, new ActionPopMpls());
 423                         } else {
 424                             // FIXME: Log message here.
 425                             LOG.debug("mac address for new local nexthop is null");
 426                         }
 427                         listActionInfo.addAll(getEgressActionsForInterface(ifName, actionKey, false));
 428                         BucketInfo bucket = new BucketInfo(listActionInfo);
 429
 430                         listBucketInfo.add(bucket);
 431                         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, groupId, primaryIpAddress,
 432                                 GroupTypes.GroupAll, listBucketInfo);
 433                         LOG.trace("Install LNH Group: id {}, mac address {}, interface {} for prefix {}", groupId,
 434                                 encMacAddress, ifName, primaryIpAddress);
 435                         //Try to install group directly on the DPN bypassing the FRM, in order to avoid waiting for the
 436                         // group to get installed before programming the flows
 437                         installGroupOnDpn(groupId, dpnId, primaryIpAddress, listBucketInfo,
 438                                 getNextHopKey(vpnId, primaryIpAddress), GroupTypes.GroupAll);
 439                         // install Group
 440                         mdsalApiManager.syncInstallGroup(groupEntity);
 441                         // update MD-SAL DS
 442                         addVpnNexthopToDS(dpnId, vpnId, primaryIpAddress, currDestIpPrefix, groupId);
 443
 444                     } else {
 445                         // Ignore adding new prefix , if it already exists
 446                         List<IpAdjacencies> prefixesList = nexthop.getIpAdjacencies();
 447                         IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currDestIpPrefix).build();
 448                         if (prefixesList != null && prefixesList.contains(prefix)) {
 449                             LOG.trace("Prefix {} is already present in l3nextHop {} ", currDestIpPrefix, nexthop);
 450                         } else {
 451                             IpAdjacenciesBuilder ipPrefixesBuilder =
 452                                     new IpAdjacenciesBuilder().withKey(new IpAdjacenciesKey(currDestIpPrefix));
 453                             LOG.trace("Updating prefix {} to vpnNextHop {} Operational DS", currDestIpPrefix, nexthop);
 454                             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
 455                                     getVpnNextHopIpPrefixIdentifier(vpnId, primaryIpAddress, currDestIpPrefix),
 456                                     ipPrefixesBuilder.build());
 457                         }
 458                     }
 459                 }
 460             } finally {
 461                 FibUtil.unlockCluster(lockManager, nextHopLockStr);
 462             }
 463             return Collections.emptyList();
 464         });
 465         return groupId;
 466     }
 467
 468     private void installGroupOnDpn(long groupId, BigInteger dpnId, String groupName, List<BucketInfo> bucketsInfo,
 469                                      String nextHopKey, GroupTypes groupType) {
 470         NodeRef nodeRef = FibUtil.buildNodeRef(dpnId);
 471         Buckets buckets = FibUtil.buildBuckets(bucketsInfo);
 472         GroupRef groupRef = new GroupRef(FibUtil.buildGroupInstanceIdentifier(groupId, dpnId));
 473         AddGroupInput input = new AddGroupInputBuilder().setNode(nodeRef).setGroupId(new GroupId(groupId))
 474                 .setBuckets(buckets).setGroupRef(groupRef).setGroupType(groupType)
 475                 .setGroupName(groupName).build();
 476         Future<RpcResult<AddGroupOutput>> groupStats = salGroupService.addGroup(input);
 477         RpcResult<AddGroupOutput> rpcResult = null;
 478         try {
 479             rpcResult = groupStats.get();
 480             if (rpcResult != null && rpcResult.isSuccessful()) {
 481                 LOG.info("Group {} with key {} has been successfully installed directly on dpn {}.", groupId,
 482                         nextHopKey, dpnId);
 483             } else {
 484                 LOG.error("Unable to install group {} with key {} directly on dpn {} due to {}.", groupId, nextHopKey,
 485                         dpnId, rpcResult != null ? rpcResult.getErrors() : null);
 486             }
 487         } catch (InterruptedException | ExecutionException e) {
 488             LOG.error("Error while installing group {} directly on dpn {}", groupId, dpnId);
 489         }
 490     }
 491
 492     protected void addVpnNexthopToDS(BigInteger dpnId, long vpnId, String primaryIpAddr,
 493                                      String currIpAddr, long egressPointer) {
 494         InstanceIdentifierBuilder<VpnNexthops> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
 495             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId));
 496
 497         List<IpAdjacencies> ipPrefixesList = new ArrayList<>();
 498         IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currIpAddr).build();
 499         ipPrefixesList.add(prefix);
 500         // Add nexthop to vpn node
 501         VpnNexthop nh = new VpnNexthopBuilder()
 502             .withKey(new VpnNexthopKey(primaryIpAddr))
 503             .setDpnId(dpnId)
 504             .setIpAdjacencies(ipPrefixesList)
 505             .setEgressPointer(egressPointer).build();
 506
 507         InstanceIdentifier<VpnNexthop> id1 = idBuilder
 508             .child(VpnNexthop.class, new VpnNexthopKey(primaryIpAddr)).build();
 509         LOG.trace("Adding vpnnextHop {} to Operational DS", nh);
 510         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id1, nh);
 511
 512     }
 513
 514     protected InstanceIdentifier<IpAdjacencies> getVpnNextHopIpPrefixIdentifier(long vpnId, String primaryIpAddress,
 515                                                                                 String ipPrefix) {
 516         InstanceIdentifier<IpAdjacencies> id = InstanceIdentifier.builder(L3nexthop.class)
 517                 .child(VpnNexthops.class, new VpnNexthopsKey(vpnId))
 518                 .child(VpnNexthop.class, new VpnNexthopKey(primaryIpAddress))
 519                 .child(IpAdjacencies.class, new IpAdjacenciesKey(ipPrefix)).build();
 520         return id;
 521     }
 522
 523     protected VpnNexthop getVpnNexthop(long vpnId, String ipAddress) {
 524
 525         // check if vpn node is there
 526         InstanceIdentifierBuilder<VpnNexthops> idBuilder =
 527             InstanceIdentifier.builder(L3nexthop.class).child(VpnNexthops.class,
 528                 new VpnNexthopsKey(vpnId));
 529         InstanceIdentifier<VpnNexthops> id = idBuilder.build();
 530         Optional<VpnNexthops> vpnNexthops = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 531         if (vpnNexthops.isPresent()) {
 532             // get nexthops list for vpn
 533             List<VpnNexthop> nexthops = vpnNexthops.get().getVpnNexthop();
 534             for (VpnNexthop nexthop : nexthops) {
 535                 if (nexthop.getIpAddress().equals(ipAddress)) {
 536                     // return nexthop
 537                     LOG.trace("VpnNextHop : {}", nexthop);
 538                     return nexthop;
 539                 }
 540             }
 541             // return null if not found
 542         }
 543         return null;
 544     }
 545
 546     public AdjacencyResult getRemoteNextHopPointer(BigInteger remoteDpnId, long vpnId, String prefixIp,
 547             String nextHopIp, Class<? extends TunnelTypeBase> tunnelType) {
 548         String egressIfName = null;
 549         LOG.trace("getRemoteNextHopPointer: input [remoteDpnId {}, vpnId {}, prefixIp {}, nextHopIp {} ]", remoteDpnId,
 550             vpnId, prefixIp, nextHopIp);
 551
 552         Class<? extends InterfaceType> egressIfType;
 553         ElanInstance elanInstance = getElanInstanceForPrefix(vpnId, prefixIp);
 554         if (elanInstance != null) {
 555             egressIfType = getInterfaceType(elanInstance);
 556         } else {
 557             LOG.warn("Failed to determine network type for prefixIp {} using tunnel", prefixIp);
 558             egressIfType = Tunnel.class;
 559         }
 560
 561         if (Tunnel.class.equals(egressIfType)) {
 562             egressIfName = getTunnelRemoteNextHopPointer(remoteDpnId, nextHopIp, tunnelType);
 563         } else {
 564             egressIfName = getExtPortRemoteNextHopPointer(remoteDpnId, elanInstance);
 565         }
 566
 567         LOG.trace("NextHop pointer for prefixIp {} vpnId {} dpnId {} is {}", prefixIp, vpnId, remoteDpnId,
 568             egressIfName);
 569         return egressIfName != null ? new AdjacencyResult(egressIfName, egressIfType, nextHopIp,
 570                 prefixIp) : null;
 571     }
 572
 573     public BigInteger getDpnForPrefix(long vpnId, String prefixIp) {
 574         VpnNexthop vpnNexthop = getVpnNexthop(vpnId, prefixIp);
 575         BigInteger localDpnId = vpnNexthop == null ? null : vpnNexthop.getDpnId();
 576         return localDpnId;
 577     }
 578
 579     private void removeVpnNexthopFromDS(long vpnId, String ipPrefix) {
 580
 581         InstanceIdentifierBuilder<VpnNexthop> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
 582             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId))
 583             .child(VpnNexthop.class, new VpnNexthopKey(ipPrefix));
 584         InstanceIdentifier<VpnNexthop> id = idBuilder.build();
 585         // remove from DS
 586         LOG.trace("Removing vpn next hop from datastore : {}", id);
 587         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 588     }
 589
 590     public void removeLocalNextHop(BigInteger dpnId, Long vpnId, String primaryIpAddress, String currDestIpPrefix) {
 591         String nextHopLockStr = vpnId + primaryIpAddress;
 592         try {
 593             if (FibUtil.lockCluster(lockManager, nextHopLockStr, WAIT_TIME_TO_ACQUIRE_LOCK)) {
 594                 VpnNexthop nh = getVpnNexthop(vpnId, primaryIpAddress);
 595                 if (nh != null) {
 596                     List<IpAdjacencies> prefixesList = nh.getIpAdjacencies();
 597                     IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currDestIpPrefix).build();
 598                     prefixesList.remove(prefix);
 599                     if (prefixesList.isEmpty()) { //remove the group only if there are no more flows using this group
 600                         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, nh.getEgressPointer(),
 601                                 primaryIpAddress, GroupTypes.GroupAll, Collections.EMPTY_LIST);
 602                         // remove Group ...
 603                         mdsalApiManager.removeGroup(groupEntity);
 604                         //update MD-SAL DS
 605                         removeVpnNexthopFromDS(vpnId, primaryIpAddress);
 606                         //release groupId
 607                         removeNextHopPointer(getNextHopKey(vpnId, primaryIpAddress));
 608                         LOG.debug("Local Next hop {} for {} {} on dpn {} successfully deleted",
 609                                 nh.getEgressPointer(), vpnId, primaryIpAddress, dpnId);
 610                     } else {
 611                         //remove the currIpPrefx from IpPrefixList of the vpnNexthop
 612                         LOG.trace("Removing the prefix {} from vpnNextHop {} Operational DS", currDestIpPrefix, nh);
 613                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
 614                                 getVpnNextHopIpPrefixIdentifier(vpnId, primaryIpAddress, currDestIpPrefix));
 615                     }
 616                 } else {
 617                     //throw error
 618                     LOG.error("Local NextHop for VpnId {} curIpPrefix {} on dpn {} primaryIpAddress {} not deleted",
 619                             vpnId, currDestIpPrefix, dpnId, primaryIpAddress);
 620                 }
 621             }
 622         } finally {
 623             FibUtil.unlockCluster(lockManager, nextHopLockStr);
 624         }
 625     }
 626
 627     public void setConfTransType(String service, String transportType) {
 628
 629         if (!service.equalsIgnoreCase("L3VPN")) {
 630             LOG.error("Incorrect service {} provided for setting the transport type.", service);
 631             return;
 632         }
 633
 634         L3VPNTransportTypes transType = L3VPNTransportTypes.validateTransportType(transportType
 635                 .toUpperCase(Locale.getDefault()));
 636
 637         if (transType != L3VPNTransportTypes.Invalid) {
 638             configuredTransportTypeL3VPN = transType;
 639         }
 640     }
 641
 642     public void writeConfTransTypeConfigDS() {
 643         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, getConfTransportTypeIdentifier(),
 644             createConfTransportType(configuredTransportTypeL3VPN.getTransportType()));
 645     }
 646
 647     public L3VPNTransportTypes getConfiguredTransportTypeL3VPN() {
 648         return this.configuredTransportTypeL3VPN;
 649     }
 650
 651     public String getReqTransType() {
 652         if (configuredTransportTypeL3VPN == L3VPNTransportTypes.Invalid) {
 653             /*
 654              * Restart scenario, Read from the ConfigDS.
 655              * if the value is Unset, cache value as VxLAN.
 656              */
 657             LOG.trace("configureTransportType is not yet set.");
 658             Optional<ConfTransportTypeL3vpn> configuredTransTypeFromConfig =
 659                 MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, getConfTransportTypeIdentifier());
 660
 661             if (configuredTransTypeFromConfig.isPresent()) {
 662                 if (configuredTransTypeFromConfig.get().getTransportType().equals(TunnelTypeGre.class)) {
 663                     configuredTransportTypeL3VPN = L3VPNTransportTypes.GRE;
 664                 } else {
 665                     configuredTransportTypeL3VPN = L3VPNTransportTypes.VxLAN;
 666                 }
 667                 LOG.trace("configuredTransportType set from config DS to {}",
 668                     getConfiguredTransportTypeL3VPN().getTransportType());
 669             } else {
 670                 setConfTransType("L3VPN", L3VPNTransportTypes.VxLAN.getTransportType());
 671                 LOG.trace("configuredTransportType is not set in the Config DS. VxLAN as default will be used.");
 672             }
 673         } else {
 674             LOG.trace("configuredTransportType is set as {}", getConfiguredTransportTypeL3VPN().getTransportType());
 675         }
 676         return getConfiguredTransportTypeL3VPN().getTransportType();
 677     }
 678
 679     public InstanceIdentifier<ConfTransportTypeL3vpn> getConfTransportTypeIdentifier() {
 680         return InstanceIdentifier.builder(ConfTransportTypeL3vpn.class).build();
 681     }
 682
 683     private ConfTransportTypeL3vpn createConfTransportType(String type) {
 684         ConfTransportTypeL3vpn confTransType;
 685         switch (type) {
 686             case ITMConstants.TUNNEL_TYPE_GRE:
 687                 confTransType = new ConfTransportTypeL3vpnBuilder().setTransportType(TunnelTypeGre.class).build();
 688                 LOG.trace("Setting the confTransportType to GRE.");
 689                 break;
 690             case ITMConstants.TUNNEL_TYPE_VXLAN:
 691                 confTransType = new ConfTransportTypeL3vpnBuilder().setTransportType(TunnelTypeVxlan.class).build();
 692                 LOG.trace("Setting the confTransportType to VxLAN.");
 693                 break;
 694             default:
 695                 LOG.trace("Invalid transport type {} passed to Config DS ", type);
 696                 confTransType = null;
 697                 break;
 698         }
 699         return confTransType;
 700     }
 701
 702     public Class<? extends TunnelTypeBase> getReqTunType(String transportType) {
 703         switch (transportType) {
 704             case "VXLAN":
 705                 return TunnelTypeVxlan.class;
 706             case "GRE":
 707                 return TunnelTypeGre.class;
 708             default:
 709                 return TunnelTypeMplsOverGre.class;
 710         }
 711     }
 712
 713     public String getTransportTypeStr(String tunType) {
 714         if (tunType.equals(TunnelTypeVxlan.class.toString())) {
 715             return ITMConstants.TUNNEL_TYPE_VXLAN;
 716         } else if (tunType.equals(TunnelTypeGre.class.toString())) {
 717             return ITMConstants.TUNNEL_TYPE_GRE;
 718         } else if (tunType.equals(TunnelTypeMplsOverGre.class.toString())) {
 719             return ITMConstants.TUNNEL_TYPE_MPLSoGRE;
 720         } else {
 721             return ITMConstants.TUNNEL_TYPE_INVALID;
 722         }
 723     }
 724
 725     @Override
 726     @PreDestroy
 727     public void close() {
 728         LOG.info("{} close", getClass().getSimpleName());
 729     }
 730
 731     // TODO Clean up the exception handling
 732     @SuppressWarnings("checkstyle:IllegalCatch")
 733     private String getTunnelRemoteNextHopPointer(BigInteger remoteDpnId, String nextHopIp,
 734                                                  Class<? extends TunnelTypeBase> tunnelType) {
 735         if (nextHopIp != null && !nextHopIp.isEmpty()) {
 736             try {
 737                 // here use the config for tunnel type param
 738                 return getTunnelInterfaceName(remoteDpnId,
 739                     org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder
 740                         .getDefaultInstance(nextHopIp), tunnelType);
 741             } catch (Exception ex) {
 742                 LOG.error("Error while retrieving nexthop pointer for nexthop {} remoteDpn {}",
 743                         nextHopIp, remoteDpnId, ex);
 744             }
 745         }
 746
 747         return null;
 748     }
 749
 750     private String getExtPortRemoteNextHopPointer(BigInteger remoteDpnId, ElanInstance elanInstance) {
 751         return elanService.getExternalElanInterface(elanInstance.getElanInstanceName(), remoteDpnId);
 752     }
 753
 754     /**
 755      * Get the interface type associated with the type of ELAN used for routing
 756      * traffic to/from remote compute nodes.
 757      *
 758      * @param elanInstance The elan instance
 759      * @return L2vlan for flat/VLAN network type and Tunnel otherwise
 760      */
 761     private Class<? extends InterfaceType> getInterfaceType(ElanInstance elanInstance) {
 762         Class<? extends SegmentTypeBase> segmentType = elanInstance.getSegmentType();
 763         if (SegmentTypeFlat.class.equals(segmentType) || SegmentTypeVlan.class.equals(segmentType)) {
 764             return L2vlan.class;
 765         }
 766
 767         return Tunnel.class;
 768     }
 769
 770     private ElanInstance getElanInstanceForPrefix(long vpnId, String prefixIp) {
 771         ElanInstance elanInstance = null;
 772         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, prefixIp);
 773         if (prefix != null) {
 774             if (prefix.getNetworkId() != null) {
 775                 elanInstance = elanService.getElanInstance(prefix.getNetworkId().getValue());
 776             }
 777         }
 778
 779         return elanInstance;
 780     }
 781
 782     static class AdjacencyResult {
 783         private final String interfaceName;
 784         private final Class<? extends InterfaceType> interfaceType;
 785         private final String nextHopIp;
 786         private final String prefix;
 787
 788         AdjacencyResult(String interfaceName, Class<? extends InterfaceType> interfaceType, String nextHopIp,
 789                         String prefix) {
 790             this.interfaceName = interfaceName;
 791             this.interfaceType = interfaceType;
 792             this.nextHopIp = nextHopIp;
 793             this.prefix = prefix;
 794         }
 795
 796         public String getInterfaceName() {
 797             return interfaceName;
 798         }
 799
 800         public Class<? extends InterfaceType> getInterfaceType() {
 801             return interfaceType;
 802         }
 803
 804         public String getNextHopIp() {
 805             return nextHopIp;
 806         }
 807
 808         public String getPrefix() {
 809             return prefix;
 810         }
 811
 812         @Override
 813         public int hashCode() {
 814             final int prime = 31;
 815             int result = 1;
 816             result = prime * result + (interfaceName == null ? 0 : interfaceName.hashCode());
 817             return result;
 818         }
 819
 820         @Override
 821         public boolean equals(Object obj) {
 822             if (obj == null) {
 823                 return false;
 824             }
 825
 826             boolean result = false;
 827             if (getClass() != obj.getClass()) {
 828                 return result;
 829             } else {
 830                 AdjacencyResult other = (AdjacencyResult) obj;
 831                 result = interfaceName.equals(other.interfaceName);
 832             }
 833             return result;
 834         }
 835     }
 836
 837     protected long setupLoadBalancingNextHop(Long parentVpnId, BigInteger dpnId,
 838             String destPrefix, List<BucketInfo> localBucketInfo, List<BucketInfo> remoteBucketInfo) {
 839         long remoteGroupId = createNextHopPointer(getRemoteSelectGroupKey(parentVpnId, destPrefix));
 840         if (remoteGroupId == FibConstants.INVALID_GROUP_ID) {
 841             LOG.error("Unable to allocate/retrieve remote groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 842             return remoteGroupId;
 843         }
 844         long localGroupId =  createNextHopPointer(getLocalSelectGroupKey(parentVpnId, destPrefix));
 845         if (localGroupId == FibConstants.INVALID_GROUP_ID) {
 846             LOG.error("Unable to allocate/retrieve local groupId for vpnId {} , prefix {}",
 847                 parentVpnId, destPrefix);
 848             return remoteGroupId;
 849         }
 850         List<BucketInfo> combinedBucketInfo = new ArrayList<>();
 851         combinedBucketInfo.addAll(localBucketInfo);
 852         combinedBucketInfo.addAll(remoteBucketInfo);
 853         GroupEntity remoteGroupEntity = MDSALUtil.buildGroupEntity(
 854                 dpnId, remoteGroupId, destPrefix, GroupTypes.GroupSelect, combinedBucketInfo);
 855         GroupEntity localGroupEntity = MDSALUtil.buildGroupEntity(
 856                 dpnId, localGroupId, destPrefix, GroupTypes.GroupSelect, localBucketInfo);
 857         String jobKey = FibUtil.getCreateLocalNextHopJobKey(parentVpnId, dpnId, destPrefix);
 858         jobCoordinator.enqueueJob(jobKey, () -> {
 859             mdsalApiManager.syncInstallGroup(remoteGroupEntity);
 860             if (!localBucketInfo.isEmpty()) {
 861                 mdsalApiManager.syncInstallGroup(localGroupEntity);
 862             }
 863             if (LOG.isDebugEnabled()) {
 864                 LOG.debug("Finished installing GroupEntity with jobCoordinator key {} remoteGroupEntity.groupId {}"
 865                         + "localGroupEntity.groupId {}  groupEntity.groupType {}", jobKey,
 866                         remoteGroupEntity.getGroupId(), localGroupEntity.getGroupId(),
 867                         remoteGroupEntity.getGroupType());
 868             }
 869             // Delete local group(if exists) if there is no local info or no remote info.
 870             // Local group has to be deleted if all VMs in a compute is deleted.
 871             // Local group(=remote group) is not required if all next hops are present in the same compute.
 872             if (localBucketInfo.isEmpty() ^ remoteBucketInfo.isEmpty()) {
 873                 LOG.debug("Deleting local group {} since no local nhs present for "
 874                         + "prefix {}", localGroupEntity.getGroupId(), destPrefix);
 875                 mdsalApiManager.syncRemoveGroup(localGroupEntity);
 876             }
 877             return Collections.emptyList();
 878         });
 879         return remoteGroupId;
 880     }
 881
 882     protected void deleteLoadBalancingNextHop(Long parentVpnId, BigInteger dpnId, String destPrefix) {
 883         long remoteGroupId = createNextHopPointer(getRemoteSelectGroupKey(parentVpnId, destPrefix));
 884         if (remoteGroupId == FibConstants.INVALID_GROUP_ID) {
 885             LOG.error("Unable to allocate/retrieve remote groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 886         }
 887         long localGroupId = createNextHopPointer(getLocalSelectGroupKey(parentVpnId, destPrefix));
 888         if (localGroupId == FibConstants.INVALID_GROUP_ID) {
 889             LOG.error("Unable to allocate/retrieve local groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 890         }
 891         GroupEntity remoteGroupEntity = MDSALUtil.buildGroupEntity(
 892                 dpnId, remoteGroupId, destPrefix, GroupTypes.GroupSelect, Collections.emptyList());
 893         GroupEntity localGroupEntity = MDSALUtil.buildGroupEntity(
 894                 dpnId, localGroupId, destPrefix, GroupTypes.GroupSelect, Collections.emptyList());
 895         String jobKey = FibUtil.getCreateLocalNextHopJobKey(parentVpnId, dpnId, destPrefix);
 896         jobCoordinator.enqueueJob(jobKey, () -> {
 897             mdsalApiManager.syncRemoveGroup(remoteGroupEntity);
 898             mdsalApiManager.syncRemoveGroup(localGroupEntity);
 899             if (LOG.isDebugEnabled()) {
 900                 LOG.debug("Finished removing GroupEntity with jobCoordinator key {} remoteGroupEntity.groupId {}"
 901                     + "localGroupEntity.groupId {}", jobKey, remoteGroupId, localGroupId);
 902             }
 903             return Collections.emptyList();
 904         });
 905     }
 906
 907     long createNextHopGroups(Long vpnId, String rd, BigInteger dpnId, VrfEntry vrfEntry,
 908             Routes routes, List<Routes> vpnExtraRoutes) {
 909         List<BucketInfo> localBucketInfo = new ArrayList<>();
 910         List<Routes> clonedVpnExtraRoutes  = new ArrayList<>(vpnExtraRoutes);
 911         if (clonedVpnExtraRoutes.contains(routes)) {
 912             localBucketInfo.addAll(getBucketsForLocalNexthop(vpnId, dpnId, vrfEntry, routes));
 913             clonedVpnExtraRoutes.remove(routes);
 914         }
 915         List<BucketInfo> remoteBucketInfo = new ArrayList<>();
 916         remoteBucketInfo.addAll(getBucketsForRemoteNexthop(vpnId, dpnId, vrfEntry, rd, clonedVpnExtraRoutes));
 917         return setupLoadBalancingNextHop(vpnId, dpnId,
 918             vrfEntry.getDestPrefix(), localBucketInfo, remoteBucketInfo);
 919     }
 920
 921     private List<BucketInfo> getBucketsForLocalNexthop(Long vpnId, BigInteger dpnId,
 922             VrfEntry vrfEntry, Routes routes) {
 923         if (LOG.isDebugEnabled()) {
 924             LOG.debug("NexthopManager.getBucketsForLocalNexthop invoked with vpnId {} dpnId {} "
 925                             + " vrfEntry.routePaths {}, routes.nexthopList {}", vpnId, dpnId, vrfEntry.getRoutePaths(),
 926                     routes.getNexthopIpList());
 927         }
 928         List<BucketInfo> listBucketInfo = new CopyOnWriteArrayList<>();
 929         routes.getNexthopIpList().parallelStream().forEach(nextHopIp -> {
 930             String localNextHopIP;
 931             if (isIpv4Address(nextHopIp)) {
 932                 localNextHopIP = nextHopIp + NwConstants.IPV4PREFIX;
 933             } else {
 934                 localNextHopIP = nextHopIp + NwConstants.IPV6PREFIX;
 935             }
 936             Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 937             if (localNextHopInfo != null) {
 938                 long groupId = getLocalNextHopGroup(vpnId, localNextHopIP);
 939                 if (groupId == FibConstants.INVALID_GROUP_ID) {
 940                     LOG.error("Unable to allocate groupId for vpnId {} , prefix {} , interface {}", vpnId,
 941                             vrfEntry.getDestPrefix(), localNextHopInfo.getVpnInterfaceName());
 942                     return;
 943                 }
 944                 List<ActionInfo> actionsInfos =
 945                         Collections.singletonList(new ActionGroup(groupId));
 946                 BucketInfo bucket = new BucketInfo(actionsInfos);
 947                 bucket.setWeight(1);
 948                 listBucketInfo.add(bucket);
 949             }
 950         });
 951         LOG.trace("LOCAL: listbucket {}, vpnId {}, dpnId {}, routes {}", listBucketInfo, vpnId, dpnId, routes);
 952         return listBucketInfo;
 953     }
 954
 955     private List<BucketInfo> getBucketsForRemoteNexthop(Long vpnId, BigInteger dpnId, VrfEntry vrfEntry, String rd,
 956             List<Routes> vpnExtraRoutes) {
 957         List<BucketInfo> listBucketInfo = new ArrayList<>();
 958         Map<String, List<ActionInfo>> egressActionMap = new HashMap<>();
 959         vpnExtraRoutes.forEach(vpnExtraRoute -> vpnExtraRoute.getNexthopIpList().forEach(nextHopIp -> {
 960             String nextHopPrefixIp;
 961             if (isIpv4Address(nextHopIp)) {
 962                 nextHopPrefixIp = nextHopIp + NwConstants.IPV4PREFIX;
 963             } else {
 964                 nextHopPrefixIp = nextHopIp + NwConstants.IPV6PREFIX;
 965             }
 966             List<String> tepIpAddresses = fibUtil.getNextHopAddresses(rd, nextHopPrefixIp);
 967             if (tepIpAddresses.isEmpty()) {
 968                 return;
 969             }
 970             // There would be only one nexthop address for a VM ip which would be the tep Ip
 971             String tepIp = tepIpAddresses.get(0);
 972             AdjacencyResult adjacencyResult = getRemoteNextHopPointer(dpnId, vpnId,
 973                     vrfEntry.getDestPrefix(), tepIp, TunnelTypeVxlan.class);
 974             if (adjacencyResult == null) {
 975                 return;
 976             }
 977             String egressInterface = adjacencyResult.getInterfaceName();
 978             if (!FibUtil.isTunnelInterface(adjacencyResult)) {
 979                 return;
 980             }
 981             Class<? extends TunnelTypeBase> tunnelType = VpnExtraRouteHelper.getTunnelType(itmManager, egressInterface);
 982             StateTunnelList ifState = null;
 983             try {
 984                 ifState = fibUtil.getTunnelState(egressInterface);
 985                 if (ifState == null || ifState.getOperState() != TunnelOperStatus.Up) {
 986                     LOG.trace("Tunnel is not up for interface {}", egressInterface);
 987                     return;
 988                 }
 989             } catch (ReadFailedException e) {
 990                 LOG.error("error in fetching tunnel state for interface {}", egressInterface, e);
 991             }
 992             if (!TunnelTypeVxlan.class.equals(tunnelType)) {
 993                 return;
 994             }
 995             Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).get();
 996             Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, nextHopPrefixIp);
 997             if (prefixInfo == null) {
 998                 LOG.error("No prefix info found for prefix {} in rd {} for VPN {}", nextHopPrefixIp, rd,
 999                         vpnId);
1000                 return;
1001             }
1002             BigInteger tunnelId;
1003             if (FibUtil.isVxlanNetwork(prefixInfo.getNetworkType())) {
1004                 tunnelId = BigInteger.valueOf(prefixInfo.getSegmentationId());
1005             } else {
1006                 LOG.warn("Network is not of type VXLAN for prefix {}."
1007                         + "Going with default Lport Tag.", prefixInfo.toString());
1008                 tunnelId = BigInteger.valueOf(label);
1009             }
1010             List<ActionInfo> actionInfos = new ArrayList<>();
1011             actionInfos.add(new ActionSetFieldTunnelId(tunnelId));
1012             String ifName = prefixInfo.getVpnInterfaceName();
1013             String vpnName = fibUtil.getVpnNameFromId(vpnId);
1014             if (vpnName == null) {
1015                 return;
1016             }
1017             String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, nextHopPrefixIp);
1018             actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
1019                     new MacAddress(macAddress)));
1020             List<ActionInfo> egressActions;
1021             if (egressActionMap.containsKey(egressInterface)) {
1022                 egressActions = egressActionMap.get(egressInterface);
1023             } else {
1024                 egressActions = getEgressActionsForInterface(egressInterface, actionInfos.size(), true);
1025                 egressActionMap.put(egressInterface, egressActions);
1026             }
1027             if (egressActions.isEmpty()) {
1028                 LOG.error("Failed to retrieve egress action for prefix {} route-paths {}"
1029                         + " interface {}." + " Aborting remote FIB entry creation.",
1030                         vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), egressInterface);
1031             }
1032             actionInfos.addAll(egressActions);
1033             BucketInfo bucket = new BucketInfo(actionInfos);
1034             bucket.setWeight(1);
1035             listBucketInfo.add(bucket);
1036         }));
1037         LOG.trace("LOCAL: listbucket {}, rd {}, dpnId {}, routes {}", listBucketInfo, rd, dpnId, vpnExtraRoutes);
1038         return listBucketInfo;
1039     }
1040
1041     public void createDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId, String destinationIp,
1042                                              Class<? extends TunnelTypeBase> tunnelType) {
1043         Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
1044         int noOfDcGws = availableDcGws.size();
1045         if (noOfDcGws == 1) {
1046             LOG.trace("There are no enough DC GateWays {} present to program LB group", availableDcGws);
1047             return;
1048         }
1049         // TODO : Place the logic to construct all possible DC-GW combination here.
1050         String groupIdKey = FibUtil.getGreLbGroupKey(availableDcGws);
1051         Long groupId = createNextHopPointer(groupIdKey);
1052         List<Bucket> listBucket = new ArrayList<>();
1053         for (int index = 0; index < noOfDcGws; index++) {
1054             if (isTunnelUp(availableDcGws.get(index), dpnId, tunnelType)) {
1055                 listBucket.add(buildBucketForDcGwLbGroup(availableDcGws.get(index), dpnId, index, tunnelType));
1056             }
1057         }
1058         Group group = MDSALUtil.buildGroup(groupId, groupIdKey, GroupTypes.GroupSelect,
1059                         MDSALUtil.buildBucketLists(listBucket));
1060         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1061             confTx -> mdsalApiManager.addGroup(confTx, dpnId, group)), LOG, "Error adding load-balancing group");
1062         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1063             operTx -> FibUtil.updateLbGroupInfo(dpnId, destinationIp, groupIdKey, groupId.toString(), operTx)), LOG,
1064             "Error updating load-balancing group info");
1065         LOG.trace("LB group {} towards DC-GW installed on dpn {}. Group - {}", groupIdKey, dpnId, group);
1066     }
1067
1068     private boolean isTunnelUp(String dcGwIp, BigInteger dpnId, Class<? extends TunnelTypeBase> tunnelType) {
1069         String tunnelName = getTunnelRemoteNextHopPointer(dpnId, dcGwIp, tunnelType);
1070         if (tunnelName != null) {
1071             InstanceIdentifier<StateTunnelList> tunnelStateId =
1072                     InstanceIdentifier.builder(TunnelsState.class).child(
1073                             StateTunnelList.class, new StateTunnelListKey(tunnelName)).build();
1074             return MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, tunnelStateId)
1075                     .toJavaUtil().map(StateTunnelList::getOperState)
1076                     .orElse(TunnelOperStatus.Down) == TunnelOperStatus.Up;
1077         }
1078         return false;
1079     }
1080
1081     private List<Action> getEgressActions(String interfaceName, int actionKey) {
1082         List<Action> actions = Collections.emptyList();
1083         try {
1084             GetEgressActionsForInterfaceInputBuilder egressAction =
1085                     new GetEgressActionsForInterfaceInputBuilder().setIntfName(interfaceName).setActionKey(actionKey);
1086             Future<RpcResult<GetEgressActionsForInterfaceOutput>> result =
1087                     odlInterfaceRpcService.getEgressActionsForInterface(egressAction.build());
1088             RpcResult<GetEgressActionsForInterfaceOutput> rpcResult = result.get();
1089             if (!rpcResult.isSuccessful()) {
1090                 LOG.warn("RPC Call to Get egress actions for interface {} returned with Errors {}",
1091                         interfaceName, rpcResult.getErrors());
1092             } else {
1093                 actions = rpcResult.getResult().getAction();
1094             }
1095         } catch (InterruptedException | ExecutionException e) {
1096             LOG.warn("Exception when egress actions for interface {}", interfaceName, e);
1097         }
1098         return actions;
1099     }
1100
1101     /**
1102      * This method is invoked when the tunnel state is removed from DS.
1103      * If the there is just one DC-GW left in configuration then the LB groups can be deleted.
1104      * Otherwise, the groups are just updated.
1105      */
1106     public void removeOrUpdateDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId,
1107             String destinationIp) {
1108         Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
1109         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
1110             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
1111                 int noOfDcGws = availableDcGws.size();
1112                 // If availableDcGws does not contain the destination Ip it means this is a configuration delete.
1113                 if (!availableDcGws.contains(destinationIp)) {
1114                     availableDcGws.add(destinationIp);
1115                     Collections.sort(availableDcGws);
1116                 }
1117                 // TODO : Place the logic to construct all possible DC-GW combination here.
1118                 int bucketId = availableDcGws.indexOf(destinationIp);
1119                 Optional<DpnLbNexthops> dpnLbNextHops = fibUtil.getDpnLbNexthops(dpnId, destinationIp);
1120                 if (!dpnLbNextHops.isPresent()) {
1121                     return;
1122                 }
1123                 List<String> nextHopKeys = dpnLbNextHops.get().getNexthopKey();
1124                 for (String nextHopKey : nextHopKeys) {
1125                     Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
1126                     if (!optionalNextHops.isPresent()) {
1127                         return;
1128                     }
1129                     Nexthops nexthops = optionalNextHops.get();
1130                     final String groupId = nexthops.getGroupId();
1131                     final long groupIdValue = Long.parseLong(groupId);
1132                     if (noOfDcGws > 1) {
1133                         mdsalApiManager.removeBucket(confTx, dpnId, groupIdValue, bucketId);
1134                     } else {
1135                         LOG.trace("Removed LB group {} on dpn {}", groupIdValue, dpnId);
1136                         mdsalApiManager.removeGroup(confTx, dpnId, groupIdValue);
1137                         removeNextHopPointer(nextHopKey);
1138                     }
1139                     // When the DC-GW is removed from configuration.
1140                     if (noOfDcGws != availableDcGws.size()) {
1141                         FibUtil.removeOrUpdateNextHopInfo(dpnId, nextHopKey, groupId, nexthops, operTx);
1142                     }
1143                 }
1144                 FibUtil.removeDpnIdToNextHopInfo(destinationIp, dpnId, operTx);
1145             }), LOG, "Error removing or updating load-balancing group");
1146         }), LOG, "Error removing or updating load-balancing group");
1147     }
1148
1149     /**
1150      * This method is invoked when the tunnel status is updated.
1151      * The bucket is directly removed/added based on the operational status of the tunnel.
1152      */
1153     public void updateDcGwLoadBalancingGroup(List<String> availableDcGws,
1154             BigInteger dpnId, String destinationIp, boolean isTunnelUp, Class<? extends TunnelTypeBase> tunnelType) {
1155         Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
1156         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
1157             // TODO : Place the logic to construct all possible DC-GW combination here.
1158             int bucketId = availableDcGws.indexOf(destinationIp);
1159             Optional<DpnLbNexthops> dpnLbNextHops = fibUtil.getDpnLbNexthops(dpnId, destinationIp);
1160             if (!dpnLbNextHops.isPresent()) {
1161                 return;
1162             }
1163             List<String> nextHopKeys = dpnLbNextHops.get().getNexthopKey();
1164             for (String nextHopKey : nextHopKeys) {
1165                 Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
1166                 if (!optionalNextHops.isPresent()) {
1167                     return;
1168                 }
1169                 Nexthops nexthops = optionalNextHops.get();
1170                 final String groupId = nexthops.getGroupId();
1171                 final long groupIdValue = Long.parseLong(groupId);
1172                 if (isTunnelUp) {
1173                     Bucket bucket = buildBucketForDcGwLbGroup(destinationIp, dpnId, bucketId, tunnelType);
1174                     LOG.trace("Added bucket {} to group {} on dpn {}.", bucket, groupId, dpnId);
1175                     mdsalApiManager.addBucket(confTx, dpnId, groupIdValue, bucket);
1176                 } else {
1177                     LOG.trace("Removed bucketId {} from group {} on dpn {}.", bucketId, groupId, dpnId);
1178                     mdsalApiManager.removeBucket(confTx, dpnId, groupIdValue, bucketId);
1179                 }
1180             }
1181         }), LOG, "Error updating load-balancing group");
1182     }
1183
1184     private Bucket buildBucketForDcGwLbGroup(String ipAddress, BigInteger dpnId, int index,
1185                                              Class<? extends TunnelTypeBase> tunnelType) {
1186         List<Action> listAction = new ArrayList<>();
1187         // ActionKey 0 goes to mpls label.
1188         int actionKey = 1;
1189         listAction.add(new ActionPushMpls().buildAction());
1190         listAction.add(new ActionRegMove(actionKey++, FibConstants.NXM_REG_MAPPING
1191                 .get(index), 0, 19).buildAction());
1192         String tunnelInterfaceName = getTunnelInterfaceName(dpnId, IpAddressBuilder.getDefaultInstance(ipAddress),
1193             tunnelType);
1194         List<Action> egressActions = getEgressActions(tunnelInterfaceName, actionKey++);
1195         if (!egressActions.isEmpty()) {
1196             listAction.addAll(getEgressActions(tunnelInterfaceName, actionKey++));
1197         } else {
1198             // clear off actions if there is no egress actions.
1199             listAction = Collections.emptyList();
1200         }
1201         //OVS expects a non-zero weight value for load balancing to happen in select groups
1202         return MDSALUtil.buildBucket(listAction, SELECT_GROUP_WEIGHT, index,
1203                 MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP);
1204     }
1205
1206     public void programDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId, String destinationIp,
1207                                               int addRemoveOrUpdate, boolean isTunnelUp,
1208                                               Class<? extends TunnelTypeBase> tunnelType) {
1209         if (NwConstants.ADD_FLOW == addRemoveOrUpdate) {
1210             createDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp, tunnelType);
1211         } else if (NwConstants.DEL_FLOW == addRemoveOrUpdate) {
1212             removeOrUpdateDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp);
1213         } else if (NwConstants.MOD_FLOW == addRemoveOrUpdate) {
1214             updateDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp, isTunnelUp, tunnelType);
1215         }
1216     }
1217 }