fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/NexthopManager.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static java.util.stream.Collectors.toList;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  14
  15 import com.google.common.base.Optional;
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.util.concurrent.ListenableFuture;
  18
  19 import java.math.BigInteger;
  20 import java.util.ArrayList;
  21 import java.util.Arrays;
  22 import java.util.Collections;
  23 import java.util.HashMap;
  24 import java.util.List;
  25 import java.util.Locale;
  26 import java.util.Map;
  27 import java.util.Objects;
  28 import java.util.concurrent.CopyOnWriteArrayList;
  29 import java.util.concurrent.ExecutionException;
  30 import java.util.concurrent.Future;
  31
  32 import javax.annotation.PreDestroy;
  33 import javax.inject.Inject;
  34 import javax.inject.Singleton;
  35
  36 import org.eclipse.jdt.annotation.Nullable;
  37 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  38 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  39 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  40 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  41 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  42 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  43 import org.opendaylight.genius.itm.globals.ITMConstants;
  44 import org.opendaylight.genius.mdsalutil.ActionInfo;
  45 import org.opendaylight.genius.mdsalutil.BucketInfo;
  46 import org.opendaylight.genius.mdsalutil.GroupEntity;
  47 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  48 import org.opendaylight.genius.mdsalutil.NwConstants;
  49 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  50 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionOutput;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionPushVlan;
  54 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  55 import org.opendaylight.genius.mdsalutil.actions.ActionRegMove;
  56 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  57 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetSource;
  58 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  59 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldVlanVid;
  60 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  61 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  62 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  63 import org.opendaylight.netvirt.fibmanager.api.L3VPNTransportTypes;
  64 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  65 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev170119.L2vlan;
  66 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev170119.Tunnel;
  67 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfaceType;
  69 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.OutputActionCase;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.PushVlanActionCase;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.SetFieldCase;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInput;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInputBuilder;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutput;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInput;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInputBuilder;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolOutput;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInput;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInputBuilder;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdOutput;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceInputBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceOutput;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelOperStatus;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelsState;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelListKey;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rev160406.DcGatewayIpList;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelInputBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelOutput;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameInputBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameOutput;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.lockmanager.rev160413.LockManagerService;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupInput;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupInputBuilder;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupOutput;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.SalGroupService;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupId;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupRef;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.Buckets;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeRef;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeBase;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeFlat;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeVlan;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.L3nexthop;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthops;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthopsKey;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopKey;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacencies;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacenciesBuilder;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacenciesKey;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpn;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpnBuilder;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.lb.nexthops.Nexthops;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.add.group.input.buckets.bucket.action.action.NxActionResubmitRpcAddGroupCase;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.nodes.node.table.flow.instructions.instruction.instruction.apply.actions._case.apply.actions.action.action.NxActionRegLoadNodesNodeTableFlowApplyActionsCase;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.nx.action.reg.load.grouping.NxRegLoad;
 138 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 139 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 140 import org.opendaylight.yangtools.yang.common.RpcResult;
 141 import org.slf4j.Logger;
 142 import org.slf4j.LoggerFactory;
 143
 144 @Singleton
 145 public class NexthopManager implements AutoCloseable {
 146     private static final Logger LOG = LoggerFactory.getLogger(NexthopManager.class);
 147     private static final String NEXTHOP_ID_POOL_NAME = "nextHopPointerPool";
 148     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 149     private static final long WAIT_TIME_TO_ACQUIRE_LOCK = 3000L;
 150     private static final int SELECT_GROUP_WEIGHT = 1;
 151     private static final int RETRY_COUNT = 6;
 152     private static final String NEXTHOPMANAGER_JOB_KEY_PREFIX = "NextHopManager";
 153
 154     private final DataBroker dataBroker;
 155     private final ManagedNewTransactionRunner txRunner;
 156     private final IMdsalApiManager mdsalApiManager;
 157     private final OdlInterfaceRpcService odlInterfaceRpcService;
 158     private final ItmRpcService itmManager;
 159     private final IdManagerService idManager;
 160     private final IElanService elanService;
 161     private final LockManagerService lockManager;
 162     private final SalGroupService salGroupService;
 163     private final JobCoordinator jobCoordinator;
 164     private final FibUtil fibUtil;
 165     private final IInterfaceManager interfaceManager;
 166     private volatile L3VPNTransportTypes configuredTransportTypeL3VPN = L3VPNTransportTypes.Invalid;
 167
 168     /**
 169      * Provides nexthop functions.
 170      * Creates group ID pool
 171      *
 172      * @param dataBroker       - dataBroker reference
 173      * @param mdsalApiManager  - mdsalApiManager reference
 174      * @param idManager        - idManager reference
 175      * @param odlInterfaceRpcService - odlInterfaceRpcService reference
 176      * @param itmManager       - itmManager reference
 177      */
 178     @Inject
 179     public NexthopManager(final DataBroker dataBroker,
 180                           final IMdsalApiManager mdsalApiManager,
 181                           final IdManagerService idManager,
 182                           final OdlInterfaceRpcService odlInterfaceRpcService,
 183                           final ItmRpcService itmManager,
 184                           final LockManagerService lockManager,
 185                           final IElanService elanService,
 186                           final SalGroupService salGroupService,
 187                           final JobCoordinator jobCoordinator,
 188                           final FibUtil fibUtil,
 189                           final IInterfaceManager interfaceManager) {
 190         this.dataBroker = dataBroker;
 191         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 192         this.mdsalApiManager = mdsalApiManager;
 193         this.idManager = idManager;
 194         this.odlInterfaceRpcService = odlInterfaceRpcService;
 195         this.itmManager = itmManager;
 196         this.elanService = elanService;
 197         this.salGroupService = salGroupService;
 198         this.jobCoordinator = jobCoordinator;
 199         this.fibUtil = fibUtil;
 200         this.lockManager = lockManager;
 201         this.interfaceManager = interfaceManager;
 202         createIdPool();
 203     }
 204
 205     private void createIdPool() {
 206         CreateIdPoolInput createPool = new CreateIdPoolInputBuilder()
 207             .setPoolName(NEXTHOP_ID_POOL_NAME)
 208             .setLow(150000L)
 209             .setHigh(175000L)
 210             .build();
 211         try {
 212             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPool);
 213             if (result != null && result.get().isSuccessful()) {
 214                 LOG.info("Created IdPool for NextHopPointerPool");
 215             }
 216         } catch (InterruptedException | ExecutionException e) {
 217             LOG.error("Failed to create idPool for NextHopPointerPool", e);
 218         }
 219     }
 220
 221     private String getNextHopKey(long vpnId, String ipAddress) {
 222         return "nexthop." + vpnId + ipAddress;
 223     }
 224
 225     String getRemoteSelectGroupKey(long vpnId, String ipAddress) {
 226         return "remote.ecmp.nexthop." + vpnId + ipAddress;
 227     }
 228
 229     String getLocalSelectGroupKey(long vpnId, String ipAddress) {
 230         return "local.ecmp.nexthop." + vpnId + ipAddress;
 231     }
 232
 233     public ItmRpcService getItmManager() {
 234         return itmManager;
 235     }
 236
 237     protected long createNextHopPointer(String nexthopKey) {
 238         AllocateIdInput getIdInput = new AllocateIdInputBuilder()
 239             .setPoolName(NEXTHOP_ID_POOL_NAME).setIdKey(nexthopKey)
 240             .build();
 241         //TODO: Proper error handling once IdManager code is complete
 242         try {
 243             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
 244             RpcResult<AllocateIdOutput> rpcResult = result.get();
 245             return rpcResult.getResult().getIdValue();
 246         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 247             // FIXME: NPEs should not be caught but rather their root cause should be eliminated
 248             LOG.trace("Failed to allocate {}", getIdInput, e);
 249         }
 250         return 0;
 251     }
 252
 253     protected void removeNextHopPointer(String nexthopKey) {
 254         ReleaseIdInput idInput = new ReleaseIdInputBuilder()
 255             .setPoolName(NEXTHOP_ID_POOL_NAME)
 256             .setIdKey(nexthopKey).build();
 257         try {
 258             RpcResult<ReleaseIdOutput> rpcResult = idManager.releaseId(idInput).get();
 259             if (!rpcResult.isSuccessful()) {
 260                 LOG.error("RPC Call to Get Unique Id for nexthopKey {} returned with Errors {}",
 261                         nexthopKey, rpcResult.getErrors());
 262             }
 263         } catch (InterruptedException | ExecutionException e) {
 264             LOG.warn("Exception when getting Unique Id for key {}", nexthopKey, e);
 265         }
 266     }
 267
 268     protected List<ActionInfo> getEgressActionsForInterface(final String ifName, int actionKey,
 269                                                             boolean isTunnelInterface,
 270                                                             long vpnId, String destIpPrefix) {
 271         List<Action> actions;
 272         try {
 273             if (isTunnelInterface && interfaceManager.isItmDirectTunnelsEnabled()) {
 274                 RpcResult<GetEgressActionsForTunnelOutput> rpcResult =
 275                         itmManager.getEgressActionsForTunnel(new GetEgressActionsForTunnelInputBuilder()
 276                                 .setIntfName(ifName).build()).get();
 277                 if (!rpcResult.isSuccessful()) {
 278                     LOG.error("RPC Call to Get egress tunnel actions for interface {} returned with Errors {}",
 279                             ifName, rpcResult.getErrors());
 280                     return Collections.emptyList();
 281                 } else {
 282                     actions = rpcResult.getResult().nonnullAction();
 283                 }
 284             } else {
 285                 RpcResult<GetEgressActionsForInterfaceOutput> rpcResult = odlInterfaceRpcService
 286                         .getEgressActionsForInterface(new GetEgressActionsForInterfaceInputBuilder()
 287                                 .setIntfName(ifName).build()).get();
 288                 if (!rpcResult.isSuccessful()) {
 289                     LOG.error("RPC Call to Get egress vm actions for interface {} vpnId {} ipPrefix {} returned with "
 290                                     + "Errors {}", ifName, vpnId, destIpPrefix, rpcResult.getErrors());
 291                     return Collections.emptyList();
 292                 } else {
 293                     actions = rpcResult.getResult().nonnullAction();
 294                 }
 295             }
 296             List<ActionInfo> listActionInfo = new ArrayList<>();
 297             for (Action action : actions) {
 298                 actionKey = action.key().getOrder() + actionKey;
 299                 org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action
 300                     actionClass = action.getAction();
 301                 if (actionClass instanceof OutputActionCase) {
 302                     listActionInfo.add(new ActionOutput(actionKey,
 303                         ((OutputActionCase) actionClass).getOutputAction().getOutputNodeConnector()));
 304                 } else if (actionClass instanceof PushVlanActionCase) {
 305                     listActionInfo.add(new ActionPushVlan(actionKey));
 306                 } else if (actionClass instanceof SetFieldCase) {
 307                     if (((SetFieldCase) actionClass).getSetField().getVlanMatch() != null) {
 308                         int vlanVid = ((SetFieldCase) actionClass).getSetField().getVlanMatch()
 309                             .getVlanId().getVlanId().getValue();
 310                         listActionInfo.add(new ActionSetFieldVlanVid(actionKey, vlanVid));
 311                     }
 312                 } else if (actionClass instanceof NxActionResubmitRpcAddGroupCase) {
 313                     Short tableId = ((NxActionResubmitRpcAddGroupCase) actionClass).getNxResubmit().getTable();
 314                     listActionInfo.add(new ActionNxResubmit(actionKey, tableId));
 315                 } else if (actionClass instanceof NxActionRegLoadNodesNodeTableFlowApplyActionsCase) {
 316                     NxRegLoad nxRegLoad =
 317                         ((NxActionRegLoadNodesNodeTableFlowApplyActionsCase) actionClass).getNxRegLoad();
 318                     listActionInfo.add(new ActionRegLoad(actionKey, NxmNxReg6.class,
 319                         nxRegLoad.getDst().getStart(), nxRegLoad.getDst().getEnd(),
 320                         nxRegLoad.getValue().longValue()));
 321                 }
 322             }
 323             return listActionInfo;
 324         } catch (InterruptedException | ExecutionException | NullPointerException e) {
 325             LOG.error("Exception when egress actions for interface {} isTunnel {} vpnId {} ipPrefix {}", ifName,
 326                     isTunnelInterface, vpnId, destIpPrefix, e);
 327         }
 328         LOG.warn("Exception when egress actions for interface {}", ifName);
 329         return Collections.emptyList();
 330     }
 331
 332     @Nullable
 333     protected String getTunnelInterfaceName(BigInteger srcDpId, BigInteger dstDpId) {
 334         Class<? extends TunnelTypeBase> tunType = getReqTunType(getReqTransType().toUpperCase(Locale.getDefault()));
 335         Future<RpcResult<GetTunnelInterfaceNameOutput>> result;
 336         try {
 337             result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 338                 .setSourceDpid(srcDpId)
 339                 .setDestinationDpid(dstDpId)
 340                 .setTunnelType(tunType)
 341                 .build());
 342             RpcResult<GetTunnelInterfaceNameOutput> rpcResult = result.get();
 343             if (!rpcResult.isSuccessful()) {
 344                 LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
 345             } else {
 346                 return rpcResult.getResult().getInterfaceName();
 347             }
 348         } catch (InterruptedException | ExecutionException e) {
 349             LOG.warn("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstDpId, e);
 350         }
 351         return null;
 352     }
 353
 354     @Nullable
 355     protected String getTunnelInterfaceName(BigInteger srcDpId, org.opendaylight.yang.gen.v1.urn.ietf.params
 356         .xml.ns.yang.ietf.inet.types.rev130715.IpAddress dstIp, Class<? extends TunnelTypeBase> tunnelType) {
 357         Future<RpcResult<GetInternalOrExternalInterfaceNameOutput>> result;
 358         try {
 359             LOG.debug("Trying to fetch tunnel interface name for source dpn {} destIp {} tunType {}", srcDpId,
 360                     dstIp.stringValue(), tunnelType.getName());
 361             result = itmManager.getInternalOrExternalInterfaceName(new GetInternalOrExternalInterfaceNameInputBuilder()
 362                 .setSourceDpid(srcDpId)
 363                 .setDestinationIp(dstIp)
 364                 .setTunnelType(tunnelType)
 365                 .build());
 366             RpcResult<GetInternalOrExternalInterfaceNameOutput> rpcResult = result.get();
 367             if (!rpcResult.isSuccessful()) {
 368                 LOG.warn("RPC Call to getTunnelInterfaceName returned with Errors {}", rpcResult.getErrors());
 369             } else {
 370                 return rpcResult.getResult().getInterfaceName();
 371             }
 372         } catch (InterruptedException | ExecutionException e) {
 373             LOG.error("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstIp, e);
 374         }
 375         return null;
 376     }
 377
 378
 379     public long getLocalNextHopGroup(long vpnId,
 380             String ipNextHopAddress) {
 381         long groupId = createNextHopPointer(getNextHopKey(vpnId, ipNextHopAddress));
 382         if (groupId == FibConstants.INVALID_GROUP_ID) {
 383             LOG.error("Unable to allocate groupId for vpnId {} , prefix {}", vpnId, ipNextHopAddress);
 384         }
 385         return groupId;
 386     }
 387
 388     public long getLocalSelectGroup(long vpnId,
 389             String ipNextHopAddress) {
 390         long groupId = createNextHopPointer(getLocalSelectGroupKey(vpnId, ipNextHopAddress));
 391         if (groupId == FibConstants.INVALID_GROUP_ID) {
 392             LOG.error("Unable to allocate groupId for vpnId {} , prefix {}", vpnId, ipNextHopAddress);
 393         }
 394         return groupId;
 395     }
 396
 397     public long createLocalNextHop(long vpnId, BigInteger dpnId, String ifName,
 398                                    String primaryIpAddress, String currDestIpPrefix,
 399                                    String gwMacAddress) {
 400         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 401         if (vpnName == null) {
 402             return 0;
 403         }
 404         String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, primaryIpAddress);
 405
 406         long groupId = createNextHopPointer(getNextHopKey(vpnId, primaryIpAddress));
 407         if (groupId == 0) {
 408             LOG.error("Unable to allocate groupId for vpnId {} , IntfName {}, primaryIpAddress {} curIpPrefix {}",
 409                     vpnId, ifName, primaryIpAddress, currDestIpPrefix);
 410             return groupId;
 411         }
 412         String nextHopLockStr = vpnId + primaryIpAddress;
 413         String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, currDestIpPrefix);
 414         jobCoordinator.enqueueJob(jobKey, () -> {
 415             try {
 416                 if (FibUtil.lockCluster(lockManager, nextHopLockStr, WAIT_TIME_TO_ACQUIRE_LOCK)) {
 417                     VpnNexthop nexthop = getVpnNexthop(vpnId, primaryIpAddress);
 418                     LOG.trace("nexthop: {} retrieved for vpnId {}, prefix {}, ifName {} on dpn {}", nexthop, vpnId,
 419                             primaryIpAddress, ifName, dpnId);
 420                     if (nexthop == null) {
 421                         String encMacAddress = macAddress == null
 422                                 ? fibUtil.getMacAddressFromPrefix(ifName, vpnName, primaryIpAddress) : macAddress;
 423                         List<ActionInfo> listActionInfo = new ArrayList<>();
 424                         int actionKey = 0;
 425                         // MAC re-write
 426                         if (encMacAddress != null) {
 427                             if (gwMacAddress != null) {
 428                                 LOG.trace("The Local NextHop Group Source Mac {} for VpnInterface {} on VPN {}",
 429                                         gwMacAddress, ifName, vpnId);
 430                                 listActionInfo.add(new ActionSetFieldEthernetSource(actionKey++,
 431                                         new MacAddress(gwMacAddress)));
 432                             }
 433                             listActionInfo.add(new ActionSetFieldEthernetDestination(actionKey++,
 434                                     new MacAddress(encMacAddress)));
 435                             // listActionInfo.add(0, new ActionPopMpls());
 436                         } else {
 437                             // FIXME: Log message here.
 438                             LOG.debug("mac address for new local nexthop is null");
 439                         }
 440                         List<ActionInfo> nhActionInfoList = getEgressActionsForInterface(ifName, actionKey, false,
 441                                 vpnId, currDestIpPrefix);
 442                         if (nhActionInfoList.isEmpty()) {
 443                             LOG.error("createLocalNextHop: Skipping, Empty list of egress actions received for "
 444                                     + "interface {} on dpn {} for vpn {} prefix {}", ifName, dpnId, vpnId,
 445                                     currDestIpPrefix);
 446                         }
 447                         listActionInfo.addAll(nhActionInfoList);
 448                         BucketInfo bucket = new BucketInfo(listActionInfo);
 449                         List<BucketInfo> listBucketInfo = new ArrayList<>();
 450                         listBucketInfo.add(bucket);
 451                         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, groupId, primaryIpAddress,
 452                                 GroupTypes.GroupAll, listBucketInfo);
 453                         LOG.trace("Install LNH Group: id {}, mac address {}, interface {} for prefix {}", groupId,
 454                                 encMacAddress, ifName, primaryIpAddress);
 455                         //Try to install group directly on the DPN bypassing the FRM, in order to avoid waiting for the
 456                         // group to get installed before programming the flows
 457                         installGroupOnDpn(groupId, dpnId, primaryIpAddress, listBucketInfo,
 458                                 getNextHopKey(vpnId, primaryIpAddress), GroupTypes.GroupAll);
 459                         // install Group
 460                         mdsalApiManager.syncInstallGroup(groupEntity);
 461                         // update MD-SAL DS
 462                         addVpnNexthopToDS(dpnId, vpnId, primaryIpAddress, currDestIpPrefix, groupId);
 463
 464                     } else {
 465                         // Ignore adding new prefix , if it already exists
 466                         List<IpAdjacencies> prefixesList = nexthop.getIpAdjacencies();
 467                         IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currDestIpPrefix).build();
 468                         if (prefixesList != null && prefixesList.contains(prefix)) {
 469                             LOG.trace("Prefix {} is already present in l3nextHop {} ", currDestIpPrefix, nexthop);
 470                         } else {
 471                             IpAdjacenciesBuilder ipPrefixesBuilder =
 472                                     new IpAdjacenciesBuilder().withKey(new IpAdjacenciesKey(currDestIpPrefix));
 473                             LOG.trace("Updating prefix {} to vpnNextHop {} Operational DS", currDestIpPrefix, nexthop);
 474                             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
 475                                     getVpnNextHopIpPrefixIdentifier(vpnId, primaryIpAddress, currDestIpPrefix),
 476                                     ipPrefixesBuilder.build());
 477                         }
 478                     }
 479                 }
 480             } finally {
 481                 FibUtil.unlockCluster(lockManager, nextHopLockStr);
 482             }
 483             return Collections.emptyList();
 484         });
 485         return groupId;
 486     }
 487
 488     private void installGroupOnDpn(long groupId, BigInteger dpnId, String groupName, List<BucketInfo> bucketsInfo,
 489                                      String nextHopKey, GroupTypes groupType) {
 490         NodeRef nodeRef = FibUtil.buildNodeRef(dpnId);
 491         Buckets buckets = FibUtil.buildBuckets(bucketsInfo);
 492         GroupRef groupRef = new GroupRef(FibUtil.buildGroupInstanceIdentifier(groupId, dpnId));
 493         AddGroupInput input = new AddGroupInputBuilder().setNode(nodeRef).setGroupId(new GroupId(groupId))
 494                 .setBuckets(buckets).setGroupRef(groupRef).setGroupType(groupType)
 495                 .setGroupName(groupName).build();
 496         Future<RpcResult<AddGroupOutput>> groupStats = salGroupService.addGroup(input);
 497         RpcResult<AddGroupOutput> rpcResult = null;
 498         try {
 499             rpcResult = groupStats.get();
 500             if (rpcResult != null && rpcResult.isSuccessful()) {
 501                 LOG.info("Group {} with key {} has been successfully installed directly on dpn {}.", groupId,
 502                         nextHopKey, dpnId);
 503             } else {
 504                 LOG.error("Unable to install group {} with key {} directly on dpn {} due to {}.", groupId, nextHopKey,
 505                         dpnId, rpcResult != null ? rpcResult.getErrors() : null);
 506             }
 507         } catch (InterruptedException | ExecutionException e) {
 508             LOG.error("Error while installing group {} directly on dpn {}", groupId, dpnId);
 509         }
 510     }
 511
 512     protected void addVpnNexthopToDS(BigInteger dpnId, long vpnId, String primaryIpAddr,
 513                                      String currIpAddr, long egressPointer) {
 514         InstanceIdentifierBuilder<VpnNexthops> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
 515             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId));
 516
 517         List<IpAdjacencies> ipPrefixesList = new ArrayList<>();
 518         IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currIpAddr).build();
 519         ipPrefixesList.add(prefix);
 520         // Add nexthop to vpn node
 521         VpnNexthop nh = new VpnNexthopBuilder()
 522             .withKey(new VpnNexthopKey(primaryIpAddr))
 523             .setDpnId(dpnId)
 524             .setIpAdjacencies(ipPrefixesList)
 525             .setEgressPointer(egressPointer).build();
 526
 527         InstanceIdentifier<VpnNexthop> id1 = idBuilder
 528             .child(VpnNexthop.class, new VpnNexthopKey(primaryIpAddr)).build();
 529         LOG.trace("Adding vpnnextHop {} to Operational DS", nh);
 530         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id1, nh);
 531
 532     }
 533
 534     protected InstanceIdentifier<IpAdjacencies> getVpnNextHopIpPrefixIdentifier(long vpnId, String primaryIpAddress,
 535                                                                                 String ipPrefix) {
 536         InstanceIdentifier<IpAdjacencies> id = InstanceIdentifier.builder(L3nexthop.class)
 537                 .child(VpnNexthops.class, new VpnNexthopsKey(vpnId))
 538                 .child(VpnNexthop.class, new VpnNexthopKey(primaryIpAddress))
 539                 .child(IpAdjacencies.class, new IpAdjacenciesKey(ipPrefix)).build();
 540         return id;
 541     }
 542
 543     @Nullable
 544     protected VpnNexthop getVpnNexthop(long vpnId, String ipAddress) {
 545
 546         // check if vpn node is there
 547         InstanceIdentifierBuilder<VpnNexthops> idBuilder =
 548             InstanceIdentifier.builder(L3nexthop.class).child(VpnNexthops.class,
 549                 new VpnNexthopsKey(vpnId));
 550         InstanceIdentifier<VpnNexthops> id = idBuilder.build();
 551         Optional<VpnNexthops> vpnNexthops = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 552         if (vpnNexthops.isPresent()) {
 553             // get nexthops list for vpn
 554             List<VpnNexthop> nexthops = vpnNexthops.get().nonnullVpnNexthop();
 555             for (VpnNexthop nexthop : nexthops) {
 556                 if (Objects.equals(nexthop.getIpAddress(), ipAddress)) {
 557                     // return nexthop
 558                     LOG.trace("VpnNextHop : {}", nexthop);
 559                     return nexthop;
 560                 }
 561             }
 562             // return null if not found
 563         }
 564         return null;
 565     }
 566
 567     @Nullable
 568     public AdjacencyResult getRemoteNextHopPointer(BigInteger remoteDpnId, long vpnId, String prefixIp,
 569             @Nullable String nextHopIp, Class<? extends TunnelTypeBase> tunnelType) {
 570         String egressIfName = null;
 571         LOG.trace("getRemoteNextHopPointer: input [remoteDpnId {}, vpnId {}, prefixIp {}, nextHopIp {} ]", remoteDpnId,
 572             vpnId, prefixIp, nextHopIp);
 573
 574         Class<? extends InterfaceType> egressIfType;
 575         ElanInstance elanInstance = getElanInstanceForPrefix(vpnId, prefixIp);
 576         if (elanInstance != null) {
 577             egressIfType = getInterfaceType(elanInstance);
 578         } else {
 579             LOG.warn("Failed to determine network type for prefixIp {} using tunnel", prefixIp);
 580             egressIfType = Tunnel.class;
 581         }
 582
 583         if (Tunnel.class.equals(egressIfType)) {
 584             egressIfName = getTunnelRemoteNextHopPointer(remoteDpnId, nextHopIp, tunnelType);
 585         } else {
 586             egressIfName = getExtPortRemoteNextHopPointer(remoteDpnId, elanInstance);
 587         }
 588
 589         LOG.trace("NextHop pointer for prefixIp {} vpnId {} dpnId {} is {}", prefixIp, vpnId, remoteDpnId,
 590             egressIfName);
 591         return egressIfName != null ? new AdjacencyResult(egressIfName, egressIfType, nextHopIp,
 592                 prefixIp) : null;
 593     }
 594
 595     private void removeVpnNexthopFromDS(long vpnId, String ipPrefix) {
 596
 597         InstanceIdentifierBuilder<VpnNexthop> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
 598             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId))
 599             .child(VpnNexthop.class, new VpnNexthopKey(ipPrefix));
 600         InstanceIdentifier<VpnNexthop> id = idBuilder.build();
 601         // remove from DS
 602         LOG.trace("Removing vpn next hop from datastore : {}", id);
 603         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 604     }
 605
 606     public void removeLocalNextHop(BigInteger dpnId, Long vpnId, String primaryIpAddress, String currDestIpPrefix) {
 607         String nextHopLockStr = vpnId + primaryIpAddress;
 608         try {
 609             if (FibUtil.lockCluster(lockManager, nextHopLockStr, WAIT_TIME_TO_ACQUIRE_LOCK)) {
 610                 VpnNexthop nh = getVpnNexthop(vpnId, primaryIpAddress);
 611                 if (nh != null) {
 612                     List<IpAdjacencies> prefixesList = new ArrayList<>(nh.nonnullIpAdjacencies());
 613                     IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currDestIpPrefix).build();
 614                     prefixesList.remove(prefix);
 615                     if (prefixesList.isEmpty()) { //remove the group only if there are no more flows using this group
 616                         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, nh.getEgressPointer(),
 617                                 primaryIpAddress, GroupTypes.GroupAll, Collections.emptyList());
 618                         // remove Group ...
 619                         mdsalApiManager.removeGroup(groupEntity);
 620                         //update MD-SAL DS
 621                         removeVpnNexthopFromDS(vpnId, primaryIpAddress);
 622                         //release groupId
 623                         removeNextHopPointer(getNextHopKey(vpnId, primaryIpAddress));
 624                         LOG.debug("Local Next hop {} for {} {} on dpn {} successfully deleted",
 625                                 nh.getEgressPointer(), vpnId, primaryIpAddress, dpnId);
 626                     } else {
 627                         //remove the currIpPrefx from IpPrefixList of the vpnNexthop
 628                         LOG.trace("Removing the prefix {} from vpnNextHop {} Operational DS", currDestIpPrefix, nh);
 629                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
 630                                 getVpnNextHopIpPrefixIdentifier(vpnId, primaryIpAddress, currDestIpPrefix));
 631                     }
 632                 } else {
 633                     //throw error
 634                     LOG.error("Local NextHop for VpnId {} curIpPrefix {} on dpn {} primaryIpAddress {} not deleted",
 635                             vpnId, currDestIpPrefix, dpnId, primaryIpAddress);
 636                 }
 637             }
 638         } finally {
 639             FibUtil.unlockCluster(lockManager, nextHopLockStr);
 640         }
 641     }
 642
 643     public void setConfTransType(String service, String transportType) {
 644
 645         if (!service.equalsIgnoreCase("L3VPN")) {
 646             LOG.error("Incorrect service {} provided for setting the transport type.", service);
 647             return;
 648         }
 649
 650         L3VPNTransportTypes transType = L3VPNTransportTypes.validateTransportType(transportType
 651                 .toUpperCase(Locale.getDefault()));
 652
 653         if (transType != L3VPNTransportTypes.Invalid) {
 654             configuredTransportTypeL3VPN = transType;
 655         }
 656     }
 657
 658     public void writeConfTransTypeConfigDS() {
 659         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, getConfTransportTypeIdentifier(),
 660             createConfTransportType(configuredTransportTypeL3VPN.getTransportType()));
 661     }
 662
 663     public L3VPNTransportTypes getConfiguredTransportTypeL3VPN() {
 664         return this.configuredTransportTypeL3VPN;
 665     }
 666
 667     public String getReqTransType() {
 668         if (configuredTransportTypeL3VPN == L3VPNTransportTypes.Invalid) {
 669             /*
 670              * Restart scenario, Read from the ConfigDS.
 671              * if the value is Unset, cache value as VxLAN.
 672              */
 673             LOG.trace("configureTransportType is not yet set.");
 674             Optional<ConfTransportTypeL3vpn> configuredTransTypeFromConfig =
 675                 MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, getConfTransportTypeIdentifier());
 676
 677             if (configuredTransTypeFromConfig.isPresent()) {
 678                 if (TunnelTypeGre.class.equals(configuredTransTypeFromConfig.get().getTransportType())) {
 679                     configuredTransportTypeL3VPN = L3VPNTransportTypes.GRE;
 680                 } else {
 681                     configuredTransportTypeL3VPN = L3VPNTransportTypes.VxLAN;
 682                 }
 683                 LOG.trace("configuredTransportType set from config DS to {}",
 684                     getConfiguredTransportTypeL3VPN().getTransportType());
 685             } else {
 686                 setConfTransType("L3VPN", L3VPNTransportTypes.VxLAN.getTransportType());
 687                 LOG.trace("configuredTransportType is not set in the Config DS. VxLAN as default will be used.");
 688             }
 689         } else {
 690             LOG.trace("configuredTransportType is set as {}", getConfiguredTransportTypeL3VPN().getTransportType());
 691         }
 692         return getConfiguredTransportTypeL3VPN().getTransportType();
 693     }
 694
 695     public InstanceIdentifier<ConfTransportTypeL3vpn> getConfTransportTypeIdentifier() {
 696         return InstanceIdentifier.builder(ConfTransportTypeL3vpn.class).build();
 697     }
 698
 699     private ConfTransportTypeL3vpn createConfTransportType(String type) {
 700         ConfTransportTypeL3vpn confTransType;
 701         switch (type) {
 702             case ITMConstants.TUNNEL_TYPE_GRE:
 703                 confTransType = new ConfTransportTypeL3vpnBuilder().setTransportType(TunnelTypeGre.class).build();
 704                 LOG.trace("Setting the confTransportType to GRE.");
 705                 break;
 706             case ITMConstants.TUNNEL_TYPE_VXLAN:
 707                 confTransType = new ConfTransportTypeL3vpnBuilder().setTransportType(TunnelTypeVxlan.class).build();
 708                 LOG.trace("Setting the confTransportType to VxLAN.");
 709                 break;
 710             default:
 711                 LOG.trace("Invalid transport type {} passed to Config DS ", type);
 712                 confTransType = null;
 713                 break;
 714         }
 715         return confTransType;
 716     }
 717
 718     public Class<? extends TunnelTypeBase> getReqTunType(String transportType) {
 719         switch (transportType) {
 720             case "VXLAN":
 721                 return TunnelTypeVxlan.class;
 722             case "GRE":
 723                 return TunnelTypeGre.class;
 724             default:
 725                 return TunnelTypeMplsOverGre.class;
 726         }
 727     }
 728
 729     public String getTransportTypeStr(String tunType) {
 730         if (tunType.equals(TunnelTypeVxlan.class.toString())) {
 731             return ITMConstants.TUNNEL_TYPE_VXLAN;
 732         } else if (tunType.equals(TunnelTypeGre.class.toString())) {
 733             return ITMConstants.TUNNEL_TYPE_GRE;
 734         } else if (tunType.equals(TunnelTypeMplsOverGre.class.toString())) {
 735             return ITMConstants.TUNNEL_TYPE_MPLSoGRE;
 736         } else {
 737             return ITMConstants.TUNNEL_TYPE_INVALID;
 738         }
 739     }
 740
 741     @Override
 742     @PreDestroy
 743     public void close() {
 744         LOG.info("{} close", getClass().getSimpleName());
 745     }
 746
 747     // TODO Clean up the exception handling
 748     @SuppressWarnings("checkstyle:IllegalCatch")
 749     @Nullable
 750     private String getTunnelRemoteNextHopPointer(BigInteger remoteDpnId, String nextHopIp,
 751                                                  Class<? extends TunnelTypeBase> tunnelType) {
 752         if (nextHopIp != null && !nextHopIp.isEmpty()) {
 753             try {
 754                 // here use the config for tunnel type param
 755                 return getTunnelInterfaceName(remoteDpnId,
 756                     org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder
 757                         .getDefaultInstance(nextHopIp), tunnelType);
 758             } catch (Exception ex) {
 759                 LOG.error("Error while retrieving nexthop pointer for nexthop {} remoteDpn {}",
 760                         nextHopIp, remoteDpnId, ex);
 761             }
 762         }
 763
 764         return null;
 765     }
 766
 767     private String getExtPortRemoteNextHopPointer(BigInteger remoteDpnId, ElanInstance elanInstance) {
 768         return elanService.getExternalElanInterface(elanInstance.getElanInstanceName(), remoteDpnId);
 769     }
 770
 771     /**
 772      * Get the interface type associated with the type of ELAN used for routing
 773      * traffic to/from remote compute nodes.
 774      *
 775      * @param elanInstance The elan instance
 776      * @return L2vlan for flat/VLAN network type and Tunnel otherwise
 777      */
 778     private Class<? extends InterfaceType> getInterfaceType(ElanInstance elanInstance) {
 779         Class<? extends SegmentTypeBase> segmentType = elanInstance.getSegmentType();
 780         if (SegmentTypeFlat.class.equals(segmentType) || SegmentTypeVlan.class.equals(segmentType)) {
 781             return L2vlan.class;
 782         }
 783
 784         return Tunnel.class;
 785     }
 786
 787     private ElanInstance getElanInstanceForPrefix(long vpnId, String prefixIp) {
 788         ElanInstance elanInstance = null;
 789         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, prefixIp);
 790         if (prefix != null) {
 791             if (prefix.getNetworkId() != null) {
 792                 elanInstance = elanService.getElanInstance(prefix.getNetworkId().getValue());
 793             }
 794         }
 795
 796         return elanInstance;
 797     }
 798
 799     static class AdjacencyResult {
 800         private final String interfaceName;
 801         private final Class<? extends InterfaceType> interfaceType;
 802         private final String nextHopIp;
 803         private final String prefix;
 804
 805         AdjacencyResult(String interfaceName, Class<? extends InterfaceType> interfaceType, String nextHopIp,
 806                         String prefix) {
 807             this.interfaceName = interfaceName;
 808             this.interfaceType = interfaceType;
 809             this.nextHopIp = nextHopIp;
 810             this.prefix = prefix;
 811         }
 812
 813         public String getInterfaceName() {
 814             return interfaceName;
 815         }
 816
 817         public Class<? extends InterfaceType> getInterfaceType() {
 818             return interfaceType;
 819         }
 820
 821         public String getNextHopIp() {
 822             return nextHopIp;
 823         }
 824
 825         public String getPrefix() {
 826             return prefix;
 827         }
 828
 829         @Override
 830         public int hashCode() {
 831             final int prime = 31;
 832             int result = 1;
 833             result = prime * result + (interfaceName == null ? 0 : interfaceName.hashCode());
 834             return result;
 835         }
 836
 837         @Override
 838         public boolean equals(Object obj) {
 839             if (obj == null) {
 840                 return false;
 841             }
 842
 843             if (getClass() != obj.getClass()) {
 844                 return false;
 845             } else {
 846                 AdjacencyResult other = (AdjacencyResult) obj;
 847                 return interfaceName.equals(other.interfaceName);
 848             }
 849         }
 850     }
 851
 852     protected long setupLoadBalancingNextHop(Long parentVpnId, BigInteger dpnId,
 853             String destPrefix, List<BucketInfo> localBucketInfo, List<BucketInfo> remoteBucketInfo) {
 854         long remoteGroupId = createNextHopPointer(getRemoteSelectGroupKey(parentVpnId, destPrefix));
 855         if (remoteGroupId == FibConstants.INVALID_GROUP_ID) {
 856             LOG.error("Unable to allocate/retrieve remote groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 857             return remoteGroupId;
 858         }
 859         long localGroupId =  createNextHopPointer(getLocalSelectGroupKey(parentVpnId, destPrefix));
 860         if (localGroupId == FibConstants.INVALID_GROUP_ID) {
 861             LOG.error("Unable to allocate/retrieve local groupId for vpnId {} , prefix {}",
 862                 parentVpnId, destPrefix);
 863             return remoteGroupId;
 864         }
 865         List<BucketInfo> combinedBucketInfo = new ArrayList<>();
 866         combinedBucketInfo.addAll(localBucketInfo);
 867         combinedBucketInfo.addAll(remoteBucketInfo);
 868         GroupEntity remoteGroupEntity = MDSALUtil.buildGroupEntity(
 869                 dpnId, remoteGroupId, destPrefix, GroupTypes.GroupSelect, combinedBucketInfo);
 870         GroupEntity localGroupEntity = MDSALUtil.buildGroupEntity(
 871                 dpnId, localGroupId, destPrefix, GroupTypes.GroupSelect, localBucketInfo);
 872         String jobKey = FibUtil.getCreateLocalNextHopJobKey(parentVpnId, dpnId, destPrefix);
 873         jobCoordinator.enqueueJob(jobKey, () -> {
 874             mdsalApiManager.syncInstallGroup(remoteGroupEntity);
 875             if (!localBucketInfo.isEmpty()) {
 876                 mdsalApiManager.syncInstallGroup(localGroupEntity);
 877             }
 878             if (LOG.isDebugEnabled()) {
 879                 LOG.debug("Finished installing GroupEntity with jobCoordinator key {} remoteGroupEntity.groupId {}"
 880                         + "localGroupEntity.groupId {}  groupEntity.groupType {}", jobKey,
 881                         remoteGroupEntity.getGroupId(), localGroupEntity.getGroupId(),
 882                         remoteGroupEntity.getGroupType());
 883             }
 884             // Delete local group(if exists) if there is no local info.
 885             // Local group has to be deleted if all VMs in a compute is deleted.
 886             if (localBucketInfo.isEmpty()) {
 887                 LOG.debug("Deleting local group {} since no local nhs present for "
 888                         + "prefix {}", localGroupEntity.getGroupId(), destPrefix);
 889                 mdsalApiManager.syncRemoveGroup(localGroupEntity);
 890             }
 891             return Collections.emptyList();
 892         });
 893         return remoteGroupId;
 894     }
 895
 896     protected void deleteLoadBalancingNextHop(Long parentVpnId, BigInteger dpnId, String destPrefix) {
 897         long remoteGroupId = createNextHopPointer(getRemoteSelectGroupKey(parentVpnId, destPrefix));
 898         if (remoteGroupId == FibConstants.INVALID_GROUP_ID) {
 899             LOG.error("Unable to allocate/retrieve remote groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 900         }
 901         long localGroupId = createNextHopPointer(getLocalSelectGroupKey(parentVpnId, destPrefix));
 902         if (localGroupId == FibConstants.INVALID_GROUP_ID) {
 903             LOG.error("Unable to allocate/retrieve local groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 904         }
 905         GroupEntity remoteGroupEntity = MDSALUtil.buildGroupEntity(
 906                 dpnId, remoteGroupId, destPrefix, GroupTypes.GroupSelect, Collections.emptyList());
 907         GroupEntity localGroupEntity = MDSALUtil.buildGroupEntity(
 908                 dpnId, localGroupId, destPrefix, GroupTypes.GroupSelect, Collections.emptyList());
 909         String jobKey = FibUtil.getCreateLocalNextHopJobKey(parentVpnId, dpnId, destPrefix);
 910         jobCoordinator.enqueueJob(jobKey, () -> {
 911             mdsalApiManager.syncRemoveGroup(remoteGroupEntity);
 912             mdsalApiManager.syncRemoveGroup(localGroupEntity);
 913             if (LOG.isDebugEnabled()) {
 914                 LOG.debug("Finished removing GroupEntity with jobCoordinator key {} remoteGroupEntity.groupId {}"
 915                     + "localGroupEntity.groupId {}", jobKey, remoteGroupId, localGroupId);
 916             }
 917             return Collections.emptyList();
 918         });
 919     }
 920
 921     long createNextHopGroups(Long vpnId, String rd, BigInteger dpnId, VrfEntry vrfEntry,
 922             @Nullable Routes routes, List<Routes> vpnExtraRoutes) {
 923         List<BucketInfo> localBucketInfo = new ArrayList<>();
 924         List<Routes> clonedVpnExtraRoutes  = new ArrayList<>(vpnExtraRoutes);
 925         if (clonedVpnExtraRoutes.contains(routes)) {
 926             localBucketInfo.addAll(getBucketsForLocalNexthop(vpnId, dpnId, vrfEntry, routes));
 927             clonedVpnExtraRoutes.remove(routes);
 928         }
 929         List<BucketInfo> remoteBucketInfo =
 930             new ArrayList<>(getBucketsForRemoteNexthop(vpnId, dpnId, vrfEntry, rd, clonedVpnExtraRoutes));
 931         return setupLoadBalancingNextHop(vpnId, dpnId,
 932             vrfEntry.getDestPrefix(), localBucketInfo, remoteBucketInfo);
 933     }
 934
 935     private List<BucketInfo> getBucketsForLocalNexthop(Long vpnId, BigInteger dpnId,
 936             VrfEntry vrfEntry, Routes routes) {
 937         @Nullable List<String> nexthopIpList = routes.getNexthopIpList();
 938         if (LOG.isDebugEnabled()) {
 939             LOG.debug("NexthopManager.getBucketsForLocalNexthop invoked with vpnId {} dpnId {} "
 940                             + " vrfEntry.routePaths {}, routes.nexthopList {}", vpnId, dpnId, vrfEntry.getRoutePaths(),
 941                 nexthopIpList);
 942         }
 943         List<BucketInfo> listBucketInfo = new CopyOnWriteArrayList<>();
 944         if (nexthopIpList != null) {
 945             nexthopIpList.parallelStream().forEach(nextHopIp -> {
 946                 String localNextHopIP;
 947                 if (isIpv4Address(nextHopIp)) {
 948                     localNextHopIP = nextHopIp + NwConstants.IPV4PREFIX;
 949                 } else {
 950                     localNextHopIP = nextHopIp + NwConstants.IPV6PREFIX;
 951                 }
 952                 Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 953                 if (localNextHopInfo != null) {
 954                     long groupId = getLocalNextHopGroup(vpnId, localNextHopIP);
 955                     if (groupId == FibConstants.INVALID_GROUP_ID) {
 956                         LOG.error("Unable to allocate groupId for vpnId {} , prefix {} , interface {}", vpnId,
 957                             vrfEntry.getDestPrefix(), localNextHopInfo.getVpnInterfaceName());
 958                         return;
 959                     }
 960                     List<ActionInfo> actionsInfos =
 961                         Collections.singletonList(new ActionGroup(groupId));
 962                     BucketInfo bucket = new BucketInfo(actionsInfos);
 963                     bucket.setWeight(1);
 964                     listBucketInfo.add(bucket);
 965                 }
 966             });
 967         }
 968         LOG.trace("LOCAL: listbucket {}, vpnId {}, dpnId {}, routes {}", listBucketInfo, vpnId, dpnId, routes);
 969         return listBucketInfo;
 970     }
 971
 972     private List<BucketInfo> getBucketsForRemoteNexthop(Long vpnId, BigInteger dpnId, VrfEntry vrfEntry, String rd,
 973             List<Routes> vpnExtraRoutes) {
 974         List<BucketInfo> listBucketInfo = new ArrayList<>();
 975         Map<String, List<ActionInfo>> egressActionMap = new HashMap<>();
 976         vpnExtraRoutes.stream().filter(vpnExtraRoute -> vpnExtraRoute.getNexthopIpList() != null).forEach(
 977             vpnExtraRoute -> vpnExtraRoute.getNexthopIpList().forEach(nextHopIp -> {
 978                 String nextHopPrefixIp;
 979                 if (isIpv4Address(nextHopIp)) {
 980                     nextHopPrefixIp = nextHopIp + NwConstants.IPV4PREFIX;
 981                 } else {
 982                     nextHopPrefixIp = nextHopIp + NwConstants.IPV6PREFIX;
 983                 }
 984                 List<String> tepIpAddresses = fibUtil.getNextHopAddresses(rd, nextHopPrefixIp);
 985                 if (tepIpAddresses.isEmpty()) {
 986                     return;
 987                 }
 988                 // There would be only one nexthop address for a VM ip which would be the tep Ip
 989                 String tepIp = tepIpAddresses.get(0);
 990                 AdjacencyResult adjacencyResult = getRemoteNextHopPointer(dpnId, vpnId,
 991                     vrfEntry.getDestPrefix(), tepIp, TunnelTypeVxlan.class);
 992                 if (adjacencyResult == null) {
 993                     return;
 994                 }
 995                 String egressInterface = adjacencyResult.getInterfaceName();
 996                 if (!FibUtil.isTunnelInterface(adjacencyResult)) {
 997                     return;
 998                 }
 999                 Class<? extends TunnelTypeBase> tunnelType =
1000                     VpnExtraRouteHelper.getTunnelType(itmManager, egressInterface);
1001                 StateTunnelList ifState = null;
1002                 try {
1003                     ifState = fibUtil.getTunnelState(egressInterface);
1004                     if (ifState == null || ifState.getOperState() != TunnelOperStatus.Up) {
1005                         LOG.trace("Tunnel is not up for interface {}", egressInterface);
1006                         return;
1007                     }
1008                 } catch (ReadFailedException e) {
1009                     LOG.error("getBucketsForRemoteNexthop: error in fetching tunnel state for interface {}",
1010                         egressInterface, e);
1011                     return;
1012                 }
1013                 if (!TunnelTypeVxlan.class.equals(tunnelType)) {
1014                     return;
1015                 }
1016                 Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).get();
1017                 Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, nextHopPrefixIp);
1018                 if (prefixInfo == null) {
1019                     LOG.error("No prefix info found for prefix {} in rd {} for VPN {}", nextHopPrefixIp, rd,
1020                         vpnId);
1021                     return;
1022                 }
1023                 BigInteger tunnelId;
1024                 if (FibUtil.isVxlanNetwork(prefixInfo.getNetworkType())) {
1025                     tunnelId = BigInteger.valueOf(prefixInfo.getSegmentationId());
1026                 } else {
1027                     LOG.warn("Network is not of type VXLAN for prefix {}."
1028                         + "Going with default Lport Tag.", prefixInfo.toString());
1029                     tunnelId = BigInteger.valueOf(label);
1030                 }
1031                 List<ActionInfo> actionInfos = new ArrayList<>();
1032                 actionInfos.add(new ActionSetFieldTunnelId(tunnelId));
1033                 String ifName = prefixInfo.getVpnInterfaceName();
1034                 String vpnName = fibUtil.getVpnNameFromId(vpnId);
1035                 if (vpnName == null) {
1036                     return;
1037                 }
1038                 String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, nextHopPrefixIp);
1039                 actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
1040                     new MacAddress(macAddress)));
1041                 List<ActionInfo> egressActions;
1042                 if (egressActionMap.containsKey(egressInterface)) {
1043                     egressActions = egressActionMap.get(egressInterface);
1044                 } else {
1045                     egressActions = getEgressActionsForInterface(egressInterface, actionInfos.size(),
1046                             true, vpnId, vrfEntry.getDestPrefix());
1047                     if (egressActions.isEmpty()) {
1048                         LOG.error("Skipping getBucketsForRemoteNexthop: Empty list of egress actions received for "
1049                                         + "interface {} on dpn {} for vpn {} prefix {} nextHop {}", ifName, dpnId,
1050                                 vpnId, vrfEntry.getDestPrefix(), nextHopPrefixIp);
1051                     }
1052                     egressActionMap.put(egressInterface, egressActions);
1053                 }
1054                 if (egressActions.isEmpty()) {
1055                     LOG.error("Failed to retrieve egress action for prefix {} route-paths {}"
1056                             + " interface {}." + " Aborting remote FIB entry creation.",
1057                         vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), egressInterface);
1058                 }
1059                 actionInfos.addAll(egressActions);
1060                 BucketInfo bucket = new BucketInfo(actionInfos);
1061                 bucket.setWeight(1);
1062                 listBucketInfo.add(bucket);
1063             }));
1064         LOG.trace("LOCAL: listbucket {}, rd {}, dpnId {}, routes {}", listBucketInfo, rd, dpnId, vpnExtraRoutes);
1065         return listBucketInfo;
1066     }
1067
1068     public void createDcGwLoadBalancingGroup(BigInteger dpnId, String destinationIp,
1069                                              Class<? extends TunnelTypeBase> tunnelType) {
1070         jobCoordinator.enqueueJob(getJobKey(dpnId), () -> {
1071             List<ListenableFuture<Void>> futures = new ArrayList<>();
1072             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operationalTx -> {
1073                 synchronized (getDcGateWaySyncKey(destinationIp)) {
1074                     FibUtil.addL3vpnDcGateWay(destinationIp, operationalTx);
1075                 }
1076                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, configTx -> {
1077                     List<String> availableDcGws = getDcGwIps();
1078                     Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
1079                     int noOfDcGws = availableDcGws.size();
1080                     if (noOfDcGws == 1) {
1081                         LOG.trace("There are no enough DC GateWays {} present to program LB group", availableDcGws);
1082                         return;
1083                     }
1084                     if (availableDcGws.contains(destinationIp)) {
1085                         availableDcGws.remove(destinationIp);
1086                     }
1087                     availableDcGws.forEach(dcGwIp -> {
1088                         List<String> dcGws = Arrays.asList(dcGwIp, destinationIp);
1089                         Collections.sort(dcGws);
1090                         String groupIdKey = FibUtil.getGreLbGroupKey(dcGws);
1091                         Long groupId = createNextHopPointer(groupIdKey);
1092                         List<Bucket> listBucket = new ArrayList<>();
1093                         for (int index = 0; index < dcGws.size(); index++) {
1094                             if (isTunnelUp(dcGws.get(index), dpnId, tunnelType)) {
1095                                 listBucket.add(buildBucketForDcGwLbGroup(dcGws.get(index),
1096                                         dpnId, index, tunnelType, true));
1097                             }
1098                         }
1099                         Group group = MDSALUtil.buildGroup(groupId, groupIdKey, GroupTypes.GroupSelect,
1100                                         MDSALUtil.buildBucketLists(listBucket));
1101                         mdsalApiManager.addGroup(configTx, dpnId, group);
1102                         FibUtil.updateLbGroupInfo(dpnId, groupIdKey, groupId.toString(), operationalTx);
1103                         LOG.trace("LB group {} towards DC-GW installed on dpn {}. Group - {}",
1104                                 groupIdKey, dpnId, group);
1105                     });
1106                 }));
1107             }));
1108             return futures;
1109         }, RETRY_COUNT);
1110     }
1111
1112     private String getJobKey(BigInteger dpnId) {
1113         return new StringBuilder().append(NEXTHOPMANAGER_JOB_KEY_PREFIX).append(dpnId).toString();
1114     }
1115
1116     private String getDcGateWaySyncKey(String destinationIp) {
1117         String mutex = new StringBuilder().append("L3vpncDcGateWay").append(destinationIp).toString();
1118         return mutex.intern();
1119     }
1120
1121     private List<String> getDcGwIps() {
1122         InstanceIdentifier<DcGatewayIpList> dcGatewayIpListid =
1123                 InstanceIdentifier.builder(DcGatewayIpList.class).build();
1124         DcGatewayIpList dcGatewayIpListConfig =
1125                 MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, dcGatewayIpListid).orNull();
1126         if (dcGatewayIpListConfig == null) {
1127             return Collections.emptyList();
1128         }
1129         return dcGatewayIpListConfig.getDcGatewayIp()
1130                 .stream()
1131                 .filter(dcGwIp -> dcGwIp.getTunnnelType().equals(TunnelTypeMplsOverGre.class))
1132                 .map(dcGwIp -> dcGwIp.getIpAddress().stringValue()).sorted()
1133                 .collect(toList());
1134     }
1135
1136     private boolean isTunnelUp(String dcGwIp, BigInteger dpnId, Class<? extends TunnelTypeBase> tunnelType) {
1137         String tunnelName = getTunnelRemoteNextHopPointer(dpnId, dcGwIp, tunnelType);
1138         if (tunnelName != null) {
1139             InstanceIdentifier<StateTunnelList> tunnelStateId =
1140                     InstanceIdentifier.builder(TunnelsState.class).child(
1141                             StateTunnelList.class, new StateTunnelListKey(tunnelName)).build();
1142             return MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, tunnelStateId)
1143                     .toJavaUtil().map(StateTunnelList::getOperState)
1144                     .orElse(TunnelOperStatus.Down) == TunnelOperStatus.Up;
1145         }
1146         return false;
1147     }
1148
1149     private List<Action> getEgressActions(String interfaceName, int actionKey) {
1150         List<Action> actions = Collections.emptyList();
1151         try {
1152             GetEgressActionsForInterfaceInputBuilder egressAction =
1153                     new GetEgressActionsForInterfaceInputBuilder().setIntfName(interfaceName).setActionKey(actionKey);
1154             Future<RpcResult<GetEgressActionsForInterfaceOutput>> result =
1155                     odlInterfaceRpcService.getEgressActionsForInterface(egressAction.build());
1156             RpcResult<GetEgressActionsForInterfaceOutput> rpcResult = result.get();
1157             if (!rpcResult.isSuccessful()) {
1158                 LOG.warn("RPC Call to Get egress actions for interface {} returned with Errors {}",
1159                         interfaceName, rpcResult.getErrors());
1160             } else {
1161                 actions = rpcResult.getResult().nonnullAction();
1162             }
1163         } catch (InterruptedException | ExecutionException e) {
1164             LOG.warn("Exception when egress actions for interface {}", interfaceName, e);
1165         }
1166         return actions;
1167     }
1168
1169     /**
1170      * This method is invoked when the neighbor is removed from DS.
1171      * All the LB groups which point to the given destination will be deleted.
1172      */
1173     public void removeDcGwLoadBalancingGroup(BigInteger dpnId,
1174             String destinationIp) {
1175         jobCoordinator.enqueueJob(getJobKey(dpnId), () -> {
1176             List<String> availableDcGws = fibUtil.getL3VpnDcGateWays();
1177             if (availableDcGws.contains(destinationIp)) {
1178                 availableDcGws.remove(destinationIp);
1179             }
1180             List<ListenableFuture<Void>> futures = new ArrayList<>();
1181             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operationalTx -> {
1182                 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx -> {
1183                     availableDcGws.forEach(dcGwIp -> {
1184                         List<String> dcGws = Arrays.asList(dcGwIp, destinationIp);
1185                         Collections.sort(dcGws);
1186                         String nextHopKey = FibUtil.getGreLbGroupKey(dcGws);
1187                         Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
1188                         if (!optionalNextHops.isPresent()) {
1189                             return;
1190                         }
1191                         Nexthops nexthops = optionalNextHops.get();
1192                         final String groupId = nexthops.getGroupId();
1193                         final long groupIdValue = Long.parseLong(groupId);
1194                         Group group = MDSALUtil.buildGroup(groupIdValue, nextHopKey, GroupTypes.GroupSelect,
1195                                 MDSALUtil.buildBucketLists(Collections.emptyList()));
1196                         LOG.trace("Removed LB group {} on dpn {}", group, dpnId);
1197                         try {
1198                             mdsalApiManager.removeGroup(configTx, dpnId, group);
1199                         } catch (ExecutionException | InterruptedException e) {
1200                             LOG.error("Group removal failed for group {} with exception", groupId, e);
1201                         }
1202                         removeNextHopPointer(nextHopKey);
1203                         FibUtil.removeOrUpdateNextHopInfo(dpnId, nextHopKey, groupId, nexthops, operationalTx);
1204                     });
1205                     synchronized (getDcGateWaySyncKey(destinationIp)) {
1206                         FibUtil.removeL3vpnDcGateWay(destinationIp, operationalTx);
1207                     }
1208                 }));
1209             }));
1210             return futures;
1211         }, RETRY_COUNT);
1212     }
1213
1214     /**
1215      * This method is invoked when the tunnel status is deleted.
1216      * All the buckets which point to given destination will be marked down.
1217      */
1218     public void updateDcGwLoadBalancingGroup(BigInteger dpnId, String destinationIp,
1219             boolean isTunnelUp, Class<? extends TunnelTypeBase> tunnelType) {
1220         jobCoordinator.enqueueJob(getJobKey(dpnId), () -> {
1221             List<String> availableDcGws = fibUtil.getL3VpnDcGateWays();
1222             if (availableDcGws.contains(destinationIp)) {
1223                 availableDcGws.remove(destinationIp);
1224             }
1225             List<ListenableFuture<Void>> futures = new ArrayList<>();
1226             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx -> {
1227                 availableDcGws.forEach(dcGwIp -> {
1228                     List<String> dcGws = Arrays.asList(dcGwIp, destinationIp);
1229                     Collections.sort(dcGws);
1230                     String nextHopKey = FibUtil.getGreLbGroupKey(dcGws);
1231                     int bucketId = dcGws.indexOf(destinationIp);
1232                     Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
1233                     if (!optionalNextHops.isPresent()) {
1234                         return;
1235                     }
1236                     Nexthops nexthops = optionalNextHops.get();
1237                     final String groupId = nexthops.getGroupId();
1238                     final long groupIdValue = Long.parseLong(groupId);
1239                     Bucket bucket = buildBucketForDcGwLbGroup(destinationIp, dpnId, bucketId, tunnelType, isTunnelUp);
1240                     LOG.trace("updated bucket {} to group {} on dpn {}.", bucket, groupId, dpnId);
1241                     try {
1242                         mdsalApiManager.addBucket(configTx, dpnId, groupIdValue, bucket);
1243                     } catch (ExecutionException | InterruptedException e) {
1244                         LOG.error("Bucket addition failed for bucket {} with exception", bucketId, e);
1245                     }
1246                 });
1247             }));
1248             return futures;
1249         }, RETRY_COUNT);
1250     }
1251
1252     private Bucket buildBucketForDcGwLbGroup(String ipAddress, BigInteger dpnId,
1253             int index, Class<? extends TunnelTypeBase> tunnelType, boolean isTunnelUp) {
1254         List<Action> listAction = new ArrayList<>();
1255         // ActionKey 0 goes to mpls label.
1256         int actionKey = 1;
1257         listAction.add(new ActionPushMpls().buildAction());
1258         listAction.add(new ActionRegMove(actionKey++, FibConstants.NXM_REG_MAPPING
1259                 .get(index), 0, 19).buildAction());
1260         String tunnelInterfaceName = getTunnelInterfaceName(dpnId, IpAddressBuilder.getDefaultInstance(ipAddress),
1261             tunnelType);
1262         List<Action> egressActions = getEgressActions(tunnelInterfaceName, actionKey++);
1263         if (!egressActions.isEmpty()) {
1264             listAction.addAll(getEgressActions(tunnelInterfaceName, actionKey++));
1265         } else {
1266             // clear off actions if there is no egress actions.
1267             listAction = Collections.emptyList();
1268         }
1269         long watchPort = MDSALUtil.WATCH_PORT;
1270         if (!isTunnelUp) {
1271             watchPort = 0xFFFFFFFEL;
1272         }
1273         //OVS expects a non-zero weight value for load balancing to happen in select groups
1274         return MDSALUtil.buildBucket(listAction, SELECT_GROUP_WEIGHT, index,
1275                 watchPort, MDSALUtil.WATCH_GROUP);
1276     }
1277
1278     public void programDcGwLoadBalancingGroup(BigInteger dpnId, String destinationIp,
1279                                               int addRemoveOrUpdate, boolean isTunnelUp,
1280                                               Class<? extends TunnelTypeBase> tunnelType) {
1281         if (NwConstants.ADD_FLOW == addRemoveOrUpdate) {
1282             createDcGwLoadBalancingGroup(dpnId, destinationIp, tunnelType);
1283         } else if (NwConstants.DEL_FLOW == addRemoveOrUpdate) {
1284             removeDcGwLoadBalancingGroup(dpnId, destinationIp);
1285         } else if (NwConstants.MOD_FLOW == addRemoveOrUpdate) {
1286             updateDcGwLoadBalancingGroup(dpnId, destinationIp, isTunnelUp, tunnelType);
1287         }
1288     }
1289 }