fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/NexthopManager.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static java.util.stream.Collectors.toList;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  14
  15 import com.google.common.base.Optional;
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.util.concurrent.ListenableFuture;
  18
  19 import java.util.ArrayList;
  20 import java.util.Arrays;
  21 import java.util.Collections;
  22 import java.util.HashMap;
  23 import java.util.List;
  24 import java.util.Locale;
  25 import java.util.Map;
  26 import java.util.Objects;
  27 import java.util.concurrent.CopyOnWriteArrayList;
  28 import java.util.concurrent.ExecutionException;
  29 import java.util.concurrent.Future;
  30
  31 import javax.annotation.PreDestroy;
  32 import javax.inject.Inject;
  33 import javax.inject.Singleton;
  34
  35 import org.eclipse.jdt.annotation.Nullable;
  36 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  37 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  38 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  39 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  40 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  41 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  42 import org.opendaylight.genius.itm.globals.ITMConstants;
  43 import org.opendaylight.genius.mdsalutil.ActionInfo;
  44 import org.opendaylight.genius.mdsalutil.BucketInfo;
  45 import org.opendaylight.genius.mdsalutil.GroupEntity;
  46 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  47 import org.opendaylight.genius.mdsalutil.NwConstants;
  48 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  49 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  50 import org.opendaylight.genius.mdsalutil.actions.ActionOutput;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionPushVlan;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  54 import org.opendaylight.genius.mdsalutil.actions.ActionRegMove;
  55 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  56 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetSource;
  57 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  58 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldVlanVid;
  59 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  60 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  61 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  62 import org.opendaylight.netvirt.fibmanager.api.L3VPNTransportTypes;
  63 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  64 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev170119.L2vlan;
  65 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev170119.Tunnel;
  66 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
  67 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfaceType;
  68 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.OutputActionCase;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.PushVlanActionCase;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.SetFieldCase;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInput;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInputBuilder;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutput;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInput;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInputBuilder;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolOutput;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInput;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInputBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdOutput;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceInputBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceOutput;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelOperStatus;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelsState;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelListKey;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rev160406.DcGatewayIpList;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelInputBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelOutput;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameInputBuilder;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameOutput;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.lockmanager.rev160413.LockManagerService;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupInput;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupInputBuilder;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupOutput;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.SalGroupService;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupId;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupRef;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.Buckets;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeRef;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeBase;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeFlat;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeVlan;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.L3nexthop;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthops;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthopsKey;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopKey;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacencies;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacenciesBuilder;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacenciesKey;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpn;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpnBuilder;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.lb.nexthops.Nexthops;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.add.group.input.buckets.bucket.action.action.NxActionResubmitRpcAddGroupCase;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.nodes.node.table.flow.instructions.instruction.instruction.apply.actions._case.apply.actions.action.action.NxActionRegLoadNodesNodeTableFlowApplyActionsCase;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.nx.action.reg.load.grouping.NxRegLoad;
 137 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 138 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 139 import org.opendaylight.yangtools.yang.common.RpcResult;
 140 import org.opendaylight.yangtools.yang.common.Uint32;
 141 import org.opendaylight.yangtools.yang.common.Uint64;
 142 import org.slf4j.Logger;
 143 import org.slf4j.LoggerFactory;
 144
 145 @Singleton
 146 public class NexthopManager implements AutoCloseable {
 147     private static final Logger LOG = LoggerFactory.getLogger(NexthopManager.class);
 148     private static final String NEXTHOP_ID_POOL_NAME = "nextHopPointerPool";
 149     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 150     private static final long WAIT_TIME_TO_ACQUIRE_LOCK = 3000L;
 151     private static final int SELECT_GROUP_WEIGHT = 1;
 152     private static final int RETRY_COUNT = 6;
 153     private static final String NEXTHOPMANAGER_JOB_KEY_PREFIX = "NextHopManager";
 154
 155     private final DataBroker dataBroker;
 156     private final ManagedNewTransactionRunner txRunner;
 157     private final IMdsalApiManager mdsalApiManager;
 158     private final OdlInterfaceRpcService odlInterfaceRpcService;
 159     private final ItmRpcService itmManager;
 160     private final IdManagerService idManager;
 161     private final IElanService elanService;
 162     private final LockManagerService lockManager;
 163     private final SalGroupService salGroupService;
 164     private final JobCoordinator jobCoordinator;
 165     private final FibUtil fibUtil;
 166     private final IInterfaceManager interfaceManager;
 167     private volatile L3VPNTransportTypes configuredTransportTypeL3VPN = L3VPNTransportTypes.Invalid;
 168
 169     /**
 170      * Provides nexthop functions.
 171      * Creates group ID pool
 172      *
 173      * @param dataBroker       - dataBroker reference
 174      * @param mdsalApiManager  - mdsalApiManager reference
 175      * @param idManager        - idManager reference
 176      * @param odlInterfaceRpcService - odlInterfaceRpcService reference
 177      * @param itmManager       - itmManager reference
 178      */
 179     @Inject
 180     public NexthopManager(final DataBroker dataBroker,
 181                           final IMdsalApiManager mdsalApiManager,
 182                           final IdManagerService idManager,
 183                           final OdlInterfaceRpcService odlInterfaceRpcService,
 184                           final ItmRpcService itmManager,
 185                           final LockManagerService lockManager,
 186                           final IElanService elanService,
 187                           final SalGroupService salGroupService,
 188                           final JobCoordinator jobCoordinator,
 189                           final FibUtil fibUtil,
 190                           final IInterfaceManager interfaceManager) {
 191         this.dataBroker = dataBroker;
 192         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 193         this.mdsalApiManager = mdsalApiManager;
 194         this.idManager = idManager;
 195         this.odlInterfaceRpcService = odlInterfaceRpcService;
 196         this.itmManager = itmManager;
 197         this.elanService = elanService;
 198         this.salGroupService = salGroupService;
 199         this.jobCoordinator = jobCoordinator;
 200         this.fibUtil = fibUtil;
 201         this.lockManager = lockManager;
 202         this.interfaceManager = interfaceManager;
 203         createIdPool();
 204     }
 205
 206     private void createIdPool() {
 207         CreateIdPoolInput createPool = new CreateIdPoolInputBuilder()
 208             .setPoolName(NEXTHOP_ID_POOL_NAME)
 209             .setLow(150000L)
 210             .setHigh(175000L)
 211             .build();
 212         try {
 213             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPool);
 214             if (result != null && result.get().isSuccessful()) {
 215                 LOG.info("Created IdPool for NextHopPointerPool");
 216             }
 217         } catch (InterruptedException | ExecutionException e) {
 218             LOG.error("Failed to create idPool for NextHopPointerPool", e);
 219         }
 220     }
 221
 222     private String getNextHopKey(Uint32 vpnId, String ipAddress) {
 223         return "nexthop." + vpnId + ipAddress;
 224     }
 225
 226     String getRemoteSelectGroupKey(Uint32 vpnId, String ipAddress) {
 227         return "remote.ecmp.nexthop." + vpnId + ipAddress;
 228     }
 229
 230     String getLocalSelectGroupKey(Uint32 vpnId, String ipAddress) {
 231         return "local.ecmp.nexthop." + vpnId + ipAddress;
 232     }
 233
 234     public ItmRpcService getItmManager() {
 235         return itmManager;
 236     }
 237
 238     protected long createNextHopPointer(String nexthopKey) {
 239         AllocateIdInput getIdInput = new AllocateIdInputBuilder()
 240             .setPoolName(NEXTHOP_ID_POOL_NAME).setIdKey(nexthopKey)
 241             .build();
 242         //TODO: Proper error handling once IdManager code is complete
 243         try {
 244             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
 245             RpcResult<AllocateIdOutput> rpcResult = result.get();
 246             return rpcResult.getResult().getIdValue().toJava();
 247         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 248             // FIXME: NPEs should not be caught but rather their root cause should be eliminated
 249             LOG.trace("Failed to allocate {}", getIdInput, e);
 250         }
 251         return 0;
 252     }
 253
 254     protected void removeNextHopPointer(String nexthopKey) {
 255         ReleaseIdInput idInput = new ReleaseIdInputBuilder()
 256             .setPoolName(NEXTHOP_ID_POOL_NAME)
 257             .setIdKey(nexthopKey).build();
 258         try {
 259             RpcResult<ReleaseIdOutput> rpcResult = idManager.releaseId(idInput).get();
 260             if (!rpcResult.isSuccessful()) {
 261                 LOG.error("RPC Call to Get Unique Id for nexthopKey {} returned with Errors {}",
 262                         nexthopKey, rpcResult.getErrors());
 263             }
 264         } catch (InterruptedException | ExecutionException e) {
 265             LOG.warn("Exception when getting Unique Id for key {}", nexthopKey, e);
 266         }
 267     }
 268
 269     protected List<ActionInfo> getEgressActionsForInterface(final String ifName, int actionKey,
 270                                                             boolean isTunnelInterface,
 271                                                             Uint32 vpnId, String destIpPrefix) {
 272         List<Action> actions;
 273         try {
 274             if (isTunnelInterface && interfaceManager.isItmDirectTunnelsEnabled()) {
 275                 RpcResult<GetEgressActionsForTunnelOutput> rpcResult =
 276                         itmManager.getEgressActionsForTunnel(new GetEgressActionsForTunnelInputBuilder()
 277                                 .setIntfName(ifName).build()).get();
 278                 if (!rpcResult.isSuccessful()) {
 279                     LOG.error("RPC Call to Get egress tunnel actions for interface {} returned with Errors {}",
 280                             ifName, rpcResult.getErrors());
 281                     return Collections.emptyList();
 282                 } else {
 283                     actions = rpcResult.getResult().nonnullAction();
 284                 }
 285             } else {
 286                 RpcResult<GetEgressActionsForInterfaceOutput> rpcResult = odlInterfaceRpcService
 287                         .getEgressActionsForInterface(new GetEgressActionsForInterfaceInputBuilder()
 288                                 .setIntfName(ifName).build()).get();
 289                 if (!rpcResult.isSuccessful()) {
 290                     LOG.error("RPC Call to Get egress vm actions for interface {} vpnId {} ipPrefix {} returned with "
 291                                     + "Errors {}", ifName, vpnId, destIpPrefix, rpcResult.getErrors());
 292                     return Collections.emptyList();
 293                 } else {
 294                     actions = rpcResult.getResult().nonnullAction();
 295                 }
 296             }
 297             List<ActionInfo> listActionInfo = new ArrayList<>();
 298             for (Action action : actions) {
 299                 actionKey = action.key().getOrder() + actionKey;
 300                 org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action
 301                     actionClass = action.getAction();
 302                 if (actionClass instanceof OutputActionCase) {
 303                     listActionInfo.add(new ActionOutput(actionKey,
 304                         ((OutputActionCase) actionClass).getOutputAction().getOutputNodeConnector()));
 305                 } else if (actionClass instanceof PushVlanActionCase) {
 306                     listActionInfo.add(new ActionPushVlan(actionKey));
 307                 } else if (actionClass instanceof SetFieldCase) {
 308                     if (((SetFieldCase) actionClass).getSetField().getVlanMatch() != null) {
 309                         int vlanVid = ((SetFieldCase) actionClass).getSetField().getVlanMatch()
 310                             .getVlanId().getVlanId().getValue().toJava();
 311                         listActionInfo.add(new ActionSetFieldVlanVid(actionKey, vlanVid));
 312                     }
 313                 } else if (actionClass instanceof NxActionResubmitRpcAddGroupCase) {
 314                     Short tableId = ((NxActionResubmitRpcAddGroupCase) actionClass).getNxResubmit().getTable().toJava();
 315                     listActionInfo.add(new ActionNxResubmit(actionKey, tableId));
 316                 } else if (actionClass instanceof NxActionRegLoadNodesNodeTableFlowApplyActionsCase) {
 317                     NxRegLoad nxRegLoad =
 318                         ((NxActionRegLoadNodesNodeTableFlowApplyActionsCase) actionClass).getNxRegLoad();
 319                     listActionInfo.add(new ActionRegLoad(actionKey, NxmNxReg6.class,
 320                         nxRegLoad.getDst().getStart().toJava(), nxRegLoad.getDst().getEnd().toJava(),
 321                         nxRegLoad.getValue().longValue()));
 322                 }
 323             }
 324             return listActionInfo;
 325         } catch (InterruptedException | ExecutionException | NullPointerException e) {
 326             LOG.error("Exception when egress actions for interface {} isTunnel {} vpnId {} ipPrefix {}", ifName,
 327                     isTunnelInterface, vpnId, destIpPrefix, e);
 328         }
 329         LOG.warn("Exception when egress actions for interface {}", ifName);
 330         return Collections.emptyList();
 331     }
 332
 333     @Nullable
 334     protected String getTunnelInterfaceName(Uint64 srcDpId, Uint64 dstDpId) {
 335         Class<? extends TunnelTypeBase> tunType = getReqTunType(getReqTransType().toUpperCase(Locale.getDefault()));
 336         Future<RpcResult<GetTunnelInterfaceNameOutput>> result;
 337         try {
 338             result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 339                 .setSourceDpid(srcDpId)
 340                 .setDestinationDpid(dstDpId)
 341                 .setTunnelType(tunType)
 342                 .build());
 343             RpcResult<GetTunnelInterfaceNameOutput> rpcResult = result.get();
 344             if (!rpcResult.isSuccessful()) {
 345                 LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
 346             } else {
 347                 return rpcResult.getResult().getInterfaceName();
 348             }
 349         } catch (InterruptedException | ExecutionException e) {
 350             LOG.warn("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstDpId, e);
 351         }
 352         return null;
 353     }
 354
 355     @Nullable
 356     protected String getTunnelInterfaceName(Uint64 srcDpId, org.opendaylight.yang.gen.v1.urn.ietf.params
 357         .xml.ns.yang.ietf.inet.types.rev130715.IpAddress dstIp, Class<? extends TunnelTypeBase> tunnelType) {
 358         Future<RpcResult<GetInternalOrExternalInterfaceNameOutput>> result;
 359         try {
 360             LOG.debug("Trying to fetch tunnel interface name for source dpn {} destIp {} tunType {}", srcDpId,
 361                     dstIp.stringValue(), tunnelType.getName());
 362             result = itmManager.getInternalOrExternalInterfaceName(new GetInternalOrExternalInterfaceNameInputBuilder()
 363                 .setSourceDpid(srcDpId)
 364                 .setDestinationIp(dstIp)
 365                 .setTunnelType(tunnelType)
 366                 .build());
 367             RpcResult<GetInternalOrExternalInterfaceNameOutput> rpcResult = result.get();
 368             if (!rpcResult.isSuccessful()) {
 369                 LOG.warn("RPC Call to getTunnelInterfaceName returned with Errors {}", rpcResult.getErrors());
 370             } else {
 371                 return rpcResult.getResult().getInterfaceName();
 372             }
 373         } catch (InterruptedException | ExecutionException e) {
 374             LOG.error("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstIp, e);
 375         }
 376         return null;
 377     }
 378
 379
 380     public long getLocalNextHopGroup(Uint32 vpnId,
 381             String ipNextHopAddress) {
 382         long groupId = createNextHopPointer(getNextHopKey(vpnId, ipNextHopAddress));
 383         if (groupId == FibConstants.INVALID_GROUP_ID) {
 384             LOG.error("Unable to allocate groupId for vpnId {} , prefix {}", vpnId, ipNextHopAddress);
 385         }
 386         return groupId;
 387     }
 388
 389     public long getLocalSelectGroup(Uint32 vpnId,
 390             String ipNextHopAddress) {
 391         long groupId = createNextHopPointer(getLocalSelectGroupKey(vpnId, ipNextHopAddress));
 392         if (groupId == FibConstants.INVALID_GROUP_ID) {
 393             LOG.error("Unable to allocate groupId for vpnId {} , prefix {}", vpnId, ipNextHopAddress);
 394         }
 395         return groupId;
 396     }
 397
 398     public long createLocalNextHop(Uint32 vpnId, Uint64 dpnId, String ifName,
 399                                    String primaryIpAddress, String currDestIpPrefix,
 400                                    String gwMacAddress, Uint32 parentVpnId) {
 401         //For VPN Imported routes, getting VPN Instance name using parentVpnId
 402         String vpnName = parentVpnId != null ? fibUtil.getVpnNameFromId(parentVpnId) : fibUtil.getVpnNameFromId(vpnId);
 403         if (vpnName == null) {
 404             return 0;
 405         }
 406         String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, primaryIpAddress);
 407
 408         long groupId = createNextHopPointer(getNextHopKey(vpnId, primaryIpAddress));
 409         if (groupId == 0) {
 410             LOG.error("Unable to allocate groupId for vpnId {} , IntfName {}, primaryIpAddress {} curIpPrefix {}",
 411                     vpnId, ifName, primaryIpAddress, currDestIpPrefix);
 412             return groupId;
 413         }
 414         String nextHopLockStr = vpnId + primaryIpAddress;
 415         String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, currDestIpPrefix);
 416         jobCoordinator.enqueueJob(jobKey, () -> {
 417             try {
 418                 if (FibUtil.lockCluster(lockManager, nextHopLockStr, WAIT_TIME_TO_ACQUIRE_LOCK)) {
 419                     VpnNexthop nexthop = getVpnNexthop(vpnId, primaryIpAddress);
 420                     LOG.trace("nexthop: {} retrieved for vpnId {}, prefix {}, ifName {} on dpn {}", nexthop, vpnId,
 421                             primaryIpAddress, ifName, dpnId);
 422                     if (nexthop == null) {
 423                         String encMacAddress = macAddress == null
 424                                 ? fibUtil.getMacAddressFromPrefix(ifName, vpnName, primaryIpAddress) : macAddress;
 425                         List<ActionInfo> listActionInfo = new ArrayList<>();
 426                         int actionKey = 0;
 427                         // MAC re-write
 428                         if (encMacAddress != null) {
 429                             if (gwMacAddress != null) {
 430                                 LOG.trace("The Local NextHop Group Source Mac {} for VpnInterface {} on VPN {}",
 431                                         gwMacAddress, ifName, vpnId);
 432                                 listActionInfo.add(new ActionSetFieldEthernetSource(actionKey++,
 433                                         new MacAddress(gwMacAddress)));
 434                             }
 435                             listActionInfo.add(new ActionSetFieldEthernetDestination(actionKey++,
 436                                     new MacAddress(encMacAddress)));
 437                             // listActionInfo.add(0, new ActionPopMpls());
 438                         } else {
 439                             LOG.error("mac address for new local nexthop group {} is null for vpnId {}, prefix {}, "
 440                                     + "ifName {} on dpn {}", groupId, vpnId, primaryIpAddress, ifName, dpnId);
 441                         }
 442                         List<ActionInfo> nhActionInfoList = getEgressActionsForInterface(ifName, actionKey, false,
 443                                 vpnId, currDestIpPrefix);
 444                         if (nhActionInfoList.isEmpty()) {
 445                             LOG.error("createLocalNextHop: Skipping, Empty list of egress actions received for "
 446                                     + "interface {} on dpn {} for vpn {} prefix {}", ifName, dpnId, vpnId,
 447                                     currDestIpPrefix);
 448                         }
 449                         listActionInfo.addAll(nhActionInfoList);
 450                         BucketInfo bucket = new BucketInfo(listActionInfo);
 451                         List<BucketInfo> listBucketInfo = new ArrayList<>();
 452                         listBucketInfo.add(bucket);
 453                         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, groupId, primaryIpAddress,
 454                                 GroupTypes.GroupAll, listBucketInfo);
 455                         LOG.trace("Install LNH Group: id {}, mac address {}, interface {} for prefix {}", groupId,
 456                                 encMacAddress, ifName, primaryIpAddress);
 457                         //Try to install group directly on the DPN bypassing the FRM, in order to avoid waiting for the
 458                         // group to get installed before programming the flows
 459                         installGroupOnDpn(groupId, dpnId, primaryIpAddress, listBucketInfo,
 460                                 getNextHopKey(vpnId, primaryIpAddress), GroupTypes.GroupAll);
 461                         // install Group
 462                         mdsalApiManager.syncInstallGroup(groupEntity);
 463                         // update MD-SAL DS
 464                         addVpnNexthopToDS(dpnId, vpnId, primaryIpAddress, currDestIpPrefix, groupId);
 465
 466                     } else {
 467                         // Ignore adding new prefix , if it already exists
 468                         List<IpAdjacencies> prefixesList = nexthop.getIpAdjacencies();
 469                         IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currDestIpPrefix).build();
 470                         if (prefixesList != null && prefixesList.contains(prefix)) {
 471                             LOG.trace("Prefix {} is already present in l3nextHop {} ", currDestIpPrefix, nexthop);
 472                         } else {
 473                             IpAdjacenciesBuilder ipPrefixesBuilder =
 474                                     new IpAdjacenciesBuilder().withKey(new IpAdjacenciesKey(currDestIpPrefix));
 475                             LOG.trace("Updating prefix {} to vpnNextHop {} Operational DS", currDestIpPrefix, nexthop);
 476                             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
 477                                     getVpnNextHopIpPrefixIdentifier(vpnId, primaryIpAddress, currDestIpPrefix),
 478                                     ipPrefixesBuilder.build());
 479                         }
 480                     }
 481                 }
 482             } finally {
 483                 FibUtil.unlockCluster(lockManager, nextHopLockStr);
 484             }
 485             return Collections.emptyList();
 486         });
 487         return groupId;
 488     }
 489
 490     private void installGroupOnDpn(long groupId, Uint64 dpnId, String groupName, List<BucketInfo> bucketsInfo,
 491                                      String nextHopKey, GroupTypes groupType) {
 492         NodeRef nodeRef = FibUtil.buildNodeRef(dpnId);
 493         Buckets buckets = FibUtil.buildBuckets(bucketsInfo);
 494         GroupRef groupRef = new GroupRef(FibUtil.buildGroupInstanceIdentifier(groupId, dpnId));
 495         AddGroupInput input = new AddGroupInputBuilder().setNode(nodeRef).setGroupId(new GroupId(groupId))
 496                 .setBuckets(buckets).setGroupRef(groupRef).setGroupType(groupType)
 497                 .setGroupName(groupName).build();
 498         Future<RpcResult<AddGroupOutput>> groupStats = salGroupService.addGroup(input);
 499         RpcResult<AddGroupOutput> rpcResult = null;
 500         try {
 501             rpcResult = groupStats.get();
 502             if (rpcResult != null && rpcResult.isSuccessful()) {
 503                 LOG.info("Group {} with key {} has been successfully installed directly on dpn {}.", groupId,
 504                         nextHopKey, dpnId);
 505             } else {
 506                 LOG.error("Unable to install group {} with key {} directly on dpn {} due to {}.", groupId, nextHopKey,
 507                         dpnId, rpcResult != null ? rpcResult.getErrors() : null);
 508             }
 509         } catch (InterruptedException | ExecutionException e) {
 510             LOG.error("Error while installing group {} directly on dpn {}", groupId, dpnId);
 511         }
 512     }
 513
 514     protected void addVpnNexthopToDS(Uint64 dpnId, Uint32 vpnId, String primaryIpAddr,
 515                                      String currIpAddr, long egressPointer) {
 516         InstanceIdentifierBuilder<VpnNexthops> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
 517             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId));
 518
 519         List<IpAdjacencies> ipPrefixesList = new ArrayList<>();
 520         IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currIpAddr).build();
 521         ipPrefixesList.add(prefix);
 522         // Add nexthop to vpn node
 523         VpnNexthop nh = new VpnNexthopBuilder()
 524             .withKey(new VpnNexthopKey(primaryIpAddr))
 525             .setDpnId(dpnId)
 526             .setIpAdjacencies(ipPrefixesList)
 527             .setEgressPointer(egressPointer).build();
 528
 529         InstanceIdentifier<VpnNexthop> id1 = idBuilder
 530             .child(VpnNexthop.class, new VpnNexthopKey(primaryIpAddr)).build();
 531         LOG.trace("Adding vpnnextHop {} to Operational DS", nh);
 532         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id1, nh);
 533
 534     }
 535
 536     protected InstanceIdentifier<IpAdjacencies> getVpnNextHopIpPrefixIdentifier(Uint32 vpnId, String primaryIpAddress,
 537                                                                                 String ipPrefix) {
 538         InstanceIdentifier<IpAdjacencies> id = InstanceIdentifier.builder(L3nexthop.class)
 539                 .child(VpnNexthops.class, new VpnNexthopsKey(vpnId))
 540                 .child(VpnNexthop.class, new VpnNexthopKey(primaryIpAddress))
 541                 .child(IpAdjacencies.class, new IpAdjacenciesKey(ipPrefix)).build();
 542         return id;
 543     }
 544
 545     @Nullable
 546     protected VpnNexthop getVpnNexthop(Uint32 vpnId, String ipAddress) {
 547
 548         // check if vpn node is there
 549         InstanceIdentifierBuilder<VpnNexthops> idBuilder =
 550             InstanceIdentifier.builder(L3nexthop.class).child(VpnNexthops.class,
 551                 new VpnNexthopsKey(vpnId));
 552         InstanceIdentifier<VpnNexthops> id = idBuilder.build();
 553         Optional<VpnNexthops> vpnNexthops = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 554         if (vpnNexthops.isPresent()) {
 555             // get nexthops list for vpn
 556             List<VpnNexthop> nexthops = vpnNexthops.get().nonnullVpnNexthop();
 557             for (VpnNexthop nexthop : nexthops) {
 558                 if (Objects.equals(nexthop.getIpAddress(), ipAddress)) {
 559                     // return nexthop
 560                     LOG.trace("VpnNextHop : {}", nexthop);
 561                     return nexthop;
 562                 }
 563             }
 564             // return null if not found
 565         }
 566         return null;
 567     }
 568
 569     @Nullable
 570     public AdjacencyResult getRemoteNextHopPointer(Uint64 remoteDpnId, Uint32 vpnId, String prefixIp,
 571             @Nullable String nextHopIp, Class<? extends TunnelTypeBase> tunnelType) {
 572         String egressIfName = null;
 573         LOG.trace("getRemoteNextHopPointer: input [remoteDpnId {}, vpnId {}, prefixIp {}, nextHopIp {} ]", remoteDpnId,
 574             vpnId, prefixIp, nextHopIp);
 575
 576         Class<? extends InterfaceType> egressIfType;
 577         ElanInstance elanInstance = getElanInstanceForPrefix(vpnId, prefixIp);
 578         if (elanInstance != null) {
 579             egressIfType = getInterfaceType(elanInstance);
 580         } else {
 581             LOG.warn("Failed to determine network type for prefixIp {} using tunnel", prefixIp);
 582             egressIfType = Tunnel.class;
 583         }
 584
 585         if (Tunnel.class.equals(egressIfType)) {
 586             egressIfName = getTunnelRemoteNextHopPointer(remoteDpnId, nextHopIp, tunnelType);
 587         } else {
 588             egressIfName = getExtPortRemoteNextHopPointer(remoteDpnId, elanInstance);
 589         }
 590
 591         LOG.trace("NextHop pointer for prefixIp {} vpnId {} dpnId {} is {}", prefixIp, vpnId, remoteDpnId,
 592             egressIfName);
 593         return egressIfName != null ? new AdjacencyResult(egressIfName, egressIfType, nextHopIp,
 594                 prefixIp) : null;
 595     }
 596
 597     private void removeVpnNexthopFromDS(Uint32 vpnId, String ipPrefix) {
 598
 599         InstanceIdentifierBuilder<VpnNexthop> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
 600             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId))
 601             .child(VpnNexthop.class, new VpnNexthopKey(ipPrefix));
 602         InstanceIdentifier<VpnNexthop> id = idBuilder.build();
 603         // remove from DS
 604         LOG.trace("Removing vpn next hop from datastore : {}", id);
 605         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 606     }
 607
 608     public void removeLocalNextHop(Uint64 dpnId, Uint32 vpnId, String primaryIpAddress, String currDestIpPrefix) {
 609         String nextHopLockStr = vpnId + primaryIpAddress;
 610         try {
 611             if (FibUtil.lockCluster(lockManager, nextHopLockStr, WAIT_TIME_TO_ACQUIRE_LOCK)) {
 612                 VpnNexthop nh = getVpnNexthop(vpnId, primaryIpAddress);
 613                 if (nh != null) {
 614                     List<IpAdjacencies> prefixesList = new ArrayList<>(nh.nonnullIpAdjacencies());
 615                     IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currDestIpPrefix).build();
 616                     prefixesList.remove(prefix);
 617                     if (prefixesList.isEmpty()) { //remove the group only if there are no more flows using this group
 618                         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, nh.getEgressPointer().toJava(),
 619                                 primaryIpAddress, GroupTypes.GroupAll, Collections.emptyList());
 620                         // remove Group ...
 621                         mdsalApiManager.removeGroup(groupEntity);
 622                         //update MD-SAL DS
 623                         removeVpnNexthopFromDS(vpnId, primaryIpAddress);
 624                         //release groupId
 625                         removeNextHopPointer(getNextHopKey(vpnId, primaryIpAddress));
 626                         LOG.debug("Local Next hop {} for {} {} on dpn {} successfully deleted",
 627                                 nh.getEgressPointer(), vpnId, primaryIpAddress, dpnId);
 628                     } else {
 629                         //remove the currIpPrefx from IpPrefixList of the vpnNexthop
 630                         LOG.trace("Removing the prefix {} from vpnNextHop {} Operational DS", currDestIpPrefix, nh);
 631                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
 632                                 getVpnNextHopIpPrefixIdentifier(vpnId, primaryIpAddress, currDestIpPrefix));
 633                     }
 634                 } else {
 635                     //throw error
 636                     LOG.error("Local NextHop for VpnId {} curIpPrefix {} on dpn {} primaryIpAddress {} not deleted",
 637                             vpnId, currDestIpPrefix, dpnId, primaryIpAddress);
 638                 }
 639             }
 640         } finally {
 641             FibUtil.unlockCluster(lockManager, nextHopLockStr);
 642         }
 643     }
 644
 645     public void setConfTransType(String service, String transportType) {
 646
 647         if (!service.equalsIgnoreCase("L3VPN")) {
 648             LOG.error("Incorrect service {} provided for setting the transport type.", service);
 649             return;
 650         }
 651
 652         L3VPNTransportTypes transType = L3VPNTransportTypes.validateTransportType(transportType
 653                 .toUpperCase(Locale.getDefault()));
 654
 655         if (transType != L3VPNTransportTypes.Invalid) {
 656             configuredTransportTypeL3VPN = transType;
 657         }
 658     }
 659
 660     public void writeConfTransTypeConfigDS() {
 661         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, getConfTransportTypeIdentifier(),
 662             createConfTransportType(configuredTransportTypeL3VPN.getTransportType()));
 663     }
 664
 665     public L3VPNTransportTypes getConfiguredTransportTypeL3VPN() {
 666         return this.configuredTransportTypeL3VPN;
 667     }
 668
 669     public String getReqTransType() {
 670         if (configuredTransportTypeL3VPN == L3VPNTransportTypes.Invalid) {
 671             /*
 672              * Restart scenario, Read from the ConfigDS.
 673              * if the value is Unset, cache value as VxLAN.
 674              */
 675             LOG.trace("configureTransportType is not yet set.");
 676             Optional<ConfTransportTypeL3vpn> configuredTransTypeFromConfig =
 677                 MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, getConfTransportTypeIdentifier());
 678
 679             if (configuredTransTypeFromConfig.isPresent()) {
 680                 if (TunnelTypeGre.class.equals(configuredTransTypeFromConfig.get().getTransportType())) {
 681                     configuredTransportTypeL3VPN = L3VPNTransportTypes.GRE;
 682                 } else {
 683                     configuredTransportTypeL3VPN = L3VPNTransportTypes.VxLAN;
 684                 }
 685                 LOG.trace("configuredTransportType set from config DS to {}",
 686                     getConfiguredTransportTypeL3VPN().getTransportType());
 687             } else {
 688                 setConfTransType("L3VPN", L3VPNTransportTypes.VxLAN.getTransportType());
 689                 LOG.trace("configuredTransportType is not set in the Config DS. VxLAN as default will be used.");
 690             }
 691         } else {
 692             LOG.trace("configuredTransportType is set as {}", getConfiguredTransportTypeL3VPN().getTransportType());
 693         }
 694         return getConfiguredTransportTypeL3VPN().getTransportType();
 695     }
 696
 697     public InstanceIdentifier<ConfTransportTypeL3vpn> getConfTransportTypeIdentifier() {
 698         return InstanceIdentifier.builder(ConfTransportTypeL3vpn.class).build();
 699     }
 700
 701     private ConfTransportTypeL3vpn createConfTransportType(String type) {
 702         ConfTransportTypeL3vpn confTransType;
 703         switch (type) {
 704             case ITMConstants.TUNNEL_TYPE_GRE:
 705                 confTransType = new ConfTransportTypeL3vpnBuilder().setTransportType(TunnelTypeGre.class).build();
 706                 LOG.trace("Setting the confTransportType to GRE.");
 707                 break;
 708             case ITMConstants.TUNNEL_TYPE_VXLAN:
 709                 confTransType = new ConfTransportTypeL3vpnBuilder().setTransportType(TunnelTypeVxlan.class).build();
 710                 LOG.trace("Setting the confTransportType to VxLAN.");
 711                 break;
 712             default:
 713                 LOG.trace("Invalid transport type {} passed to Config DS ", type);
 714                 confTransType = null;
 715                 break;
 716         }
 717         return confTransType;
 718     }
 719
 720     public Class<? extends TunnelTypeBase> getReqTunType(String transportType) {
 721         switch (transportType) {
 722             case "VXLAN":
 723                 return TunnelTypeVxlan.class;
 724             case "GRE":
 725                 return TunnelTypeGre.class;
 726             default:
 727                 return TunnelTypeMplsOverGre.class;
 728         }
 729     }
 730
 731     public String getTransportTypeStr(String tunType) {
 732         if (tunType.equals(TunnelTypeVxlan.class.toString())) {
 733             return ITMConstants.TUNNEL_TYPE_VXLAN;
 734         } else if (tunType.equals(TunnelTypeGre.class.toString())) {
 735             return ITMConstants.TUNNEL_TYPE_GRE;
 736         } else if (tunType.equals(TunnelTypeMplsOverGre.class.toString())) {
 737             return ITMConstants.TUNNEL_TYPE_MPLSoGRE;
 738         } else {
 739             return ITMConstants.TUNNEL_TYPE_INVALID;
 740         }
 741     }
 742
 743     @Override
 744     @PreDestroy
 745     public void close() {
 746         LOG.info("{} close", getClass().getSimpleName());
 747     }
 748
 749     // TODO Clean up the exception handling
 750     @SuppressWarnings("checkstyle:IllegalCatch")
 751     @Nullable
 752     private String getTunnelRemoteNextHopPointer(Uint64 remoteDpnId, String nextHopIp,
 753                                                  Class<? extends TunnelTypeBase> tunnelType) {
 754         if (nextHopIp != null && !nextHopIp.isEmpty()) {
 755             try {
 756                 // here use the config for tunnel type param
 757                 return getTunnelInterfaceName(remoteDpnId,
 758                     org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder
 759                         .getDefaultInstance(nextHopIp), tunnelType);
 760             } catch (Exception ex) {
 761                 LOG.error("Error while retrieving nexthop pointer for nexthop {} remoteDpn {}",
 762                         nextHopIp, remoteDpnId, ex);
 763             }
 764         }
 765
 766         return null;
 767     }
 768
 769     private String getExtPortRemoteNextHopPointer(Uint64 remoteDpnId, ElanInstance elanInstance) {
 770         return elanService.getExternalElanInterface(elanInstance.getElanInstanceName(), remoteDpnId);
 771     }
 772
 773     /**
 774      * Get the interface type associated with the type of ELAN used for routing
 775      * traffic to/from remote compute nodes.
 776      *
 777      * @param elanInstance The elan instance
 778      * @return L2vlan for flat/VLAN network type and Tunnel otherwise
 779      */
 780     private Class<? extends InterfaceType> getInterfaceType(ElanInstance elanInstance) {
 781         Class<? extends SegmentTypeBase> segmentType = elanInstance.getSegmentType();
 782         if (SegmentTypeFlat.class.equals(segmentType) || SegmentTypeVlan.class.equals(segmentType)) {
 783             return L2vlan.class;
 784         }
 785
 786         return Tunnel.class;
 787     }
 788
 789     private ElanInstance getElanInstanceForPrefix(Uint32 vpnId, String prefixIp) {
 790         ElanInstance elanInstance = null;
 791         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, prefixIp);
 792         if (prefix != null) {
 793             if (prefix.getNetworkId() != null) {
 794                 elanInstance = elanService.getElanInstance(prefix.getNetworkId().getValue());
 795             }
 796         }
 797
 798         return elanInstance;
 799     }
 800
 801     static class AdjacencyResult {
 802         private final String interfaceName;
 803         private final Class<? extends InterfaceType> interfaceType;
 804         private final String nextHopIp;
 805         private final String prefix;
 806
 807         AdjacencyResult(String interfaceName, Class<? extends InterfaceType> interfaceType, String nextHopIp,
 808                         String prefix) {
 809             this.interfaceName = interfaceName;
 810             this.interfaceType = interfaceType;
 811             this.nextHopIp = nextHopIp;
 812             this.prefix = prefix;
 813         }
 814
 815         public String getInterfaceName() {
 816             return interfaceName;
 817         }
 818
 819         public Class<? extends InterfaceType> getInterfaceType() {
 820             return interfaceType;
 821         }
 822
 823         public String getNextHopIp() {
 824             return nextHopIp;
 825         }
 826
 827         public String getPrefix() {
 828             return prefix;
 829         }
 830
 831         @Override
 832         public int hashCode() {
 833             final int prime = 31;
 834             int result = 1;
 835             result = prime * result + (interfaceName == null ? 0 : interfaceName.hashCode());
 836             return result;
 837         }
 838
 839         @Override
 840         public boolean equals(Object obj) {
 841             if (obj == null) {
 842                 return false;
 843             }
 844
 845             if (getClass() != obj.getClass()) {
 846                 return false;
 847             } else {
 848                 AdjacencyResult other = (AdjacencyResult) obj;
 849                 return interfaceName.equals(other.interfaceName);
 850             }
 851         }
 852     }
 853
 854     protected long setupLoadBalancingNextHop(Uint32 parentVpnId, Uint64 dpnId,
 855             String destPrefix, List<BucketInfo> localBucketInfo, List<BucketInfo> remoteBucketInfo) {
 856         long remoteGroupId = createNextHopPointer(getRemoteSelectGroupKey(parentVpnId, destPrefix));
 857         if (remoteGroupId == FibConstants.INVALID_GROUP_ID) {
 858             LOG.error("Unable to allocate/retrieve remote groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 859             return remoteGroupId;
 860         }
 861         long localGroupId =  createNextHopPointer(getLocalSelectGroupKey(parentVpnId, destPrefix));
 862         if (localGroupId == FibConstants.INVALID_GROUP_ID) {
 863             LOG.error("Unable to allocate/retrieve local groupId for vpnId {} , prefix {}",
 864                 parentVpnId, destPrefix);
 865             return remoteGroupId;
 866         }
 867         List<BucketInfo> combinedBucketInfo = new ArrayList<>();
 868         combinedBucketInfo.addAll(localBucketInfo);
 869         combinedBucketInfo.addAll(remoteBucketInfo);
 870         GroupEntity remoteGroupEntity = MDSALUtil.buildGroupEntity(
 871                 dpnId, remoteGroupId, destPrefix, GroupTypes.GroupSelect, combinedBucketInfo);
 872         GroupEntity localGroupEntity = MDSALUtil.buildGroupEntity(
 873                 dpnId, localGroupId, destPrefix, GroupTypes.GroupSelect, localBucketInfo);
 874         String jobKey = FibUtil.getCreateLocalNextHopJobKey(parentVpnId, dpnId, destPrefix);
 875         jobCoordinator.enqueueJob(jobKey, () -> {
 876             mdsalApiManager.syncInstallGroup(remoteGroupEntity);
 877             if (!localBucketInfo.isEmpty()) {
 878                 mdsalApiManager.syncInstallGroup(localGroupEntity);
 879             }
 880             if (LOG.isDebugEnabled()) {
 881                 LOG.debug("Finished installing GroupEntity with jobCoordinator key {} remoteGroupEntity.groupId {}"
 882                         + "localGroupEntity.groupId {}  groupEntity.groupType {}", jobKey,
 883                         remoteGroupEntity.getGroupId(), localGroupEntity.getGroupId(),
 884                         remoteGroupEntity.getGroupType());
 885             }
 886             // Delete local group(if exists) if there is no local info.
 887             // Local group has to be deleted if all VMs in a compute is deleted.
 888             if (localBucketInfo.isEmpty()) {
 889                 LOG.debug("Deleting local group {} since no local nhs present for "
 890                         + "prefix {}", localGroupEntity.getGroupId(), destPrefix);
 891                 mdsalApiManager.syncRemoveGroup(localGroupEntity);
 892             }
 893             return Collections.emptyList();
 894         });
 895         return remoteGroupId;
 896     }
 897
 898     protected void deleteLoadBalancingNextHop(Uint32 parentVpnId, Uint64 dpnId, String destPrefix) {
 899         long remoteGroupId = createNextHopPointer(getRemoteSelectGroupKey(parentVpnId, destPrefix));
 900         if (remoteGroupId == FibConstants.INVALID_GROUP_ID) {
 901             LOG.error("Unable to allocate/retrieve remote groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 902         }
 903         long localGroupId = createNextHopPointer(getLocalSelectGroupKey(parentVpnId, destPrefix));
 904         if (localGroupId == FibConstants.INVALID_GROUP_ID) {
 905             LOG.error("Unable to allocate/retrieve local groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 906         }
 907         GroupEntity remoteGroupEntity = MDSALUtil.buildGroupEntity(
 908                 dpnId, remoteGroupId, destPrefix, GroupTypes.GroupSelect, Collections.emptyList());
 909         GroupEntity localGroupEntity = MDSALUtil.buildGroupEntity(
 910                 dpnId, localGroupId, destPrefix, GroupTypes.GroupSelect, Collections.emptyList());
 911         String jobKey = FibUtil.getCreateLocalNextHopJobKey(parentVpnId, dpnId, destPrefix);
 912         jobCoordinator.enqueueJob(jobKey, () -> {
 913             mdsalApiManager.syncRemoveGroup(remoteGroupEntity);
 914             mdsalApiManager.syncRemoveGroup(localGroupEntity);
 915             if (LOG.isDebugEnabled()) {
 916                 LOG.debug("Finished removing GroupEntity with jobCoordinator key {} remoteGroupEntity.groupId {}"
 917                     + "localGroupEntity.groupId {}", jobKey, remoteGroupId, localGroupId);
 918             }
 919             return Collections.emptyList();
 920         });
 921     }
 922
 923     long createNextHopGroups(Uint32 vpnId, String rd, Uint64 dpnId, VrfEntry vrfEntry,
 924             @Nullable Routes routes, List<Routes> vpnExtraRoutes) {
 925         List<BucketInfo> localBucketInfo = new ArrayList<>();
 926         List<Routes> clonedVpnExtraRoutes  = new ArrayList<>(vpnExtraRoutes);
 927         if (clonedVpnExtraRoutes.contains(routes)) {
 928             localBucketInfo.addAll(getBucketsForLocalNexthop(vpnId, dpnId, vrfEntry, routes));
 929             clonedVpnExtraRoutes.remove(routes);
 930         }
 931         List<BucketInfo> remoteBucketInfo =
 932             new ArrayList<>(getBucketsForRemoteNexthop(vpnId, dpnId, vrfEntry, rd, clonedVpnExtraRoutes));
 933         return setupLoadBalancingNextHop(vpnId, dpnId,
 934             vrfEntry.getDestPrefix(), localBucketInfo, remoteBucketInfo);
 935     }
 936
 937     private List<BucketInfo> getBucketsForLocalNexthop(Uint32 vpnId, Uint64 dpnId,
 938             VrfEntry vrfEntry, Routes routes) {
 939         @Nullable List<String> nexthopIpList = routes.getNexthopIpList();
 940         if (LOG.isDebugEnabled()) {
 941             LOG.debug("NexthopManager.getBucketsForLocalNexthop invoked with vpnId {} dpnId {} "
 942                             + " vrfEntry.routePaths {}, routes.nexthopList {}", vpnId, dpnId, vrfEntry.getRoutePaths(),
 943                 nexthopIpList);
 944         }
 945         List<BucketInfo> listBucketInfo = new CopyOnWriteArrayList<>();
 946         if (nexthopIpList != null) {
 947             nexthopIpList.parallelStream().forEach(nextHopIp -> {
 948                 String localNextHopIP;
 949                 if (isIpv4Address(nextHopIp)) {
 950                     localNextHopIP = nextHopIp + NwConstants.IPV4PREFIX;
 951                 } else {
 952                     localNextHopIP = nextHopIp + NwConstants.IPV6PREFIX;
 953                 }
 954                 Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 955                 if (localNextHopInfo != null) {
 956                     long groupId = getLocalNextHopGroup(vpnId, localNextHopIP);
 957                     if (groupId == FibConstants.INVALID_GROUP_ID) {
 958                         LOG.error("Unable to allocate groupId for vpnId {} , prefix {} , interface {}", vpnId,
 959                             vrfEntry.getDestPrefix(), localNextHopInfo.getVpnInterfaceName());
 960                         return;
 961                     }
 962                     List<ActionInfo> actionsInfos =
 963                         Collections.singletonList(new ActionGroup(groupId));
 964                     BucketInfo bucket = new BucketInfo(actionsInfos);
 965                     bucket.setWeight(1);
 966                     listBucketInfo.add(bucket);
 967                 }
 968             });
 969         }
 970         LOG.trace("LOCAL: listbucket {}, vpnId {}, dpnId {}, routes {}", listBucketInfo, vpnId, dpnId, routes);
 971         return listBucketInfo;
 972     }
 973
 974     private List<BucketInfo> getBucketsForRemoteNexthop(Uint32 vpnId, Uint64 dpnId, VrfEntry vrfEntry, String rd,
 975             List<Routes> vpnExtraRoutes) {
 976         List<BucketInfo> listBucketInfo = new ArrayList<>();
 977         Map<String, List<ActionInfo>> egressActionMap = new HashMap<>();
 978         vpnExtraRoutes.stream().filter(vpnExtraRoute -> vpnExtraRoute.getNexthopIpList() != null).forEach(
 979             vpnExtraRoute -> vpnExtraRoute.getNexthopIpList().forEach(nextHopIp -> {
 980                 String nextHopPrefixIp;
 981                 if (isIpv4Address(nextHopIp)) {
 982                     nextHopPrefixIp = nextHopIp + NwConstants.IPV4PREFIX;
 983                 } else {
 984                     nextHopPrefixIp = nextHopIp + NwConstants.IPV6PREFIX;
 985                 }
 986                 List<String> tepIpAddresses = fibUtil.getNextHopAddresses(rd, nextHopPrefixIp);
 987                 if (tepIpAddresses.isEmpty()) {
 988                     return;
 989                 }
 990                 // There would be only one nexthop address for a VM ip which would be the tep Ip
 991                 String tepIp = tepIpAddresses.get(0);
 992                 AdjacencyResult adjacencyResult = getRemoteNextHopPointer(dpnId, vpnId,
 993                     vrfEntry.getDestPrefix(), tepIp, TunnelTypeVxlan.class);
 994                 if (adjacencyResult == null) {
 995                     return;
 996                 }
 997                 String egressInterface = adjacencyResult.getInterfaceName();
 998                 if (!FibUtil.isTunnelInterface(adjacencyResult)) {
 999                     return;
1000                 }
1001                 Class<? extends TunnelTypeBase> tunnelType =
1002                     VpnExtraRouteHelper.getTunnelType(itmManager, egressInterface);
1003                 StateTunnelList ifState = null;
1004                 try {
1005                     ifState = fibUtil.getTunnelState(egressInterface);
1006                     if (ifState == null || ifState.getOperState() != TunnelOperStatus.Up) {
1007                         LOG.trace("Tunnel is not up for interface {}", egressInterface);
1008                         return;
1009                     }
1010                 } catch (ReadFailedException e) {
1011                     LOG.error("getBucketsForRemoteNexthop: error in fetching tunnel state for interface {}",
1012                         egressInterface, e);
1013                     return;
1014                 }
1015                 if (!TunnelTypeVxlan.class.equals(tunnelType)) {
1016                     return;
1017                 }
1018                 Uint32 label = FibUtil.getLabelFromRoutePaths(vrfEntry).get();
1019                 Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, nextHopPrefixIp);
1020                 if (prefixInfo == null) {
1021                     LOG.error("No prefix info found for prefix {} in rd {} for VPN {}", nextHopPrefixIp, rd,
1022                         vpnId);
1023                     return;
1024                 }
1025                 Uint32 tunnelId;
1026                 if (FibUtil.isVxlanNetwork(prefixInfo.getNetworkType())) {
1027                     tunnelId = prefixInfo.getSegmentationId();
1028                 } else {
1029                     LOG.warn("Network is not of type VXLAN for prefix {}."
1030                         + "Going with default Lport Tag.", prefixInfo.toString());
1031                     tunnelId = label;
1032                 }
1033                 List<ActionInfo> actionInfos = new ArrayList<>();
1034                 actionInfos.add(new ActionSetFieldTunnelId(Uint64.valueOf(tunnelId.longValue())));
1035                 String ifName = prefixInfo.getVpnInterfaceName();
1036                 String vpnName = fibUtil.getVpnNameFromId(vpnId);
1037                 if (vpnName == null) {
1038                     return;
1039                 }
1040                 String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, nextHopPrefixIp);
1041                 actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
1042                     new MacAddress(macAddress)));
1043                 List<ActionInfo> egressActions;
1044                 if (egressActionMap.containsKey(egressInterface)) {
1045                     egressActions = egressActionMap.get(egressInterface);
1046                 } else {
1047                     egressActions = getEgressActionsForInterface(egressInterface, actionInfos.size(),
1048                             true, vpnId, vrfEntry.getDestPrefix());
1049                     if (egressActions.isEmpty()) {
1050                         LOG.error("Skipping getBucketsForRemoteNexthop: Empty list of egress actions received for "
1051                                         + "interface {} on dpn {} for vpn {} prefix {} nextHop {}", ifName, dpnId,
1052                                 vpnId, vrfEntry.getDestPrefix(), nextHopPrefixIp);
1053                     }
1054                     egressActionMap.put(egressInterface, egressActions);
1055                 }
1056                 if (egressActions.isEmpty()) {
1057                     LOG.error("Failed to retrieve egress action for prefix {} route-paths {}"
1058                             + " interface {}." + " Aborting remote FIB entry creation.",
1059                         vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), egressInterface);
1060                 }
1061                 actionInfos.addAll(egressActions);
1062                 BucketInfo bucket = new BucketInfo(actionInfos);
1063                 bucket.setWeight(1);
1064                 listBucketInfo.add(bucket);
1065             }));
1066         LOG.trace("LOCAL: listbucket {}, rd {}, dpnId {}, routes {}", listBucketInfo, rd, dpnId, vpnExtraRoutes);
1067         return listBucketInfo;
1068     }
1069
1070     public void createDcGwLoadBalancingGroup(Uint64 dpnId, String destinationIp,
1071                                              Class<? extends TunnelTypeBase> tunnelType) {
1072         jobCoordinator.enqueueJob(getJobKey(dpnId), () -> {
1073             List<ListenableFuture<Void>> futures = new ArrayList<>();
1074             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operationalTx -> {
1075                 synchronized (getDcGateWaySyncKey(destinationIp)) {
1076                     FibUtil.addL3vpnDcGateWay(destinationIp, operationalTx);
1077                 }
1078                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, configTx -> {
1079                     List<String> availableDcGws = getDcGwIps();
1080                     Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
1081                     int noOfDcGws = availableDcGws.size();
1082                     if (noOfDcGws == 1) {
1083                         LOG.trace("There are no enough DC GateWays {} present to program LB group", availableDcGws);
1084                         return;
1085                     }
1086                     if (availableDcGws.contains(destinationIp)) {
1087                         availableDcGws.remove(destinationIp);
1088                     }
1089                     availableDcGws.forEach(dcGwIp -> {
1090                         List<String> dcGws = Arrays.asList(dcGwIp, destinationIp);
1091                         Collections.sort(dcGws);
1092                         String groupIdKey = FibUtil.getGreLbGroupKey(dcGws);
1093                         Long groupId = createNextHopPointer(groupIdKey);
1094                         List<Bucket> listBucket = new ArrayList<>();
1095                         for (int index = 0; index < dcGws.size(); index++) {
1096                             if (isTunnelUp(dcGws.get(index), dpnId, tunnelType)) {
1097                                 listBucket.add(buildBucketForDcGwLbGroup(dcGws.get(index),
1098                                         dpnId, index, tunnelType, true));
1099                             }
1100                         }
1101                         Group group = MDSALUtil.buildGroup(groupId, groupIdKey, GroupTypes.GroupSelect,
1102                                         MDSALUtil.buildBucketLists(listBucket));
1103                         mdsalApiManager.addGroup(configTx, dpnId, group);
1104                         FibUtil.updateLbGroupInfo(dpnId, groupIdKey, groupId.toString(), operationalTx);
1105                         LOG.trace("LB group {} towards DC-GW installed on dpn {}. Group - {}",
1106                                 groupIdKey, dpnId, group);
1107                     });
1108                 }));
1109             }));
1110             return futures;
1111         }, RETRY_COUNT);
1112     }
1113
1114     private String getJobKey(Uint64 dpnId) {
1115         return new StringBuilder().append(NEXTHOPMANAGER_JOB_KEY_PREFIX).append(dpnId).toString();
1116     }
1117
1118     private String getDcGateWaySyncKey(String destinationIp) {
1119         String mutex = new StringBuilder().append("L3vpncDcGateWay").append(destinationIp).toString();
1120         return mutex.intern();
1121     }
1122
1123     private List<String> getDcGwIps() {
1124         InstanceIdentifier<DcGatewayIpList> dcGatewayIpListid =
1125                 InstanceIdentifier.builder(DcGatewayIpList.class).build();
1126         DcGatewayIpList dcGatewayIpListConfig =
1127                 MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, dcGatewayIpListid).orNull();
1128         if (dcGatewayIpListConfig == null) {
1129             return Collections.emptyList();
1130         }
1131         return dcGatewayIpListConfig.getDcGatewayIp()
1132                 .stream()
1133                 .filter(dcGwIp -> dcGwIp.getTunnnelType().equals(TunnelTypeMplsOverGre.class))
1134                 .map(dcGwIp -> dcGwIp.getIpAddress().stringValue()).sorted()
1135                 .collect(toList());
1136     }
1137
1138     private boolean isTunnelUp(String dcGwIp, Uint64 dpnId, Class<? extends TunnelTypeBase> tunnelType) {
1139         String tunnelName = getTunnelRemoteNextHopPointer(dpnId, dcGwIp, tunnelType);
1140         if (tunnelName != null) {
1141             InstanceIdentifier<StateTunnelList> tunnelStateId =
1142                     InstanceIdentifier.builder(TunnelsState.class).child(
1143                             StateTunnelList.class, new StateTunnelListKey(tunnelName)).build();
1144             return MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, tunnelStateId)
1145                     .toJavaUtil().map(StateTunnelList::getOperState)
1146                     .orElse(TunnelOperStatus.Down) == TunnelOperStatus.Up;
1147         }
1148         return false;
1149     }
1150
1151     private List<Action> getEgressActions(String interfaceName, int actionKey) {
1152         List<Action> actions = Collections.emptyList();
1153         try {
1154             GetEgressActionsForInterfaceInputBuilder egressAction =
1155                     new GetEgressActionsForInterfaceInputBuilder().setIntfName(interfaceName).setActionKey(actionKey);
1156             Future<RpcResult<GetEgressActionsForInterfaceOutput>> result =
1157                     odlInterfaceRpcService.getEgressActionsForInterface(egressAction.build());
1158             RpcResult<GetEgressActionsForInterfaceOutput> rpcResult = result.get();
1159             if (!rpcResult.isSuccessful()) {
1160                 LOG.warn("RPC Call to Get egress actions for interface {} returned with Errors {}",
1161                         interfaceName, rpcResult.getErrors());
1162             } else {
1163                 actions = rpcResult.getResult().nonnullAction();
1164             }
1165         } catch (InterruptedException | ExecutionException e) {
1166             LOG.warn("Exception when egress actions for interface {}", interfaceName, e);
1167         }
1168         return actions;
1169     }
1170
1171     /**
1172      * This method is invoked when the neighbor is removed from DS.
1173      * All the LB groups which point to the given destination will be deleted.
1174      */
1175     public void removeDcGwLoadBalancingGroup(Uint64 dpnId,
1176             String destinationIp) {
1177         jobCoordinator.enqueueJob(getJobKey(dpnId), () -> {
1178             List<String> availableDcGws = fibUtil.getL3VpnDcGateWays();
1179             if (availableDcGws.contains(destinationIp)) {
1180                 availableDcGws.remove(destinationIp);
1181             }
1182             List<ListenableFuture<Void>> futures = new ArrayList<>();
1183             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operationalTx -> {
1184                 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx -> {
1185                     availableDcGws.forEach(dcGwIp -> {
1186                         List<String> dcGws = Arrays.asList(dcGwIp, destinationIp);
1187                         Collections.sort(dcGws);
1188                         String nextHopKey = FibUtil.getGreLbGroupKey(dcGws);
1189                         Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
1190                         if (!optionalNextHops.isPresent()) {
1191                             return;
1192                         }
1193                         Nexthops nexthops = optionalNextHops.get();
1194                         final String groupId = nexthops.getGroupId();
1195                         final long groupIdValue = Long.parseLong(groupId);
1196                         Group group = MDSALUtil.buildGroup(groupIdValue, nextHopKey, GroupTypes.GroupSelect,
1197                                 MDSALUtil.buildBucketLists(Collections.emptyList()));
1198                         LOG.trace("Removed LB group {} on dpn {}", group, dpnId);
1199                         try {
1200                             mdsalApiManager.removeGroup(configTx, dpnId, group);
1201                         } catch (ExecutionException | InterruptedException e) {
1202                             LOG.error("Group removal failed for group {} with exception", groupId, e);
1203                         }
1204                         removeNextHopPointer(nextHopKey);
1205                         FibUtil.removeOrUpdateNextHopInfo(dpnId, nextHopKey, groupId, nexthops, operationalTx);
1206                     });
1207                     synchronized (getDcGateWaySyncKey(destinationIp)) {
1208                         FibUtil.removeL3vpnDcGateWay(destinationIp, operationalTx);
1209                     }
1210                 }));
1211             }));
1212             return futures;
1213         }, RETRY_COUNT);
1214     }
1215
1216     /**
1217      * This method is invoked when the tunnel status is deleted.
1218      * All the buckets which point to given destination will be marked down.
1219      */
1220     public void updateDcGwLoadBalancingGroup(Uint64 dpnId, String destinationIp,
1221             boolean isTunnelUp, Class<? extends TunnelTypeBase> tunnelType) {
1222         jobCoordinator.enqueueJob(getJobKey(dpnId), () -> {
1223             List<String> availableDcGws = fibUtil.getL3VpnDcGateWays();
1224             if (availableDcGws.contains(destinationIp)) {
1225                 availableDcGws.remove(destinationIp);
1226             }
1227             List<ListenableFuture<Void>> futures = new ArrayList<>();
1228             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx -> {
1229                 availableDcGws.forEach(dcGwIp -> {
1230                     List<String> dcGws = Arrays.asList(dcGwIp, destinationIp);
1231                     Collections.sort(dcGws);
1232                     String nextHopKey = FibUtil.getGreLbGroupKey(dcGws);
1233                     int bucketId = dcGws.indexOf(destinationIp);
1234                     Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
1235                     if (!optionalNextHops.isPresent()) {
1236                         return;
1237                     }
1238                     Nexthops nexthops = optionalNextHops.get();
1239                     final String groupId = nexthops.getGroupId();
1240                     final long groupIdValue = Long.parseLong(groupId);
1241                     Bucket bucket = buildBucketForDcGwLbGroup(destinationIp, dpnId, bucketId, tunnelType, isTunnelUp);
1242                     LOG.trace("updated bucket {} to group {} on dpn {}.", bucket, groupId, dpnId);
1243                     try {
1244                         mdsalApiManager.addBucket(configTx, dpnId, groupIdValue, bucket);
1245                     } catch (ExecutionException | InterruptedException e) {
1246                         LOG.error("Bucket addition failed for bucket {} with exception", bucketId, e);
1247                     }
1248                 });
1249             }));
1250             return futures;
1251         }, RETRY_COUNT);
1252     }
1253
1254     private Bucket buildBucketForDcGwLbGroup(String ipAddress, Uint64 dpnId,
1255             int index, Class<? extends TunnelTypeBase> tunnelType, boolean isTunnelUp) {
1256         List<Action> listAction = new ArrayList<>();
1257         // ActionKey 0 goes to mpls label.
1258         int actionKey = 1;
1259         listAction.add(new ActionPushMpls().buildAction());
1260         listAction.add(new ActionRegMove(actionKey++, FibConstants.NXM_REG_MAPPING
1261                 .get(index), 0, 19).buildAction());
1262         String tunnelInterfaceName = getTunnelInterfaceName(dpnId, IpAddressBuilder.getDefaultInstance(ipAddress),
1263             tunnelType);
1264         List<Action> egressActions = getEgressActions(tunnelInterfaceName, actionKey++);
1265         if (!egressActions.isEmpty()) {
1266             listAction.addAll(getEgressActions(tunnelInterfaceName, actionKey++));
1267         } else {
1268             // clear off actions if there is no egress actions.
1269             listAction = Collections.emptyList();
1270         }
1271         long watchPort = MDSALUtil.WATCH_PORT;
1272         if (!isTunnelUp) {
1273             watchPort = 0xFFFFFFFEL;
1274         }
1275         //OVS expects a non-zero weight value for load balancing to happen in select groups
1276         return MDSALUtil.buildBucket(listAction, SELECT_GROUP_WEIGHT, index,
1277                 watchPort, MDSALUtil.WATCH_GROUP);
1278     }
1279
1280     public void programDcGwLoadBalancingGroup(Uint64 dpnId, String destinationIp,
1281                                               int addRemoveOrUpdate, boolean isTunnelUp,
1282                                               Class<? extends TunnelTypeBase> tunnelType) {
1283         if (NwConstants.ADD_FLOW == addRemoveOrUpdate) {
1284             createDcGwLoadBalancingGroup(dpnId, destinationIp, tunnelType);
1285         } else if (NwConstants.DEL_FLOW == addRemoveOrUpdate) {
1286             removeDcGwLoadBalancingGroup(dpnId, destinationIp);
1287         } else if (NwConstants.MOD_FLOW == addRemoveOrUpdate) {
1288             updateDcGwLoadBalancingGroup(dpnId, destinationIp, isTunnelUp, tunnelType);
1289         }
1290     }
1291 }