fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/NexthopManager.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static java.util.stream.Collectors.toList;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  14
  15 import com.google.common.base.Optional;
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.util.concurrent.ListenableFuture;
  18
  19 import java.util.ArrayList;
  20 import java.util.Arrays;
  21 import java.util.Collections;
  22 import java.util.HashMap;
  23 import java.util.List;
  24 import java.util.Locale;
  25 import java.util.Map;
  26 import java.util.Objects;
  27 import java.util.concurrent.CopyOnWriteArrayList;
  28 import java.util.concurrent.ExecutionException;
  29 import java.util.concurrent.Future;
  30 import java.util.concurrent.TimeUnit;
  31 import java.util.concurrent.TimeoutException;
  32 import javax.annotation.PreDestroy;
  33 import javax.inject.Inject;
  34 import javax.inject.Singleton;
  35
  36 import org.eclipse.jdt.annotation.Nullable;
  37 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  38 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  39 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  40 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  41 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  42 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  43 import org.opendaylight.genius.itm.globals.ITMConstants;
  44 import org.opendaylight.genius.mdsalutil.ActionInfo;
  45 import org.opendaylight.genius.mdsalutil.BucketInfo;
  46 import org.opendaylight.genius.mdsalutil.GroupEntity;
  47 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  48 import org.opendaylight.genius.mdsalutil.NwConstants;
  49 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  50 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionOutput;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionPushVlan;
  54 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  55 import org.opendaylight.genius.mdsalutil.actions.ActionRegMove;
  56 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  57 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetSource;
  58 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  59 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldVlanVid;
  60 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  61 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  62 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  63 import org.opendaylight.netvirt.fibmanager.api.L3VPNTransportTypes;
  64 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  65 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev170119.L2vlan;
  66 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev170119.Tunnel;
  67 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfaceType;
  69 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.OutputActionCase;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.PushVlanActionCase;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.SetFieldCase;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInput;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInputBuilder;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutput;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInput;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInputBuilder;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolOutput;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInput;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInputBuilder;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdOutput;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceInputBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceOutput;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelOperStatus;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelsState;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelListKey;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rev160406.DcGatewayIpList;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelInputBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelOutput;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameInputBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameOutput;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.lockmanager.rev160413.LockManagerService;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupInput;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupInputBuilder;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupOutput;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.SalGroupService;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupId;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupRef;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.Buckets;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeRef;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeBase;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeFlat;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeVlan;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.L3nexthop;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthops;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthopsKey;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopKey;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacencies;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacenciesBuilder;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacenciesKey;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpn;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpnBuilder;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.lb.nexthops.Nexthops;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.add.group.input.buckets.bucket.action.action.NxActionResubmitRpcAddGroupCase;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.nodes.node.table.flow.instructions.instruction.instruction.apply.actions._case.apply.actions.action.action.NxActionRegLoadNodesNodeTableFlowApplyActionsCase;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.nx.action.reg.load.grouping.NxRegLoad;
 138 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 139 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 140 import org.opendaylight.yangtools.yang.common.RpcResult;
 141 import org.opendaylight.yangtools.yang.common.Uint32;
 142 import org.opendaylight.yangtools.yang.common.Uint64;
 143 import org.slf4j.Logger;
 144 import org.slf4j.LoggerFactory;
 145
 146 @Singleton
 147 public class NexthopManager implements AutoCloseable {
 148     private static final Logger LOG = LoggerFactory.getLogger(NexthopManager.class);
 149     private static final String NEXTHOP_ID_POOL_NAME = "nextHopPointerPool";
 150     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 151     private static final long WAIT_TIME_TO_ACQUIRE_LOCK = 3000L;
 152     private static final int SELECT_GROUP_WEIGHT = 1;
 153     private static final int RETRY_COUNT = 6;
 154     private static final String NEXTHOPMANAGER_JOB_KEY_PREFIX = "NextHopManager";
 155
 156     private final DataBroker dataBroker;
 157     private final ManagedNewTransactionRunner txRunner;
 158     private final IMdsalApiManager mdsalApiManager;
 159     private final OdlInterfaceRpcService odlInterfaceRpcService;
 160     private final ItmRpcService itmManager;
 161     private final IdManagerService idManager;
 162     private final IElanService elanService;
 163     private final LockManagerService lockManager;
 164     private final SalGroupService salGroupService;
 165     private final JobCoordinator jobCoordinator;
 166     private final FibUtil fibUtil;
 167     private final IInterfaceManager interfaceManager;
 168     private volatile L3VPNTransportTypes configuredTransportTypeL3VPN = L3VPNTransportTypes.Invalid;
 169
 170     /**
 171      * Provides nexthop functions.
 172      * Creates group ID pool
 173      *
 174      * @param dataBroker       - dataBroker reference
 175      * @param mdsalApiManager  - mdsalApiManager reference
 176      * @param idManager        - idManager reference
 177      * @param odlInterfaceRpcService - odlInterfaceRpcService reference
 178      * @param itmManager       - itmManager reference
 179      */
 180     @Inject
 181     public NexthopManager(final DataBroker dataBroker,
 182                           final IMdsalApiManager mdsalApiManager,
 183                           final IdManagerService idManager,
 184                           final OdlInterfaceRpcService odlInterfaceRpcService,
 185                           final ItmRpcService itmManager,
 186                           final LockManagerService lockManager,
 187                           final IElanService elanService,
 188                           final SalGroupService salGroupService,
 189                           final JobCoordinator jobCoordinator,
 190                           final FibUtil fibUtil,
 191                           final IInterfaceManager interfaceManager) {
 192         this.dataBroker = dataBroker;
 193         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 194         this.mdsalApiManager = mdsalApiManager;
 195         this.idManager = idManager;
 196         this.odlInterfaceRpcService = odlInterfaceRpcService;
 197         this.itmManager = itmManager;
 198         this.elanService = elanService;
 199         this.salGroupService = salGroupService;
 200         this.jobCoordinator = jobCoordinator;
 201         this.fibUtil = fibUtil;
 202         this.lockManager = lockManager;
 203         this.interfaceManager = interfaceManager;
 204         createIdPool();
 205     }
 206
 207     private void createIdPool() {
 208         CreateIdPoolInput createPool = new CreateIdPoolInputBuilder()
 209             .setPoolName(NEXTHOP_ID_POOL_NAME)
 210             .setLow(150000L)
 211             .setHigh(175000L)
 212             .build();
 213         try {
 214             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPool);
 215             if (result != null && result.get().isSuccessful()) {
 216                 LOG.info("Created IdPool for NextHopPointerPool");
 217             }
 218         } catch (InterruptedException | ExecutionException e) {
 219             LOG.error("Failed to create idPool for NextHopPointerPool", e);
 220         }
 221     }
 222
 223     private String getNextHopKey(Uint32 vpnId, String ipAddress) {
 224         return "nexthop." + vpnId + ipAddress;
 225     }
 226
 227     String getRemoteSelectGroupKey(Uint32 vpnId, String ipAddress) {
 228         return "remote.ecmp.nexthop." + vpnId + ipAddress;
 229     }
 230
 231     String getLocalSelectGroupKey(Uint32 vpnId, String ipAddress) {
 232         return "local.ecmp.nexthop." + vpnId + ipAddress;
 233     }
 234
 235     public ItmRpcService getItmManager() {
 236         return itmManager;
 237     }
 238
 239     protected long createNextHopPointer(String nexthopKey) {
 240         AllocateIdInput getIdInput = new AllocateIdInputBuilder()
 241             .setPoolName(NEXTHOP_ID_POOL_NAME).setIdKey(nexthopKey)
 242             .build();
 243         //TODO: Proper error handling once IdManager code is complete
 244         try {
 245             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
 246             RpcResult<AllocateIdOutput> rpcResult = result.get();
 247             return rpcResult.getResult().getIdValue().toJava();
 248         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 249             // FIXME: NPEs should not be caught but rather their root cause should be eliminated
 250             LOG.trace("Failed to allocate {}", getIdInput, e);
 251         }
 252         return 0;
 253     }
 254
 255     protected void removeNextHopPointer(String nexthopKey) {
 256         ReleaseIdInput idInput = new ReleaseIdInputBuilder()
 257             .setPoolName(NEXTHOP_ID_POOL_NAME)
 258             .setIdKey(nexthopKey).build();
 259         try {
 260             RpcResult<ReleaseIdOutput> rpcResult = idManager.releaseId(idInput).get();
 261             if (!rpcResult.isSuccessful()) {
 262                 LOG.error("RPC Call to Get Unique Id for nexthopKey {} returned with Errors {}",
 263                         nexthopKey, rpcResult.getErrors());
 264             }
 265         } catch (InterruptedException | ExecutionException e) {
 266             LOG.warn("Exception when getting Unique Id for key {}", nexthopKey, e);
 267         }
 268     }
 269
 270     protected List<ActionInfo> getEgressActionsForInterface(final String ifName, int actionKey,
 271                                                             boolean isTunnelInterface,
 272                                                             Uint32 vpnId, String destIpPrefix) {
 273         List<Action> actions;
 274         try {
 275             if (isTunnelInterface && interfaceManager.isItmDirectTunnelsEnabled()) {
 276                 RpcResult<GetEgressActionsForTunnelOutput> rpcResult =
 277                         itmManager.getEgressActionsForTunnel(new GetEgressActionsForTunnelInputBuilder()
 278                                 .setIntfName(ifName).build()).get();
 279                 if (!rpcResult.isSuccessful()) {
 280                     LOG.error("RPC Call to Get egress tunnel actions for interface {} returned with Errors {}",
 281                             ifName, rpcResult.getErrors());
 282                     return Collections.emptyList();
 283                 } else {
 284                     actions = rpcResult.getResult().nonnullAction();
 285                 }
 286             } else {
 287                 RpcResult<GetEgressActionsForInterfaceOutput> rpcResult = odlInterfaceRpcService
 288                         .getEgressActionsForInterface(new GetEgressActionsForInterfaceInputBuilder()
 289                                 .setIntfName(ifName).build()).get();
 290                 if (!rpcResult.isSuccessful()) {
 291                     LOG.error("RPC Call to Get egress vm actions for interface {} vpnId {} ipPrefix {} returned with "
 292                                     + "Errors {}", ifName, vpnId, destIpPrefix, rpcResult.getErrors());
 293                     return Collections.emptyList();
 294                 } else {
 295                     actions = rpcResult.getResult().nonnullAction();
 296                 }
 297             }
 298             List<ActionInfo> listActionInfo = new ArrayList<>();
 299             for (Action action : actions) {
 300                 actionKey = action.key().getOrder() + actionKey;
 301                 org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action
 302                     actionClass = action.getAction();
 303                 if (actionClass instanceof OutputActionCase) {
 304                     listActionInfo.add(new ActionOutput(actionKey,
 305                         ((OutputActionCase) actionClass).getOutputAction().getOutputNodeConnector()));
 306                 } else if (actionClass instanceof PushVlanActionCase) {
 307                     listActionInfo.add(new ActionPushVlan(actionKey));
 308                 } else if (actionClass instanceof SetFieldCase) {
 309                     if (((SetFieldCase) actionClass).getSetField().getVlanMatch() != null) {
 310                         int vlanVid = ((SetFieldCase) actionClass).getSetField().getVlanMatch()
 311                             .getVlanId().getVlanId().getValue().toJava();
 312                         listActionInfo.add(new ActionSetFieldVlanVid(actionKey, vlanVid));
 313                     }
 314                 } else if (actionClass instanceof NxActionResubmitRpcAddGroupCase) {
 315                     Short tableId = ((NxActionResubmitRpcAddGroupCase) actionClass).getNxResubmit().getTable().toJava();
 316                     listActionInfo.add(new ActionNxResubmit(actionKey, tableId));
 317                 } else if (actionClass instanceof NxActionRegLoadNodesNodeTableFlowApplyActionsCase) {
 318                     NxRegLoad nxRegLoad =
 319                         ((NxActionRegLoadNodesNodeTableFlowApplyActionsCase) actionClass).getNxRegLoad();
 320                     listActionInfo.add(new ActionRegLoad(actionKey, NxmNxReg6.class,
 321                         nxRegLoad.getDst().getStart().toJava(), nxRegLoad.getDst().getEnd().toJava(),
 322                         nxRegLoad.getValue().longValue()));
 323                 }
 324             }
 325             return listActionInfo;
 326         } catch (InterruptedException | ExecutionException | NullPointerException e) {
 327             LOG.error("Exception when egress actions for interface {} isTunnel {} vpnId {} ipPrefix {}", ifName,
 328                     isTunnelInterface, vpnId, destIpPrefix, e);
 329         }
 330         LOG.warn("Exception when egress actions for interface {}", ifName);
 331         return Collections.emptyList();
 332     }
 333
 334     @Nullable
 335     protected String getTunnelInterfaceName(Uint64 srcDpId, Uint64 dstDpId) {
 336         Class<? extends TunnelTypeBase> tunType = getReqTunType(getReqTransType().toUpperCase(Locale.getDefault()));
 337         Future<RpcResult<GetTunnelInterfaceNameOutput>> result;
 338         try {
 339             result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 340                 .setSourceDpid(srcDpId)
 341                 .setDestinationDpid(dstDpId)
 342                 .setTunnelType(tunType)
 343                 .build());
 344             RpcResult<GetTunnelInterfaceNameOutput> rpcResult = result.get();
 345             if (!rpcResult.isSuccessful()) {
 346                 LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
 347             } else {
 348                 return rpcResult.getResult().getInterfaceName();
 349             }
 350         } catch (InterruptedException | ExecutionException e) {
 351             LOG.warn("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstDpId, e);
 352         }
 353         return null;
 354     }
 355
 356     @Nullable
 357     protected String getTunnelInterfaceName(Uint64 srcDpId, org.opendaylight.yang.gen.v1.urn.ietf.params
 358         .xml.ns.yang.ietf.inet.types.rev130715.IpAddress dstIp, Class<? extends TunnelTypeBase> tunnelType) {
 359         Future<RpcResult<GetInternalOrExternalInterfaceNameOutput>> result;
 360         try {
 361             LOG.debug("Trying to fetch tunnel interface name for source dpn {} destIp {} tunType {}", srcDpId,
 362                     dstIp.stringValue(), tunnelType.getName());
 363             result = itmManager.getInternalOrExternalInterfaceName(new GetInternalOrExternalInterfaceNameInputBuilder()
 364                 .setSourceDpid(srcDpId)
 365                 .setDestinationIp(dstIp)
 366                 .setTunnelType(tunnelType)
 367                 .build());
 368             RpcResult<GetInternalOrExternalInterfaceNameOutput> rpcResult = result.get();
 369             if (!rpcResult.isSuccessful()) {
 370                 LOG.warn("RPC Call to getTunnelInterfaceName returned with Errors {}", rpcResult.getErrors());
 371             } else {
 372                 return rpcResult.getResult().getInterfaceName();
 373             }
 374         } catch (InterruptedException | ExecutionException e) {
 375             LOG.error("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstIp, e);
 376         }
 377         return null;
 378     }
 379
 380
 381     public long getLocalNextHopGroup(Uint32 vpnId,
 382             String ipNextHopAddress) {
 383         long groupId = createNextHopPointer(getNextHopKey(vpnId, ipNextHopAddress));
 384         if (groupId == FibConstants.INVALID_GROUP_ID) {
 385             LOG.error("Unable to allocate groupId for vpnId {} , prefix {}", vpnId, ipNextHopAddress);
 386         }
 387         return groupId;
 388     }
 389
 390     public long getLocalSelectGroup(Uint32 vpnId,
 391             String ipNextHopAddress) {
 392         long groupId = createNextHopPointer(getLocalSelectGroupKey(vpnId, ipNextHopAddress));
 393         if (groupId == FibConstants.INVALID_GROUP_ID) {
 394             LOG.error("Unable to allocate groupId for vpnId {} , prefix {}", vpnId, ipNextHopAddress);
 395         }
 396         return groupId;
 397     }
 398
 399     public long createLocalNextHop(Uint32 vpnId, Uint64 dpnId, String ifName,
 400                                    String primaryIpAddress, String currDestIpPrefix,
 401                                    String gwMacAddress, Uint32 parentVpnId) {
 402         //For VPN Imported routes, getting VPN Instance name using parentVpnId
 403         String vpnName = parentVpnId != null ? fibUtil.getVpnNameFromId(parentVpnId) : fibUtil.getVpnNameFromId(vpnId);
 404         if (vpnName == null) {
 405             return 0;
 406         }
 407         String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, primaryIpAddress);
 408
 409         long groupId = createNextHopPointer(getNextHopKey(vpnId, primaryIpAddress));
 410         if (groupId == 0) {
 411             LOG.error("Unable to allocate groupId for vpnId {} , IntfName {}, primaryIpAddress {} curIpPrefix {}",
 412                     vpnId, ifName, primaryIpAddress, currDestIpPrefix);
 413             return groupId;
 414         }
 415         String nextHopLockStr = vpnId + primaryIpAddress;
 416         String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, currDestIpPrefix);
 417         jobCoordinator.enqueueJob(jobKey, () -> {
 418             try {
 419                 if (FibUtil.lockCluster(lockManager, nextHopLockStr, WAIT_TIME_TO_ACQUIRE_LOCK)) {
 420                     VpnNexthop nexthop = getVpnNexthop(vpnId, primaryIpAddress);
 421                     LOG.trace("nexthop: {} retrieved for vpnId {}, prefix {}, ifName {} on dpn {}", nexthop, vpnId,
 422                             primaryIpAddress, ifName, dpnId);
 423                     if (nexthop == null) {
 424                         String encMacAddress = macAddress == null
 425                                 ? fibUtil.getMacAddressFromPrefix(ifName, vpnName, primaryIpAddress) : macAddress;
 426                         List<ActionInfo> listActionInfo = new ArrayList<>();
 427                         int actionKey = 0;
 428                         // MAC re-write
 429                         if (encMacAddress != null) {
 430                             if (gwMacAddress != null) {
 431                                 LOG.trace("The Local NextHop Group Source Mac {} for VpnInterface {} on VPN {}",
 432                                         gwMacAddress, ifName, vpnId);
 433                                 listActionInfo.add(new ActionSetFieldEthernetSource(actionKey++,
 434                                         new MacAddress(gwMacAddress)));
 435                             }
 436                             listActionInfo.add(new ActionSetFieldEthernetDestination(actionKey++,
 437                                     new MacAddress(encMacAddress)));
 438                             // listActionInfo.add(0, new ActionPopMpls());
 439                         } else {
 440                             LOG.error("mac address for new local nexthop group {} is null for vpnId {}, prefix {}, "
 441                                     + "ifName {} on dpn {}", groupId, vpnId, primaryIpAddress, ifName, dpnId);
 442                         }
 443                         List<ActionInfo> nhActionInfoList = getEgressActionsForInterface(ifName, actionKey, false,
 444                                 vpnId, currDestIpPrefix);
 445                         if (nhActionInfoList.isEmpty()) {
 446                             LOG.error("createLocalNextHop: Skipping, Empty list of egress actions received for "
 447                                     + "interface {} on dpn {} for vpn {} prefix {}", ifName, dpnId, vpnId,
 448                                     currDestIpPrefix);
 449                         }
 450                         listActionInfo.addAll(nhActionInfoList);
 451                         BucketInfo bucket = new BucketInfo(listActionInfo);
 452                         List<BucketInfo> listBucketInfo = new ArrayList<>();
 453                         listBucketInfo.add(bucket);
 454                         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, groupId, primaryIpAddress,
 455                                 GroupTypes.GroupAll, listBucketInfo);
 456                         LOG.trace("Install LNH Group: id {}, mac address {}, interface {} for prefix {}", groupId,
 457                                 encMacAddress, ifName, primaryIpAddress);
 458                         //Try to install group directly on the DPN bypassing the FRM, in order to avoid waiting for the
 459                         // group to get installed before programming the flows
 460                         installGroupOnDpn(groupId, dpnId, primaryIpAddress, listBucketInfo,
 461                                 getNextHopKey(vpnId, primaryIpAddress), GroupTypes.GroupAll);
 462                         // install Group
 463                         mdsalApiManager.syncInstallGroup(groupEntity);
 464                         // update MD-SAL DS
 465                         addVpnNexthopToDS(dpnId, vpnId, primaryIpAddress, currDestIpPrefix, groupId);
 466
 467                     } else {
 468                         // Ignore adding new prefix , if it already exists
 469                         List<IpAdjacencies> prefixesList = nexthop.getIpAdjacencies();
 470                         IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currDestIpPrefix).build();
 471                         if (prefixesList != null && prefixesList.contains(prefix)) {
 472                             LOG.trace("Prefix {} is already present in l3nextHop {} ", currDestIpPrefix, nexthop);
 473                         } else {
 474                             IpAdjacenciesBuilder ipPrefixesBuilder =
 475                                     new IpAdjacenciesBuilder().withKey(new IpAdjacenciesKey(currDestIpPrefix));
 476                             LOG.trace("Updating prefix {} to vpnNextHop {} Operational DS", currDestIpPrefix, nexthop);
 477                             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
 478                                     getVpnNextHopIpPrefixIdentifier(vpnId, primaryIpAddress, currDestIpPrefix),
 479                                     ipPrefixesBuilder.build());
 480                         }
 481                     }
 482                 }
 483             } finally {
 484                 FibUtil.unlockCluster(lockManager, nextHopLockStr);
 485             }
 486             return Collections.emptyList();
 487         });
 488         return groupId;
 489     }
 490
 491     private void installGroupOnDpn(long groupId, Uint64 dpnId, String groupName, List<BucketInfo> bucketsInfo,
 492                                      String nextHopKey, GroupTypes groupType) {
 493         NodeRef nodeRef = FibUtil.buildNodeRef(dpnId);
 494         Buckets buckets = FibUtil.buildBuckets(bucketsInfo);
 495         GroupRef groupRef = new GroupRef(FibUtil.buildGroupInstanceIdentifier(groupId, dpnId));
 496         AddGroupInput input = new AddGroupInputBuilder().setNode(nodeRef).setGroupId(new GroupId(groupId))
 497                 .setBuckets(buckets).setGroupRef(groupRef).setGroupType(groupType)
 498                 .setGroupName(groupName).build();
 499         Future<RpcResult<AddGroupOutput>> groupStats = salGroupService.addGroup(input);
 500         RpcResult<AddGroupOutput> rpcResult = null;
 501         try {
 502             rpcResult = groupStats.get(WAIT_TIME_FOR_SYNC_INSTALL, TimeUnit.MILLISECONDS);
 503             if (rpcResult != null && rpcResult.isSuccessful()) {
 504                 LOG.info("installGroupOnDpn: Group {} with key {} has been successfully installed directly on dpn {}.",
 505                         groupId, nextHopKey, dpnId);
 506             } else {
 507                 LOG.error("installGroupOnDpn: Unable to install group {} with key {} directly on dpn {} due to {}.",
 508                         groupId, nextHopKey, dpnId, rpcResult != null ? rpcResult.getErrors() : null);
 509             }
 510         } catch (InterruptedException | ExecutionException e) {
 511             LOG.error("installGroupOnDpn: Error while installing group {} directly on dpn {}", groupId, dpnId);
 512         } catch (TimeoutException e) {
 513             LOG.error("installGroupOnDpn: Group {} installation on dpn {} timed out.", groupId, dpnId);
 514         }
 515     }
 516
 517     protected void addVpnNexthopToDS(Uint64 dpnId, Uint32 vpnId, String primaryIpAddr,
 518                                      String currIpAddr, long egressPointer) {
 519         InstanceIdentifierBuilder<VpnNexthops> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
 520             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId));
 521
 522         List<IpAdjacencies> ipPrefixesList = new ArrayList<>();
 523         IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currIpAddr).build();
 524         ipPrefixesList.add(prefix);
 525         // Add nexthop to vpn node
 526         VpnNexthop nh = new VpnNexthopBuilder()
 527             .withKey(new VpnNexthopKey(primaryIpAddr))
 528             .setDpnId(dpnId)
 529             .setIpAdjacencies(ipPrefixesList)
 530             .setEgressPointer(egressPointer).build();
 531
 532         InstanceIdentifier<VpnNexthop> id1 = idBuilder
 533             .child(VpnNexthop.class, new VpnNexthopKey(primaryIpAddr)).build();
 534         LOG.trace("Adding vpnnextHop {} to Operational DS", nh);
 535         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id1, nh);
 536
 537     }
 538
 539     protected InstanceIdentifier<IpAdjacencies> getVpnNextHopIpPrefixIdentifier(Uint32 vpnId, String primaryIpAddress,
 540                                                                                 String ipPrefix) {
 541         InstanceIdentifier<IpAdjacencies> id = InstanceIdentifier.builder(L3nexthop.class)
 542                 .child(VpnNexthops.class, new VpnNexthopsKey(vpnId))
 543                 .child(VpnNexthop.class, new VpnNexthopKey(primaryIpAddress))
 544                 .child(IpAdjacencies.class, new IpAdjacenciesKey(ipPrefix)).build();
 545         return id;
 546     }
 547
 548     @Nullable
 549     protected VpnNexthop getVpnNexthop(Uint32 vpnId, String ipAddress) {
 550
 551         // check if vpn node is there
 552         InstanceIdentifierBuilder<VpnNexthops> idBuilder =
 553             InstanceIdentifier.builder(L3nexthop.class).child(VpnNexthops.class,
 554                 new VpnNexthopsKey(vpnId));
 555         InstanceIdentifier<VpnNexthops> id = idBuilder.build();
 556         Optional<VpnNexthops> vpnNexthops = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 557         if (vpnNexthops.isPresent()) {
 558             // get nexthops list for vpn
 559             List<VpnNexthop> nexthops = vpnNexthops.get().nonnullVpnNexthop();
 560             for (VpnNexthop nexthop : nexthops) {
 561                 if (Objects.equals(nexthop.getIpAddress(), ipAddress)) {
 562                     // return nexthop
 563                     LOG.trace("VpnNextHop : {}", nexthop);
 564                     return nexthop;
 565                 }
 566             }
 567             // return null if not found
 568         }
 569         return null;
 570     }
 571
 572     @Nullable
 573     public AdjacencyResult getRemoteNextHopPointer(Uint64 remoteDpnId, Uint32 vpnId, String prefixIp,
 574             @Nullable String nextHopIp, Class<? extends TunnelTypeBase> tunnelType) {
 575         String egressIfName = null;
 576         LOG.trace("getRemoteNextHopPointer: input [remoteDpnId {}, vpnId {}, prefixIp {}, nextHopIp {} ]", remoteDpnId,
 577             vpnId, prefixIp, nextHopIp);
 578
 579         Class<? extends InterfaceType> egressIfType;
 580         ElanInstance elanInstance = getElanInstanceForPrefix(vpnId, prefixIp);
 581         if (elanInstance != null) {
 582             egressIfType = getInterfaceType(elanInstance);
 583         } else {
 584             LOG.warn("Failed to determine network type for prefixIp {} using tunnel", prefixIp);
 585             egressIfType = Tunnel.class;
 586         }
 587
 588         if (Tunnel.class.equals(egressIfType)) {
 589             egressIfName = getTunnelRemoteNextHopPointer(remoteDpnId, nextHopIp, tunnelType);
 590         } else {
 591             egressIfName = getExtPortRemoteNextHopPointer(remoteDpnId, elanInstance);
 592         }
 593
 594         LOG.trace("NextHop pointer for prefixIp {} vpnId {} dpnId {} is {}", prefixIp, vpnId, remoteDpnId,
 595             egressIfName);
 596         return egressIfName != null ? new AdjacencyResult(egressIfName, egressIfType, nextHopIp,
 597                 prefixIp) : null;
 598     }
 599
 600     private void removeVpnNexthopFromDS(Uint32 vpnId, String ipPrefix) {
 601
 602         InstanceIdentifierBuilder<VpnNexthop> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
 603             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId))
 604             .child(VpnNexthop.class, new VpnNexthopKey(ipPrefix));
 605         InstanceIdentifier<VpnNexthop> id = idBuilder.build();
 606         // remove from DS
 607         LOG.trace("Removing vpn next hop from datastore : {}", id);
 608         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 609     }
 610
 611     public void removeLocalNextHop(Uint64 dpnId, Uint32 vpnId, String primaryIpAddress, String currDestIpPrefix) {
 612         String nextHopLockStr = vpnId + primaryIpAddress;
 613         try {
 614             if (FibUtil.lockCluster(lockManager, nextHopLockStr, WAIT_TIME_TO_ACQUIRE_LOCK)) {
 615                 VpnNexthop nh = getVpnNexthop(vpnId, primaryIpAddress);
 616                 if (nh != null) {
 617                     List<IpAdjacencies> prefixesList = new ArrayList<>(nh.nonnullIpAdjacencies());
 618                     IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currDestIpPrefix).build();
 619                     prefixesList.remove(prefix);
 620                     if (prefixesList.isEmpty()) { //remove the group only if there are no more flows using this group
 621                         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, nh.getEgressPointer().toJava(),
 622                                 primaryIpAddress, GroupTypes.GroupAll, Collections.emptyList());
 623                         // remove Group ...
 624                         mdsalApiManager.removeGroup(groupEntity);
 625                         //update MD-SAL DS
 626                         removeVpnNexthopFromDS(vpnId, primaryIpAddress);
 627                         //release groupId
 628                         removeNextHopPointer(getNextHopKey(vpnId, primaryIpAddress));
 629                         LOG.debug("Local Next hop {} for {} {} on dpn {} successfully deleted",
 630                                 nh.getEgressPointer(), vpnId, primaryIpAddress, dpnId);
 631                     } else {
 632                         //remove the currIpPrefx from IpPrefixList of the vpnNexthop
 633                         LOG.trace("Removing the prefix {} from vpnNextHop {} Operational DS", currDestIpPrefix, nh);
 634                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
 635                                 getVpnNextHopIpPrefixIdentifier(vpnId, primaryIpAddress, currDestIpPrefix));
 636                     }
 637                 } else {
 638                     //throw error
 639                     LOG.error("Local NextHop for VpnId {} curIpPrefix {} on dpn {} primaryIpAddress {} not deleted",
 640                             vpnId, currDestIpPrefix, dpnId, primaryIpAddress);
 641                 }
 642             }
 643         } finally {
 644             FibUtil.unlockCluster(lockManager, nextHopLockStr);
 645         }
 646     }
 647
 648     public void setConfTransType(String service, String transportType) {
 649
 650         if (!service.equalsIgnoreCase("L3VPN")) {
 651             LOG.error("Incorrect service {} provided for setting the transport type.", service);
 652             return;
 653         }
 654
 655         L3VPNTransportTypes transType = L3VPNTransportTypes.validateTransportType(transportType
 656                 .toUpperCase(Locale.getDefault()));
 657
 658         if (transType != L3VPNTransportTypes.Invalid) {
 659             configuredTransportTypeL3VPN = transType;
 660         }
 661     }
 662
 663     public void writeConfTransTypeConfigDS() {
 664         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, getConfTransportTypeIdentifier(),
 665             createConfTransportType(configuredTransportTypeL3VPN.getTransportType()));
 666     }
 667
 668     public L3VPNTransportTypes getConfiguredTransportTypeL3VPN() {
 669         return this.configuredTransportTypeL3VPN;
 670     }
 671
 672     public String getReqTransType() {
 673         if (configuredTransportTypeL3VPN == L3VPNTransportTypes.Invalid) {
 674             /*
 675              * Restart scenario, Read from the ConfigDS.
 676              * if the value is Unset, cache value as VxLAN.
 677              */
 678             LOG.trace("configureTransportType is not yet set.");
 679             Optional<ConfTransportTypeL3vpn> configuredTransTypeFromConfig =
 680                 MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, getConfTransportTypeIdentifier());
 681
 682             if (configuredTransTypeFromConfig.isPresent()) {
 683                 if (TunnelTypeGre.class.equals(configuredTransTypeFromConfig.get().getTransportType())) {
 684                     configuredTransportTypeL3VPN = L3VPNTransportTypes.GRE;
 685                 } else {
 686                     configuredTransportTypeL3VPN = L3VPNTransportTypes.VxLAN;
 687                 }
 688                 LOG.trace("configuredTransportType set from config DS to {}",
 689                     getConfiguredTransportTypeL3VPN().getTransportType());
 690             } else {
 691                 setConfTransType("L3VPN", L3VPNTransportTypes.VxLAN.getTransportType());
 692                 LOG.trace("configuredTransportType is not set in the Config DS. VxLAN as default will be used.");
 693             }
 694         } else {
 695             LOG.trace("configuredTransportType is set as {}", getConfiguredTransportTypeL3VPN().getTransportType());
 696         }
 697         return getConfiguredTransportTypeL3VPN().getTransportType();
 698     }
 699
 700     public InstanceIdentifier<ConfTransportTypeL3vpn> getConfTransportTypeIdentifier() {
 701         return InstanceIdentifier.builder(ConfTransportTypeL3vpn.class).build();
 702     }
 703
 704     private ConfTransportTypeL3vpn createConfTransportType(String type) {
 705         ConfTransportTypeL3vpn confTransType;
 706         switch (type) {
 707             case ITMConstants.TUNNEL_TYPE_GRE:
 708                 confTransType = new ConfTransportTypeL3vpnBuilder().setTransportType(TunnelTypeGre.class).build();
 709                 LOG.trace("Setting the confTransportType to GRE.");
 710                 break;
 711             case ITMConstants.TUNNEL_TYPE_VXLAN:
 712                 confTransType = new ConfTransportTypeL3vpnBuilder().setTransportType(TunnelTypeVxlan.class).build();
 713                 LOG.trace("Setting the confTransportType to VxLAN.");
 714                 break;
 715             default:
 716                 LOG.trace("Invalid transport type {} passed to Config DS ", type);
 717                 confTransType = null;
 718                 break;
 719         }
 720         return confTransType;
 721     }
 722
 723     public Class<? extends TunnelTypeBase> getReqTunType(String transportType) {
 724         switch (transportType) {
 725             case "VXLAN":
 726                 return TunnelTypeVxlan.class;
 727             case "GRE":
 728                 return TunnelTypeGre.class;
 729             default:
 730                 return TunnelTypeMplsOverGre.class;
 731         }
 732     }
 733
 734     public String getTransportTypeStr(String tunType) {
 735         if (tunType.equals(TunnelTypeVxlan.class.toString())) {
 736             return ITMConstants.TUNNEL_TYPE_VXLAN;
 737         } else if (tunType.equals(TunnelTypeGre.class.toString())) {
 738             return ITMConstants.TUNNEL_TYPE_GRE;
 739         } else if (tunType.equals(TunnelTypeMplsOverGre.class.toString())) {
 740             return ITMConstants.TUNNEL_TYPE_MPLSoGRE;
 741         } else {
 742             return ITMConstants.TUNNEL_TYPE_INVALID;
 743         }
 744     }
 745
 746     @Override
 747     @PreDestroy
 748     public void close() {
 749         LOG.info("{} close", getClass().getSimpleName());
 750     }
 751
 752     // TODO Clean up the exception handling
 753     @SuppressWarnings("checkstyle:IllegalCatch")
 754     @Nullable
 755     private String getTunnelRemoteNextHopPointer(Uint64 remoteDpnId, String nextHopIp,
 756                                                  Class<? extends TunnelTypeBase> tunnelType) {
 757         if (nextHopIp != null && !nextHopIp.isEmpty()) {
 758             try {
 759                 // here use the config for tunnel type param
 760                 return getTunnelInterfaceName(remoteDpnId,
 761                     org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder
 762                         .getDefaultInstance(nextHopIp), tunnelType);
 763             } catch (Exception ex) {
 764                 LOG.error("Error while retrieving nexthop pointer for nexthop {} remoteDpn {}",
 765                         nextHopIp, remoteDpnId, ex);
 766             }
 767         }
 768
 769         return null;
 770     }
 771
 772     private String getExtPortRemoteNextHopPointer(Uint64 remoteDpnId, ElanInstance elanInstance) {
 773         return elanService.getExternalElanInterface(elanInstance.getElanInstanceName(), remoteDpnId);
 774     }
 775
 776     /**
 777      * Get the interface type associated with the type of ELAN used for routing
 778      * traffic to/from remote compute nodes.
 779      *
 780      * @param elanInstance The elan instance
 781      * @return L2vlan for flat/VLAN network type and Tunnel otherwise
 782      */
 783     private Class<? extends InterfaceType> getInterfaceType(ElanInstance elanInstance) {
 784         Class<? extends SegmentTypeBase> segmentType = elanInstance.getSegmentType();
 785         if (SegmentTypeFlat.class.equals(segmentType) || SegmentTypeVlan.class.equals(segmentType)) {
 786             return L2vlan.class;
 787         }
 788
 789         return Tunnel.class;
 790     }
 791
 792     private ElanInstance getElanInstanceForPrefix(Uint32 vpnId, String prefixIp) {
 793         ElanInstance elanInstance = null;
 794         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, prefixIp);
 795         if (prefix != null) {
 796             if (prefix.getNetworkId() != null) {
 797                 elanInstance = elanService.getElanInstance(prefix.getNetworkId().getValue());
 798             }
 799         }
 800
 801         return elanInstance;
 802     }
 803
 804     static class AdjacencyResult {
 805         private final String interfaceName;
 806         private final Class<? extends InterfaceType> interfaceType;
 807         private final String nextHopIp;
 808         private final String prefix;
 809
 810         AdjacencyResult(String interfaceName, Class<? extends InterfaceType> interfaceType, String nextHopIp,
 811                         String prefix) {
 812             this.interfaceName = interfaceName;
 813             this.interfaceType = interfaceType;
 814             this.nextHopIp = nextHopIp;
 815             this.prefix = prefix;
 816         }
 817
 818         public String getInterfaceName() {
 819             return interfaceName;
 820         }
 821
 822         public Class<? extends InterfaceType> getInterfaceType() {
 823             return interfaceType;
 824         }
 825
 826         public String getNextHopIp() {
 827             return nextHopIp;
 828         }
 829
 830         public String getPrefix() {
 831             return prefix;
 832         }
 833
 834         @Override
 835         public int hashCode() {
 836             final int prime = 31;
 837             int result = 1;
 838             result = prime * result + (interfaceName == null ? 0 : interfaceName.hashCode());
 839             return result;
 840         }
 841
 842         @Override
 843         public boolean equals(Object obj) {
 844             if (obj == null) {
 845                 return false;
 846             }
 847
 848             if (getClass() != obj.getClass()) {
 849                 return false;
 850             } else {
 851                 AdjacencyResult other = (AdjacencyResult) obj;
 852                 return interfaceName.equals(other.interfaceName);
 853             }
 854         }
 855     }
 856
 857     protected long setupLoadBalancingNextHop(Uint32 parentVpnId, Uint64 dpnId,
 858             String destPrefix, List<BucketInfo> localBucketInfo, List<BucketInfo> remoteBucketInfo) {
 859         long remoteGroupId = createNextHopPointer(getRemoteSelectGroupKey(parentVpnId, destPrefix));
 860         if (remoteGroupId == FibConstants.INVALID_GROUP_ID) {
 861             LOG.error("Unable to allocate/retrieve remote groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 862             return remoteGroupId;
 863         }
 864         long localGroupId =  createNextHopPointer(getLocalSelectGroupKey(parentVpnId, destPrefix));
 865         if (localGroupId == FibConstants.INVALID_GROUP_ID) {
 866             LOG.error("Unable to allocate/retrieve local groupId for vpnId {} , prefix {}",
 867                 parentVpnId, destPrefix);
 868             return remoteGroupId;
 869         }
 870         List<BucketInfo> combinedBucketInfo = new ArrayList<>();
 871         combinedBucketInfo.addAll(localBucketInfo);
 872         combinedBucketInfo.addAll(remoteBucketInfo);
 873         GroupEntity remoteGroupEntity = MDSALUtil.buildGroupEntity(
 874                 dpnId, remoteGroupId, destPrefix, GroupTypes.GroupSelect, combinedBucketInfo);
 875         GroupEntity localGroupEntity = MDSALUtil.buildGroupEntity(
 876                 dpnId, localGroupId, destPrefix, GroupTypes.GroupSelect, localBucketInfo);
 877         String jobKey = FibUtil.getCreateLocalNextHopJobKey(parentVpnId, dpnId, destPrefix);
 878         jobCoordinator.enqueueJob(jobKey, () -> {
 879             mdsalApiManager.syncInstallGroup(remoteGroupEntity);
 880             if (!localBucketInfo.isEmpty()) {
 881                 mdsalApiManager.syncInstallGroup(localGroupEntity);
 882             }
 883             if (LOG.isDebugEnabled()) {
 884                 LOG.debug("Finished installing GroupEntity with jobCoordinator key {} remoteGroupEntity.groupId {}"
 885                         + "localGroupEntity.groupId {}  groupEntity.groupType {}", jobKey,
 886                         remoteGroupEntity.getGroupId(), localGroupEntity.getGroupId(),
 887                         remoteGroupEntity.getGroupType());
 888             }
 889             // Delete local group(if exists) if there is no local info.
 890             // Local group has to be deleted if all VMs in a compute is deleted.
 891             if (localBucketInfo.isEmpty()) {
 892                 LOG.debug("Deleting local group {} since no local nhs present for "
 893                         + "prefix {}", localGroupEntity.getGroupId(), destPrefix);
 894                 mdsalApiManager.syncRemoveGroup(localGroupEntity);
 895             }
 896             return Collections.emptyList();
 897         });
 898         return remoteGroupId;
 899     }
 900
 901     protected void deleteLoadBalancingNextHop(Uint32 parentVpnId, Uint64 dpnId, String destPrefix) {
 902         long remoteGroupId = createNextHopPointer(getRemoteSelectGroupKey(parentVpnId, destPrefix));
 903         if (remoteGroupId == FibConstants.INVALID_GROUP_ID) {
 904             LOG.error("Unable to allocate/retrieve remote groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 905         }
 906         long localGroupId = createNextHopPointer(getLocalSelectGroupKey(parentVpnId, destPrefix));
 907         if (localGroupId == FibConstants.INVALID_GROUP_ID) {
 908             LOG.error("Unable to allocate/retrieve local groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 909         }
 910         GroupEntity remoteGroupEntity = MDSALUtil.buildGroupEntity(
 911                 dpnId, remoteGroupId, destPrefix, GroupTypes.GroupSelect, Collections.emptyList());
 912         GroupEntity localGroupEntity = MDSALUtil.buildGroupEntity(
 913                 dpnId, localGroupId, destPrefix, GroupTypes.GroupSelect, Collections.emptyList());
 914         String jobKey = FibUtil.getCreateLocalNextHopJobKey(parentVpnId, dpnId, destPrefix);
 915         jobCoordinator.enqueueJob(jobKey, () -> {
 916             mdsalApiManager.syncRemoveGroup(remoteGroupEntity);
 917             mdsalApiManager.syncRemoveGroup(localGroupEntity);
 918             if (LOG.isDebugEnabled()) {
 919                 LOG.debug("Finished removing GroupEntity with jobCoordinator key {} remoteGroupEntity.groupId {}"
 920                     + "localGroupEntity.groupId {}", jobKey, remoteGroupId, localGroupId);
 921             }
 922             return Collections.emptyList();
 923         });
 924     }
 925
 926     long createNextHopGroups(Uint32 vpnId, String rd, Uint64 dpnId, VrfEntry vrfEntry,
 927             @Nullable Routes routes, List<Routes> vpnExtraRoutes) {
 928         List<BucketInfo> localBucketInfo = new ArrayList<>();
 929         List<Routes> clonedVpnExtraRoutes  = new ArrayList<>(vpnExtraRoutes);
 930         if (clonedVpnExtraRoutes.contains(routes)) {
 931             localBucketInfo.addAll(getBucketsForLocalNexthop(vpnId, dpnId, vrfEntry, routes));
 932             clonedVpnExtraRoutes.remove(routes);
 933         }
 934         List<BucketInfo> remoteBucketInfo =
 935             new ArrayList<>(getBucketsForRemoteNexthop(vpnId, dpnId, vrfEntry, rd, clonedVpnExtraRoutes));
 936         return setupLoadBalancingNextHop(vpnId, dpnId,
 937             vrfEntry.getDestPrefix(), localBucketInfo, remoteBucketInfo);
 938     }
 939
 940     private List<BucketInfo> getBucketsForLocalNexthop(Uint32 vpnId, Uint64 dpnId,
 941             VrfEntry vrfEntry, Routes routes) {
 942         @Nullable List<String> nexthopIpList = routes.getNexthopIpList();
 943         if (LOG.isDebugEnabled()) {
 944             LOG.debug("NexthopManager.getBucketsForLocalNexthop invoked with vpnId {} dpnId {} "
 945                             + " vrfEntry.routePaths {}, routes.nexthopList {}", vpnId, dpnId, vrfEntry.getRoutePaths(),
 946                 nexthopIpList);
 947         }
 948         List<BucketInfo> listBucketInfo = new CopyOnWriteArrayList<>();
 949         if (nexthopIpList != null) {
 950             nexthopIpList.parallelStream().forEach(nextHopIp -> {
 951                 String localNextHopIP;
 952                 if (isIpv4Address(nextHopIp)) {
 953                     localNextHopIP = nextHopIp + NwConstants.IPV4PREFIX;
 954                 } else {
 955                     localNextHopIP = nextHopIp + NwConstants.IPV6PREFIX;
 956                 }
 957                 Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 958                 if (localNextHopInfo != null) {
 959                     long groupId = getLocalNextHopGroup(vpnId, localNextHopIP);
 960                     if (groupId == FibConstants.INVALID_GROUP_ID) {
 961                         LOG.error("Unable to allocate groupId for vpnId {} , prefix {} , interface {}", vpnId,
 962                             vrfEntry.getDestPrefix(), localNextHopInfo.getVpnInterfaceName());
 963                         return;
 964                     }
 965                     List<ActionInfo> actionsInfos =
 966                         Collections.singletonList(new ActionGroup(groupId));
 967                     BucketInfo bucket = new BucketInfo(actionsInfos);
 968                     bucket.setWeight(1);
 969                     listBucketInfo.add(bucket);
 970                 }
 971             });
 972         }
 973         LOG.trace("LOCAL: listbucket {}, vpnId {}, dpnId {}, routes {}", listBucketInfo, vpnId, dpnId, routes);
 974         return listBucketInfo;
 975     }
 976
 977     private List<BucketInfo> getBucketsForRemoteNexthop(Uint32 vpnId, Uint64 dpnId, VrfEntry vrfEntry, String rd,
 978             List<Routes> vpnExtraRoutes) {
 979         List<BucketInfo> listBucketInfo = new ArrayList<>();
 980         Map<String, List<ActionInfo>> egressActionMap = new HashMap<>();
 981         vpnExtraRoutes.stream().filter(vpnExtraRoute -> vpnExtraRoute.getNexthopIpList() != null).forEach(
 982             vpnExtraRoute -> vpnExtraRoute.getNexthopIpList().forEach(nextHopIp -> {
 983                 String nextHopPrefixIp;
 984                 if (isIpv4Address(nextHopIp)) {
 985                     nextHopPrefixIp = nextHopIp + NwConstants.IPV4PREFIX;
 986                 } else {
 987                     nextHopPrefixIp = nextHopIp + NwConstants.IPV6PREFIX;
 988                 }
 989                 List<String> tepIpAddresses = fibUtil.getNextHopAddresses(rd, nextHopPrefixIp);
 990                 if (tepIpAddresses.isEmpty()) {
 991                     return;
 992                 }
 993                 // There would be only one nexthop address for a VM ip which would be the tep Ip
 994                 String tepIp = tepIpAddresses.get(0);
 995                 AdjacencyResult adjacencyResult = getRemoteNextHopPointer(dpnId, vpnId,
 996                     vrfEntry.getDestPrefix(), tepIp, TunnelTypeVxlan.class);
 997                 if (adjacencyResult == null) {
 998                     return;
 999                 }
1000                 String egressInterface = adjacencyResult.getInterfaceName();
1001                 if (!FibUtil.isTunnelInterface(adjacencyResult)) {
1002                     return;
1003                 }
1004                 Class<? extends TunnelTypeBase> tunnelType =
1005                     VpnExtraRouteHelper.getTunnelType(itmManager, egressInterface);
1006                 StateTunnelList ifState = null;
1007                 try {
1008                     ifState = fibUtil.getTunnelState(egressInterface);
1009                     if (ifState == null || ifState.getOperState() != TunnelOperStatus.Up) {
1010                         LOG.trace("Tunnel is not up for interface {}", egressInterface);
1011                         return;
1012                     }
1013                 } catch (ReadFailedException e) {
1014                     LOG.error("getBucketsForRemoteNexthop: error in fetching tunnel state for interface {}",
1015                         egressInterface, e);
1016                     return;
1017                 }
1018                 if (!TunnelTypeVxlan.class.equals(tunnelType)) {
1019                     return;
1020                 }
1021                 Uint32 label = FibUtil.getLabelFromRoutePaths(vrfEntry).get();
1022                 Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, nextHopPrefixIp);
1023                 if (prefixInfo == null) {
1024                     LOG.error("No prefix info found for prefix {} in rd {} for VPN {}", nextHopPrefixIp, rd,
1025                         vpnId);
1026                     return;
1027                 }
1028                 Uint32 tunnelId;
1029                 if (FibUtil.isVxlanNetwork(prefixInfo.getNetworkType())) {
1030                     tunnelId = prefixInfo.getSegmentationId();
1031                 } else {
1032                     LOG.warn("Network is not of type VXLAN for prefix {}."
1033                         + "Going with default Lport Tag.", prefixInfo.toString());
1034                     tunnelId = label;
1035                 }
1036                 List<ActionInfo> actionInfos = new ArrayList<>();
1037                 actionInfos.add(new ActionSetFieldTunnelId(Uint64.valueOf(tunnelId.longValue())));
1038                 String ifName = prefixInfo.getVpnInterfaceName();
1039                 String vpnName = fibUtil.getVpnNameFromId(vpnId);
1040                 if (vpnName == null) {
1041                     return;
1042                 }
1043                 String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, nextHopPrefixIp);
1044                 actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
1045                     new MacAddress(macAddress)));
1046                 List<ActionInfo> egressActions;
1047                 if (egressActionMap.containsKey(egressInterface)) {
1048                     egressActions = egressActionMap.get(egressInterface);
1049                 } else {
1050                     egressActions = getEgressActionsForInterface(egressInterface, actionInfos.size(),
1051                             true, vpnId, vrfEntry.getDestPrefix());
1052                     if (egressActions.isEmpty()) {
1053                         LOG.error("Skipping getBucketsForRemoteNexthop: Empty list of egress actions received for "
1054                                         + "interface {} on dpn {} for vpn {} prefix {} nextHop {}", ifName, dpnId,
1055                                 vpnId, vrfEntry.getDestPrefix(), nextHopPrefixIp);
1056                     }
1057                     egressActionMap.put(egressInterface, egressActions);
1058                 }
1059                 if (egressActions.isEmpty()) {
1060                     LOG.error("Failed to retrieve egress action for prefix {} route-paths {}"
1061                             + " interface {}." + " Aborting remote FIB entry creation.",
1062                         vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), egressInterface);
1063                 }
1064                 actionInfos.addAll(egressActions);
1065                 BucketInfo bucket = new BucketInfo(actionInfos);
1066                 bucket.setWeight(1);
1067                 listBucketInfo.add(bucket);
1068             }));
1069         LOG.trace("LOCAL: listbucket {}, rd {}, dpnId {}, routes {}", listBucketInfo, rd, dpnId, vpnExtraRoutes);
1070         return listBucketInfo;
1071     }
1072
1073     public void createDcGwLoadBalancingGroup(Uint64 dpnId, String destinationIp,
1074                                              Class<? extends TunnelTypeBase> tunnelType) {
1075         jobCoordinator.enqueueJob(getJobKey(dpnId), () -> {
1076             List<ListenableFuture<Void>> futures = new ArrayList<>();
1077             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operationalTx -> {
1078                 synchronized (getDcGateWaySyncKey(destinationIp)) {
1079                     FibUtil.addL3vpnDcGateWay(destinationIp, operationalTx);
1080                 }
1081                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, configTx -> {
1082                     List<String> availableDcGws = getDcGwIps();
1083                     Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
1084                     int noOfDcGws = availableDcGws.size();
1085                     if (noOfDcGws == 1) {
1086                         LOG.trace("There are no enough DC GateWays {} present to program LB group", availableDcGws);
1087                         return;
1088                     }
1089                     if (availableDcGws.contains(destinationIp)) {
1090                         availableDcGws.remove(destinationIp);
1091                     }
1092                     availableDcGws.forEach(dcGwIp -> {
1093                         List<String> dcGws = Arrays.asList(dcGwIp, destinationIp);
1094                         Collections.sort(dcGws);
1095                         String groupIdKey = FibUtil.getGreLbGroupKey(dcGws);
1096                         Long groupId = createNextHopPointer(groupIdKey);
1097                         List<Bucket> listBucket = new ArrayList<>();
1098                         for (int index = 0; index < dcGws.size(); index++) {
1099                             if (isTunnelUp(dcGws.get(index), dpnId, tunnelType)) {
1100                                 listBucket.add(buildBucketForDcGwLbGroup(dcGws.get(index),
1101                                         dpnId, index, tunnelType, true));
1102                             }
1103                         }
1104                         Group group = MDSALUtil.buildGroup(groupId, groupIdKey, GroupTypes.GroupSelect,
1105                                         MDSALUtil.buildBucketLists(listBucket));
1106                         mdsalApiManager.addGroup(configTx, dpnId, group);
1107                         FibUtil.updateLbGroupInfo(dpnId, groupIdKey, groupId.toString(), operationalTx);
1108                         LOG.trace("LB group {} towards DC-GW installed on dpn {}. Group - {}",
1109                                 groupIdKey, dpnId, group);
1110                     });
1111                 }));
1112             }));
1113             return futures;
1114         }, RETRY_COUNT);
1115     }
1116
1117     private String getJobKey(Uint64 dpnId) {
1118         return new StringBuilder().append(NEXTHOPMANAGER_JOB_KEY_PREFIX).append(dpnId).toString();
1119     }
1120
1121     private String getDcGateWaySyncKey(String destinationIp) {
1122         String mutex = new StringBuilder().append("L3vpncDcGateWay").append(destinationIp).toString();
1123         return mutex.intern();
1124     }
1125
1126     private List<String> getDcGwIps() {
1127         InstanceIdentifier<DcGatewayIpList> dcGatewayIpListid =
1128                 InstanceIdentifier.builder(DcGatewayIpList.class).build();
1129         DcGatewayIpList dcGatewayIpListConfig =
1130                 MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, dcGatewayIpListid).orNull();
1131         if (dcGatewayIpListConfig == null) {
1132             return Collections.emptyList();
1133         }
1134         return dcGatewayIpListConfig.getDcGatewayIp()
1135                 .stream()
1136                 .filter(dcGwIp -> dcGwIp.getTunnnelType().equals(TunnelTypeMplsOverGre.class))
1137                 .map(dcGwIp -> dcGwIp.getIpAddress().stringValue()).sorted()
1138                 .collect(toList());
1139     }
1140
1141     private boolean isTunnelUp(String dcGwIp, Uint64 dpnId, Class<? extends TunnelTypeBase> tunnelType) {
1142         String tunnelName = getTunnelRemoteNextHopPointer(dpnId, dcGwIp, tunnelType);
1143         if (tunnelName != null) {
1144             InstanceIdentifier<StateTunnelList> tunnelStateId =
1145                     InstanceIdentifier.builder(TunnelsState.class).child(
1146                             StateTunnelList.class, new StateTunnelListKey(tunnelName)).build();
1147             return MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, tunnelStateId)
1148                     .toJavaUtil().map(StateTunnelList::getOperState)
1149                     .orElse(TunnelOperStatus.Down) == TunnelOperStatus.Up;
1150         }
1151         return false;
1152     }
1153
1154     private List<Action> getEgressActions(String interfaceName, int actionKey) {
1155         List<Action> actions = Collections.emptyList();
1156         try {
1157             GetEgressActionsForInterfaceInputBuilder egressAction =
1158                     new GetEgressActionsForInterfaceInputBuilder().setIntfName(interfaceName).setActionKey(actionKey);
1159             Future<RpcResult<GetEgressActionsForInterfaceOutput>> result =
1160                     odlInterfaceRpcService.getEgressActionsForInterface(egressAction.build());
1161             RpcResult<GetEgressActionsForInterfaceOutput> rpcResult = result.get();
1162             if (!rpcResult.isSuccessful()) {
1163                 LOG.warn("RPC Call to Get egress actions for interface {} returned with Errors {}",
1164                         interfaceName, rpcResult.getErrors());
1165             } else {
1166                 actions = rpcResult.getResult().nonnullAction();
1167             }
1168         } catch (InterruptedException | ExecutionException e) {
1169             LOG.warn("Exception when egress actions for interface {}", interfaceName, e);
1170         }
1171         return actions;
1172     }
1173
1174     /**
1175      * This method is invoked when the neighbor is removed from DS.
1176      * All the LB groups which point to the given destination will be deleted.
1177      */
1178     public void removeDcGwLoadBalancingGroup(Uint64 dpnId,
1179             String destinationIp) {
1180         jobCoordinator.enqueueJob(getJobKey(dpnId), () -> {
1181             List<String> availableDcGws = fibUtil.getL3VpnDcGateWays();
1182             if (availableDcGws.contains(destinationIp)) {
1183                 availableDcGws.remove(destinationIp);
1184             }
1185             List<ListenableFuture<Void>> futures = new ArrayList<>();
1186             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operationalTx -> {
1187                 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx -> {
1188                     availableDcGws.forEach(dcGwIp -> {
1189                         List<String> dcGws = Arrays.asList(dcGwIp, destinationIp);
1190                         Collections.sort(dcGws);
1191                         String nextHopKey = FibUtil.getGreLbGroupKey(dcGws);
1192                         Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
1193                         if (!optionalNextHops.isPresent()) {
1194                             return;
1195                         }
1196                         Nexthops nexthops = optionalNextHops.get();
1197                         final String groupId = nexthops.getGroupId();
1198                         final long groupIdValue = Long.parseLong(groupId);
1199                         Group group = MDSALUtil.buildGroup(groupIdValue, nextHopKey, GroupTypes.GroupSelect,
1200                                 MDSALUtil.buildBucketLists(Collections.emptyList()));
1201                         LOG.trace("Removed LB group {} on dpn {}", group, dpnId);
1202                         try {
1203                             mdsalApiManager.removeGroup(configTx, dpnId, group);
1204                         } catch (ExecutionException | InterruptedException e) {
1205                             LOG.error("Group removal failed for group {} with exception", groupId, e);
1206                         }
1207                         removeNextHopPointer(nextHopKey);
1208                         FibUtil.removeOrUpdateNextHopInfo(dpnId, nextHopKey, groupId, nexthops, operationalTx);
1209                     });
1210                     synchronized (getDcGateWaySyncKey(destinationIp)) {
1211                         FibUtil.removeL3vpnDcGateWay(destinationIp, operationalTx);
1212                     }
1213                 }));
1214             }));
1215             return futures;
1216         }, RETRY_COUNT);
1217     }
1218
1219     /**
1220      * This method is invoked when the tunnel status is deleted.
1221      * All the buckets which point to given destination will be marked down.
1222      */
1223     public void updateDcGwLoadBalancingGroup(Uint64 dpnId, String destinationIp,
1224             boolean isTunnelUp, Class<? extends TunnelTypeBase> tunnelType) {
1225         jobCoordinator.enqueueJob(getJobKey(dpnId), () -> {
1226             List<String> availableDcGws = fibUtil.getL3VpnDcGateWays();
1227             if (availableDcGws.contains(destinationIp)) {
1228                 availableDcGws.remove(destinationIp);
1229             }
1230             List<ListenableFuture<Void>> futures = new ArrayList<>();
1231             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx -> {
1232                 availableDcGws.forEach(dcGwIp -> {
1233                     List<String> dcGws = Arrays.asList(dcGwIp, destinationIp);
1234                     Collections.sort(dcGws);
1235                     String nextHopKey = FibUtil.getGreLbGroupKey(dcGws);
1236                     int bucketId = dcGws.indexOf(destinationIp);
1237                     Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
1238                     if (!optionalNextHops.isPresent()) {
1239                         return;
1240                     }
1241                     Nexthops nexthops = optionalNextHops.get();
1242                     final String groupId = nexthops.getGroupId();
1243                     final long groupIdValue = Long.parseLong(groupId);
1244                     Bucket bucket = buildBucketForDcGwLbGroup(destinationIp, dpnId, bucketId, tunnelType, isTunnelUp);
1245                     LOG.trace("updated bucket {} to group {} on dpn {}.", bucket, groupId, dpnId);
1246                     try {
1247                         mdsalApiManager.addBucket(configTx, dpnId, groupIdValue, bucket);
1248                     } catch (ExecutionException | InterruptedException e) {
1249                         LOG.error("Bucket addition failed for bucket {} with exception", bucketId, e);
1250                     }
1251                 });
1252             }));
1253             return futures;
1254         }, RETRY_COUNT);
1255     }
1256
1257     private Bucket buildBucketForDcGwLbGroup(String ipAddress, Uint64 dpnId,
1258             int index, Class<? extends TunnelTypeBase> tunnelType, boolean isTunnelUp) {
1259         List<Action> listAction = new ArrayList<>();
1260         // ActionKey 0 goes to mpls label.
1261         int actionKey = 1;
1262         listAction.add(new ActionPushMpls().buildAction());
1263         listAction.add(new ActionRegMove(actionKey++, FibConstants.NXM_REG_MAPPING
1264                 .get(index), 0, 19).buildAction());
1265         String tunnelInterfaceName = getTunnelInterfaceName(dpnId, IpAddressBuilder.getDefaultInstance(ipAddress),
1266             tunnelType);
1267         List<Action> egressActions = getEgressActions(tunnelInterfaceName, actionKey++);
1268         if (!egressActions.isEmpty()) {
1269             listAction.addAll(getEgressActions(tunnelInterfaceName, actionKey++));
1270         } else {
1271             // clear off actions if there is no egress actions.
1272             listAction = Collections.emptyList();
1273         }
1274         long watchPort = MDSALUtil.WATCH_PORT;
1275         if (!isTunnelUp) {
1276             watchPort = 0xFFFFFFFEL;
1277         }
1278         //OVS expects a non-zero weight value for load balancing to happen in select groups
1279         return MDSALUtil.buildBucket(listAction, SELECT_GROUP_WEIGHT, index,
1280                 watchPort, MDSALUtil.WATCH_GROUP);
1281     }
1282
1283     public void programDcGwLoadBalancingGroup(Uint64 dpnId, String destinationIp,
1284                                               int addRemoveOrUpdate, boolean isTunnelUp,
1285                                               Class<? extends TunnelTypeBase> tunnelType) {
1286         if (NwConstants.ADD_FLOW == addRemoveOrUpdate) {
1287             createDcGwLoadBalancingGroup(dpnId, destinationIp, tunnelType);
1288         } else if (NwConstants.DEL_FLOW == addRemoveOrUpdate) {
1289             removeDcGwLoadBalancingGroup(dpnId, destinationIp);
1290         } else if (NwConstants.MOD_FLOW == addRemoveOrUpdate) {
1291             updateDcGwLoadBalancingGroup(dpnId, destinationIp, isTunnelUp, tunnelType);
1292         }
1293     }
1294 }