fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/NexthopManager.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static java.util.stream.Collectors.toList;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  14
  15 import com.google.common.base.Optional;
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.util.concurrent.ListenableFuture;
  18
  19 import java.util.ArrayList;
  20 import java.util.Arrays;
  21 import java.util.Collections;
  22 import java.util.HashMap;
  23 import java.util.List;
  24 import java.util.Locale;
  25 import java.util.Map;
  26 import java.util.Objects;
  27 import java.util.concurrent.CopyOnWriteArrayList;
  28 import java.util.concurrent.ExecutionException;
  29 import java.util.concurrent.Future;
  30
  31 import javax.annotation.PreDestroy;
  32 import javax.inject.Inject;
  33 import javax.inject.Singleton;
  34
  35 import org.eclipse.jdt.annotation.Nullable;
  36 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  37 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  38 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  39 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  40 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  41 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  42 import org.opendaylight.genius.itm.globals.ITMConstants;
  43 import org.opendaylight.genius.mdsalutil.ActionInfo;
  44 import org.opendaylight.genius.mdsalutil.BucketInfo;
  45 import org.opendaylight.genius.mdsalutil.GroupEntity;
  46 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  47 import org.opendaylight.genius.mdsalutil.NwConstants;
  48 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  49 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  50 import org.opendaylight.genius.mdsalutil.actions.ActionOutput;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionPushVlan;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  54 import org.opendaylight.genius.mdsalutil.actions.ActionRegMove;
  55 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  56 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetSource;
  57 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  58 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldVlanVid;
  59 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  60 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  61 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  62 import org.opendaylight.netvirt.fibmanager.api.L3VPNTransportTypes;
  63 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  64 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev170119.L2vlan;
  65 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev170119.Tunnel;
  66 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
  67 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfaceType;
  68 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.OutputActionCase;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.PushVlanActionCase;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.SetFieldCase;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInput;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInputBuilder;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutput;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInput;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInputBuilder;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolOutput;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInput;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInputBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdOutput;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceInputBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceOutput;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelOperStatus;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelsState;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelListKey;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rev160406.DcGatewayIpList;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelInputBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelOutput;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameInputBuilder;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameOutput;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.lockmanager.rev160413.LockManagerService;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupInput;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupInputBuilder;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupOutput;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.SalGroupService;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupId;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupRef;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.Buckets;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeRef;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeBase;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeFlat;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeVlan;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.L3nexthop;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthops;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthopsKey;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopKey;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacencies;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacenciesBuilder;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacenciesKey;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpn;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpnBuilder;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.lb.nexthops.Nexthops;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.add.group.input.buckets.bucket.action.action.NxActionResubmitRpcAddGroupCase;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.nodes.node.table.flow.instructions.instruction.instruction.apply.actions._case.apply.actions.action.action.NxActionRegLoadNodesNodeTableFlowApplyActionsCase;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.nx.action.reg.load.grouping.NxRegLoad;
 137 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 138 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 139 import org.opendaylight.yangtools.yang.common.RpcResult;
 140 import org.opendaylight.yangtools.yang.common.Uint32;
 141 import org.opendaylight.yangtools.yang.common.Uint64;
 142 import org.slf4j.Logger;
 143 import org.slf4j.LoggerFactory;
 144
 145 @Singleton
 146 public class NexthopManager implements AutoCloseable {
 147     private static final Logger LOG = LoggerFactory.getLogger(NexthopManager.class);
 148     private static final String NEXTHOP_ID_POOL_NAME = "nextHopPointerPool";
 149     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 150     private static final long WAIT_TIME_TO_ACQUIRE_LOCK = 3000L;
 151     private static final int SELECT_GROUP_WEIGHT = 1;
 152     private static final int RETRY_COUNT = 6;
 153     private static final String NEXTHOPMANAGER_JOB_KEY_PREFIX = "NextHopManager";
 154
 155     private final DataBroker dataBroker;
 156     private final ManagedNewTransactionRunner txRunner;
 157     private final IMdsalApiManager mdsalApiManager;
 158     private final OdlInterfaceRpcService odlInterfaceRpcService;
 159     private final ItmRpcService itmManager;
 160     private final IdManagerService idManager;
 161     private final IElanService elanService;
 162     private final LockManagerService lockManager;
 163     private final SalGroupService salGroupService;
 164     private final JobCoordinator jobCoordinator;
 165     private final FibUtil fibUtil;
 166     private final IInterfaceManager interfaceManager;
 167     private volatile L3VPNTransportTypes configuredTransportTypeL3VPN = L3VPNTransportTypes.Invalid;
 168
 169     /**
 170      * Provides nexthop functions.
 171      * Creates group ID pool
 172      *
 173      * @param dataBroker       - dataBroker reference
 174      * @param mdsalApiManager  - mdsalApiManager reference
 175      * @param idManager        - idManager reference
 176      * @param odlInterfaceRpcService - odlInterfaceRpcService reference
 177      * @param itmManager       - itmManager reference
 178      */
 179     @Inject
 180     public NexthopManager(final DataBroker dataBroker,
 181                           final IMdsalApiManager mdsalApiManager,
 182                           final IdManagerService idManager,
 183                           final OdlInterfaceRpcService odlInterfaceRpcService,
 184                           final ItmRpcService itmManager,
 185                           final LockManagerService lockManager,
 186                           final IElanService elanService,
 187                           final SalGroupService salGroupService,
 188                           final JobCoordinator jobCoordinator,
 189                           final FibUtil fibUtil,
 190                           final IInterfaceManager interfaceManager) {
 191         this.dataBroker = dataBroker;
 192         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 193         this.mdsalApiManager = mdsalApiManager;
 194         this.idManager = idManager;
 195         this.odlInterfaceRpcService = odlInterfaceRpcService;
 196         this.itmManager = itmManager;
 197         this.elanService = elanService;
 198         this.salGroupService = salGroupService;
 199         this.jobCoordinator = jobCoordinator;
 200         this.fibUtil = fibUtil;
 201         this.lockManager = lockManager;
 202         this.interfaceManager = interfaceManager;
 203         createIdPool();
 204     }
 205
 206     private void createIdPool() {
 207         CreateIdPoolInput createPool = new CreateIdPoolInputBuilder()
 208             .setPoolName(NEXTHOP_ID_POOL_NAME)
 209             .setLow(150000L)
 210             .setHigh(175000L)
 211             .build();
 212         try {
 213             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPool);
 214             if (result != null && result.get().isSuccessful()) {
 215                 LOG.info("Created IdPool for NextHopPointerPool");
 216             }
 217         } catch (InterruptedException | ExecutionException e) {
 218             LOG.error("Failed to create idPool for NextHopPointerPool", e);
 219         }
 220     }
 221
 222     private String getNextHopKey(Uint32 vpnId, String ipAddress) {
 223         return "nexthop." + vpnId + ipAddress;
 224     }
 225
 226     String getRemoteSelectGroupKey(Uint32 vpnId, String ipAddress) {
 227         return "remote.ecmp.nexthop." + vpnId + ipAddress;
 228     }
 229
 230     String getLocalSelectGroupKey(Uint32 vpnId, String ipAddress) {
 231         return "local.ecmp.nexthop." + vpnId + ipAddress;
 232     }
 233
 234     public ItmRpcService getItmManager() {
 235         return itmManager;
 236     }
 237
 238     protected long createNextHopPointer(String nexthopKey) {
 239         AllocateIdInput getIdInput = new AllocateIdInputBuilder()
 240             .setPoolName(NEXTHOP_ID_POOL_NAME).setIdKey(nexthopKey)
 241             .build();
 242         //TODO: Proper error handling once IdManager code is complete
 243         try {
 244             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
 245             RpcResult<AllocateIdOutput> rpcResult = result.get();
 246             return rpcResult.getResult().getIdValue().toJava();
 247         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 248             // FIXME: NPEs should not be caught but rather their root cause should be eliminated
 249             LOG.trace("Failed to allocate {}", getIdInput, e);
 250         }
 251         return 0;
 252     }
 253
 254     protected void removeNextHopPointer(String nexthopKey) {
 255         ReleaseIdInput idInput = new ReleaseIdInputBuilder()
 256             .setPoolName(NEXTHOP_ID_POOL_NAME)
 257             .setIdKey(nexthopKey).build();
 258         try {
 259             RpcResult<ReleaseIdOutput> rpcResult = idManager.releaseId(idInput).get();
 260             if (!rpcResult.isSuccessful()) {
 261                 LOG.error("RPC Call to Get Unique Id for nexthopKey {} returned with Errors {}",
 262                         nexthopKey, rpcResult.getErrors());
 263             }
 264         } catch (InterruptedException | ExecutionException e) {
 265             LOG.warn("Exception when getting Unique Id for key {}", nexthopKey, e);
 266         }
 267     }
 268
 269     protected List<ActionInfo> getEgressActionsForInterface(final String ifName, int actionKey,
 270                                                             boolean isTunnelInterface,
 271                                                             Uint32 vpnId, String destIpPrefix) {
 272         List<Action> actions;
 273         try {
 274             if (isTunnelInterface && interfaceManager.isItmDirectTunnelsEnabled()) {
 275                 RpcResult<GetEgressActionsForTunnelOutput> rpcResult =
 276                         itmManager.getEgressActionsForTunnel(new GetEgressActionsForTunnelInputBuilder()
 277                                 .setIntfName(ifName).build()).get();
 278                 if (!rpcResult.isSuccessful()) {
 279                     LOG.error("RPC Call to Get egress tunnel actions for interface {} returned with Errors {}",
 280                             ifName, rpcResult.getErrors());
 281                     return Collections.emptyList();
 282                 } else {
 283                     actions = rpcResult.getResult().nonnullAction();
 284                 }
 285             } else {
 286                 RpcResult<GetEgressActionsForInterfaceOutput> rpcResult = odlInterfaceRpcService
 287                         .getEgressActionsForInterface(new GetEgressActionsForInterfaceInputBuilder()
 288                                 .setIntfName(ifName).build()).get();
 289                 if (!rpcResult.isSuccessful()) {
 290                     LOG.error("RPC Call to Get egress vm actions for interface {} vpnId {} ipPrefix {} returned with "
 291                                     + "Errors {}", ifName, vpnId, destIpPrefix, rpcResult.getErrors());
 292                     return Collections.emptyList();
 293                 } else {
 294                     actions = rpcResult.getResult().nonnullAction();
 295                 }
 296             }
 297             List<ActionInfo> listActionInfo = new ArrayList<>();
 298             for (Action action : actions) {
 299                 actionKey = action.key().getOrder() + actionKey;
 300                 org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action
 301                     actionClass = action.getAction();
 302                 if (actionClass instanceof OutputActionCase) {
 303                     listActionInfo.add(new ActionOutput(actionKey,
 304                         ((OutputActionCase) actionClass).getOutputAction().getOutputNodeConnector()));
 305                 } else if (actionClass instanceof PushVlanActionCase) {
 306                     listActionInfo.add(new ActionPushVlan(actionKey));
 307                 } else if (actionClass instanceof SetFieldCase) {
 308                     if (((SetFieldCase) actionClass).getSetField().getVlanMatch() != null) {
 309                         int vlanVid = ((SetFieldCase) actionClass).getSetField().getVlanMatch()
 310                             .getVlanId().getVlanId().getValue().toJava();
 311                         listActionInfo.add(new ActionSetFieldVlanVid(actionKey, vlanVid));
 312                     }
 313                 } else if (actionClass instanceof NxActionResubmitRpcAddGroupCase) {
 314                     Short tableId = ((NxActionResubmitRpcAddGroupCase) actionClass).getNxResubmit().getTable().toJava();
 315                     listActionInfo.add(new ActionNxResubmit(actionKey, tableId));
 316                 } else if (actionClass instanceof NxActionRegLoadNodesNodeTableFlowApplyActionsCase) {
 317                     NxRegLoad nxRegLoad =
 318                         ((NxActionRegLoadNodesNodeTableFlowApplyActionsCase) actionClass).getNxRegLoad();
 319                     listActionInfo.add(new ActionRegLoad(actionKey, NxmNxReg6.class,
 320                         nxRegLoad.getDst().getStart().toJava(), nxRegLoad.getDst().getEnd().toJava(),
 321                         nxRegLoad.getValue().longValue()));
 322                 }
 323             }
 324             return listActionInfo;
 325         } catch (InterruptedException | ExecutionException | NullPointerException e) {
 326             LOG.error("Exception when egress actions for interface {} isTunnel {} vpnId {} ipPrefix {}", ifName,
 327                     isTunnelInterface, vpnId, destIpPrefix, e);
 328         }
 329         LOG.warn("Exception when egress actions for interface {}", ifName);
 330         return Collections.emptyList();
 331     }
 332
 333     @Nullable
 334     protected String getTunnelInterfaceName(Uint64 srcDpId, Uint64 dstDpId) {
 335         Class<? extends TunnelTypeBase> tunType = getReqTunType(getReqTransType().toUpperCase(Locale.getDefault()));
 336         Future<RpcResult<GetTunnelInterfaceNameOutput>> result;
 337         try {
 338             result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 339                 .setSourceDpid(srcDpId)
 340                 .setDestinationDpid(dstDpId)
 341                 .setTunnelType(tunType)
 342                 .build());
 343             RpcResult<GetTunnelInterfaceNameOutput> rpcResult = result.get();
 344             if (!rpcResult.isSuccessful()) {
 345                 LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
 346             } else {
 347                 return rpcResult.getResult().getInterfaceName();
 348             }
 349         } catch (InterruptedException | ExecutionException e) {
 350             LOG.warn("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstDpId, e);
 351         }
 352         return null;
 353     }
 354
 355     @Nullable
 356     protected String getTunnelInterfaceName(Uint64 srcDpId, org.opendaylight.yang.gen.v1.urn.ietf.params
 357         .xml.ns.yang.ietf.inet.types.rev130715.IpAddress dstIp, Class<? extends TunnelTypeBase> tunnelType) {
 358         Future<RpcResult<GetInternalOrExternalInterfaceNameOutput>> result;
 359         try {
 360             LOG.debug("Trying to fetch tunnel interface name for source dpn {} destIp {} tunType {}", srcDpId,
 361                     dstIp.stringValue(), tunnelType.getName());
 362             result = itmManager.getInternalOrExternalInterfaceName(new GetInternalOrExternalInterfaceNameInputBuilder()
 363                 .setSourceDpid(srcDpId)
 364                 .setDestinationIp(dstIp)
 365                 .setTunnelType(tunnelType)
 366                 .build());
 367             RpcResult<GetInternalOrExternalInterfaceNameOutput> rpcResult = result.get();
 368             if (!rpcResult.isSuccessful()) {
 369                 LOG.warn("RPC Call to getTunnelInterfaceName returned with Errors {}", rpcResult.getErrors());
 370             } else {
 371                 return rpcResult.getResult().getInterfaceName();
 372             }
 373         } catch (InterruptedException | ExecutionException e) {
 374             LOG.error("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstIp, e);
 375         }
 376         return null;
 377     }
 378
 379
 380     public long getLocalNextHopGroup(Uint32 vpnId,
 381             String ipNextHopAddress) {
 382         long groupId = createNextHopPointer(getNextHopKey(vpnId, ipNextHopAddress));
 383         if (groupId == FibConstants.INVALID_GROUP_ID) {
 384             LOG.error("Unable to allocate groupId for vpnId {} , prefix {}", vpnId, ipNextHopAddress);
 385         }
 386         return groupId;
 387     }
 388
 389     public long getLocalSelectGroup(Uint32 vpnId,
 390             String ipNextHopAddress) {
 391         long groupId = createNextHopPointer(getLocalSelectGroupKey(vpnId, ipNextHopAddress));
 392         if (groupId == FibConstants.INVALID_GROUP_ID) {
 393             LOG.error("Unable to allocate groupId for vpnId {} , prefix {}", vpnId, ipNextHopAddress);
 394         }
 395         return groupId;
 396     }
 397
 398     public long createLocalNextHop(Uint32 vpnId, Uint64 dpnId, String ifName,
 399                                    String primaryIpAddress, String currDestIpPrefix,
 400                                    String gwMacAddress) {
 401         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 402         if (vpnName == null) {
 403             return 0;
 404         }
 405         String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, primaryIpAddress);
 406
 407         long groupId = createNextHopPointer(getNextHopKey(vpnId, primaryIpAddress));
 408         if (groupId == 0) {
 409             LOG.error("Unable to allocate groupId for vpnId {} , IntfName {}, primaryIpAddress {} curIpPrefix {}",
 410                     vpnId, ifName, primaryIpAddress, currDestIpPrefix);
 411             return groupId;
 412         }
 413         String nextHopLockStr = vpnId + primaryIpAddress;
 414         String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, currDestIpPrefix);
 415         jobCoordinator.enqueueJob(jobKey, () -> {
 416             try {
 417                 if (FibUtil.lockCluster(lockManager, nextHopLockStr, WAIT_TIME_TO_ACQUIRE_LOCK)) {
 418                     VpnNexthop nexthop = getVpnNexthop(vpnId, primaryIpAddress);
 419                     LOG.trace("nexthop: {} retrieved for vpnId {}, prefix {}, ifName {} on dpn {}", nexthop, vpnId,
 420                             primaryIpAddress, ifName, dpnId);
 421                     if (nexthop == null) {
 422                         String encMacAddress = macAddress == null
 423                                 ? fibUtil.getMacAddressFromPrefix(ifName, vpnName, primaryIpAddress) : macAddress;
 424                         List<ActionInfo> listActionInfo = new ArrayList<>();
 425                         int actionKey = 0;
 426                         // MAC re-write
 427                         if (encMacAddress != null) {
 428                             if (gwMacAddress != null) {
 429                                 LOG.trace("The Local NextHop Group Source Mac {} for VpnInterface {} on VPN {}",
 430                                         gwMacAddress, ifName, vpnId);
 431                                 listActionInfo.add(new ActionSetFieldEthernetSource(actionKey++,
 432                                         new MacAddress(gwMacAddress)));
 433                             }
 434                             listActionInfo.add(new ActionSetFieldEthernetDestination(actionKey++,
 435                                     new MacAddress(encMacAddress)));
 436                             // listActionInfo.add(0, new ActionPopMpls());
 437                         } else {
 438                             // FIXME: Log message here.
 439                             LOG.debug("mac address for new local nexthop is null");
 440                         }
 441                         List<ActionInfo> nhActionInfoList = getEgressActionsForInterface(ifName, actionKey, false,
 442                                 vpnId, currDestIpPrefix);
 443                         if (nhActionInfoList.isEmpty()) {
 444                             LOG.error("createLocalNextHop: Skipping, Empty list of egress actions received for "
 445                                     + "interface {} on dpn {} for vpn {} prefix {}", ifName, dpnId, vpnId,
 446                                     currDestIpPrefix);
 447                         }
 448                         listActionInfo.addAll(nhActionInfoList);
 449                         BucketInfo bucket = new BucketInfo(listActionInfo);
 450                         List<BucketInfo> listBucketInfo = new ArrayList<>();
 451                         listBucketInfo.add(bucket);
 452                         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, groupId, primaryIpAddress,
 453                                 GroupTypes.GroupAll, listBucketInfo);
 454                         LOG.trace("Install LNH Group: id {}, mac address {}, interface {} for prefix {}", groupId,
 455                                 encMacAddress, ifName, primaryIpAddress);
 456                         //Try to install group directly on the DPN bypassing the FRM, in order to avoid waiting for the
 457                         // group to get installed before programming the flows
 458                         installGroupOnDpn(groupId, dpnId, primaryIpAddress, listBucketInfo,
 459                                 getNextHopKey(vpnId, primaryIpAddress), GroupTypes.GroupAll);
 460                         // install Group
 461                         mdsalApiManager.syncInstallGroup(groupEntity);
 462                         // update MD-SAL DS
 463                         addVpnNexthopToDS(dpnId, vpnId, primaryIpAddress, currDestIpPrefix, groupId);
 464
 465                     } else {
 466                         // Ignore adding new prefix , if it already exists
 467                         List<IpAdjacencies> prefixesList = nexthop.getIpAdjacencies();
 468                         IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currDestIpPrefix).build();
 469                         if (prefixesList != null && prefixesList.contains(prefix)) {
 470                             LOG.trace("Prefix {} is already present in l3nextHop {} ", currDestIpPrefix, nexthop);
 471                         } else {
 472                             IpAdjacenciesBuilder ipPrefixesBuilder =
 473                                     new IpAdjacenciesBuilder().withKey(new IpAdjacenciesKey(currDestIpPrefix));
 474                             LOG.trace("Updating prefix {} to vpnNextHop {} Operational DS", currDestIpPrefix, nexthop);
 475                             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
 476                                     getVpnNextHopIpPrefixIdentifier(vpnId, primaryIpAddress, currDestIpPrefix),
 477                                     ipPrefixesBuilder.build());
 478                         }
 479                     }
 480                 }
 481             } finally {
 482                 FibUtil.unlockCluster(lockManager, nextHopLockStr);
 483             }
 484             return Collections.emptyList();
 485         });
 486         return groupId;
 487     }
 488
 489     private void installGroupOnDpn(long groupId, Uint64 dpnId, String groupName, List<BucketInfo> bucketsInfo,
 490                                      String nextHopKey, GroupTypes groupType) {
 491         NodeRef nodeRef = FibUtil.buildNodeRef(dpnId);
 492         Buckets buckets = FibUtil.buildBuckets(bucketsInfo);
 493         GroupRef groupRef = new GroupRef(FibUtil.buildGroupInstanceIdentifier(groupId, dpnId));
 494         AddGroupInput input = new AddGroupInputBuilder().setNode(nodeRef).setGroupId(new GroupId(groupId))
 495                 .setBuckets(buckets).setGroupRef(groupRef).setGroupType(groupType)
 496                 .setGroupName(groupName).build();
 497         Future<RpcResult<AddGroupOutput>> groupStats = salGroupService.addGroup(input);
 498         RpcResult<AddGroupOutput> rpcResult = null;
 499         try {
 500             rpcResult = groupStats.get();
 501             if (rpcResult != null && rpcResult.isSuccessful()) {
 502                 LOG.info("Group {} with key {} has been successfully installed directly on dpn {}.", groupId,
 503                         nextHopKey, dpnId);
 504             } else {
 505                 LOG.error("Unable to install group {} with key {} directly on dpn {} due to {}.", groupId, nextHopKey,
 506                         dpnId, rpcResult != null ? rpcResult.getErrors() : null);
 507             }
 508         } catch (InterruptedException | ExecutionException e) {
 509             LOG.error("Error while installing group {} directly on dpn {}", groupId, dpnId);
 510         }
 511     }
 512
 513     protected void addVpnNexthopToDS(Uint64 dpnId, Uint32 vpnId, String primaryIpAddr,
 514                                      String currIpAddr, long egressPointer) {
 515         InstanceIdentifierBuilder<VpnNexthops> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
 516             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId));
 517
 518         List<IpAdjacencies> ipPrefixesList = new ArrayList<>();
 519         IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currIpAddr).build();
 520         ipPrefixesList.add(prefix);
 521         // Add nexthop to vpn node
 522         VpnNexthop nh = new VpnNexthopBuilder()
 523             .withKey(new VpnNexthopKey(primaryIpAddr))
 524             .setDpnId(dpnId)
 525             .setIpAdjacencies(ipPrefixesList)
 526             .setEgressPointer(egressPointer).build();
 527
 528         InstanceIdentifier<VpnNexthop> id1 = idBuilder
 529             .child(VpnNexthop.class, new VpnNexthopKey(primaryIpAddr)).build();
 530         LOG.trace("Adding vpnnextHop {} to Operational DS", nh);
 531         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id1, nh);
 532
 533     }
 534
 535     protected InstanceIdentifier<IpAdjacencies> getVpnNextHopIpPrefixIdentifier(Uint32 vpnId, String primaryIpAddress,
 536                                                                                 String ipPrefix) {
 537         InstanceIdentifier<IpAdjacencies> id = InstanceIdentifier.builder(L3nexthop.class)
 538                 .child(VpnNexthops.class, new VpnNexthopsKey(vpnId))
 539                 .child(VpnNexthop.class, new VpnNexthopKey(primaryIpAddress))
 540                 .child(IpAdjacencies.class, new IpAdjacenciesKey(ipPrefix)).build();
 541         return id;
 542     }
 543
 544     @Nullable
 545     protected VpnNexthop getVpnNexthop(Uint32 vpnId, String ipAddress) {
 546
 547         // check if vpn node is there
 548         InstanceIdentifierBuilder<VpnNexthops> idBuilder =
 549             InstanceIdentifier.builder(L3nexthop.class).child(VpnNexthops.class,
 550                 new VpnNexthopsKey(vpnId));
 551         InstanceIdentifier<VpnNexthops> id = idBuilder.build();
 552         Optional<VpnNexthops> vpnNexthops = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 553         if (vpnNexthops.isPresent()) {
 554             // get nexthops list for vpn
 555             List<VpnNexthop> nexthops = vpnNexthops.get().nonnullVpnNexthop();
 556             for (VpnNexthop nexthop : nexthops) {
 557                 if (Objects.equals(nexthop.getIpAddress(), ipAddress)) {
 558                     // return nexthop
 559                     LOG.trace("VpnNextHop : {}", nexthop);
 560                     return nexthop;
 561                 }
 562             }
 563             // return null if not found
 564         }
 565         return null;
 566     }
 567
 568     @Nullable
 569     public AdjacencyResult getRemoteNextHopPointer(Uint64 remoteDpnId, Uint32 vpnId, String prefixIp,
 570             @Nullable String nextHopIp, Class<? extends TunnelTypeBase> tunnelType) {
 571         String egressIfName = null;
 572         LOG.trace("getRemoteNextHopPointer: input [remoteDpnId {}, vpnId {}, prefixIp {}, nextHopIp {} ]", remoteDpnId,
 573             vpnId, prefixIp, nextHopIp);
 574
 575         Class<? extends InterfaceType> egressIfType;
 576         ElanInstance elanInstance = getElanInstanceForPrefix(vpnId, prefixIp);
 577         if (elanInstance != null) {
 578             egressIfType = getInterfaceType(elanInstance);
 579         } else {
 580             LOG.warn("Failed to determine network type for prefixIp {} using tunnel", prefixIp);
 581             egressIfType = Tunnel.class;
 582         }
 583
 584         if (Tunnel.class.equals(egressIfType)) {
 585             egressIfName = getTunnelRemoteNextHopPointer(remoteDpnId, nextHopIp, tunnelType);
 586         } else {
 587             egressIfName = getExtPortRemoteNextHopPointer(remoteDpnId, elanInstance);
 588         }
 589
 590         LOG.trace("NextHop pointer for prefixIp {} vpnId {} dpnId {} is {}", prefixIp, vpnId, remoteDpnId,
 591             egressIfName);
 592         return egressIfName != null ? new AdjacencyResult(egressIfName, egressIfType, nextHopIp,
 593                 prefixIp) : null;
 594     }
 595
 596     private void removeVpnNexthopFromDS(Uint32 vpnId, String ipPrefix) {
 597
 598         InstanceIdentifierBuilder<VpnNexthop> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
 599             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId))
 600             .child(VpnNexthop.class, new VpnNexthopKey(ipPrefix));
 601         InstanceIdentifier<VpnNexthop> id = idBuilder.build();
 602         // remove from DS
 603         LOG.trace("Removing vpn next hop from datastore : {}", id);
 604         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 605     }
 606
 607     public void removeLocalNextHop(Uint64 dpnId, Uint32 vpnId, String primaryIpAddress, String currDestIpPrefix) {
 608         String nextHopLockStr = vpnId + primaryIpAddress;
 609         try {
 610             if (FibUtil.lockCluster(lockManager, nextHopLockStr, WAIT_TIME_TO_ACQUIRE_LOCK)) {
 611                 VpnNexthop nh = getVpnNexthop(vpnId, primaryIpAddress);
 612                 if (nh != null) {
 613                     List<IpAdjacencies> prefixesList = new ArrayList<>(nh.nonnullIpAdjacencies());
 614                     IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currDestIpPrefix).build();
 615                     prefixesList.remove(prefix);
 616                     if (prefixesList.isEmpty()) { //remove the group only if there are no more flows using this group
 617                         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, nh.getEgressPointer().toJava(),
 618                                 primaryIpAddress, GroupTypes.GroupAll, Collections.emptyList());
 619                         // remove Group ...
 620                         mdsalApiManager.removeGroup(groupEntity);
 621                         //update MD-SAL DS
 622                         removeVpnNexthopFromDS(vpnId, primaryIpAddress);
 623                         //release groupId
 624                         removeNextHopPointer(getNextHopKey(vpnId, primaryIpAddress));
 625                         LOG.debug("Local Next hop {} for {} {} on dpn {} successfully deleted",
 626                                 nh.getEgressPointer(), vpnId, primaryIpAddress, dpnId);
 627                     } else {
 628                         //remove the currIpPrefx from IpPrefixList of the vpnNexthop
 629                         LOG.trace("Removing the prefix {} from vpnNextHop {} Operational DS", currDestIpPrefix, nh);
 630                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
 631                                 getVpnNextHopIpPrefixIdentifier(vpnId, primaryIpAddress, currDestIpPrefix));
 632                     }
 633                 } else {
 634                     //throw error
 635                     LOG.error("Local NextHop for VpnId {} curIpPrefix {} on dpn {} primaryIpAddress {} not deleted",
 636                             vpnId, currDestIpPrefix, dpnId, primaryIpAddress);
 637                 }
 638             }
 639         } finally {
 640             FibUtil.unlockCluster(lockManager, nextHopLockStr);
 641         }
 642     }
 643
 644     public void setConfTransType(String service, String transportType) {
 645
 646         if (!service.equalsIgnoreCase("L3VPN")) {
 647             LOG.error("Incorrect service {} provided for setting the transport type.", service);
 648             return;
 649         }
 650
 651         L3VPNTransportTypes transType = L3VPNTransportTypes.validateTransportType(transportType
 652                 .toUpperCase(Locale.getDefault()));
 653
 654         if (transType != L3VPNTransportTypes.Invalid) {
 655             configuredTransportTypeL3VPN = transType;
 656         }
 657     }
 658
 659     public void writeConfTransTypeConfigDS() {
 660         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, getConfTransportTypeIdentifier(),
 661             createConfTransportType(configuredTransportTypeL3VPN.getTransportType()));
 662     }
 663
 664     public L3VPNTransportTypes getConfiguredTransportTypeL3VPN() {
 665         return this.configuredTransportTypeL3VPN;
 666     }
 667
 668     public String getReqTransType() {
 669         if (configuredTransportTypeL3VPN == L3VPNTransportTypes.Invalid) {
 670             /*
 671              * Restart scenario, Read from the ConfigDS.
 672              * if the value is Unset, cache value as VxLAN.
 673              */
 674             LOG.trace("configureTransportType is not yet set.");
 675             Optional<ConfTransportTypeL3vpn> configuredTransTypeFromConfig =
 676                 MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, getConfTransportTypeIdentifier());
 677
 678             if (configuredTransTypeFromConfig.isPresent()) {
 679                 if (TunnelTypeGre.class.equals(configuredTransTypeFromConfig.get().getTransportType())) {
 680                     configuredTransportTypeL3VPN = L3VPNTransportTypes.GRE;
 681                 } else {
 682                     configuredTransportTypeL3VPN = L3VPNTransportTypes.VxLAN;
 683                 }
 684                 LOG.trace("configuredTransportType set from config DS to {}",
 685                     getConfiguredTransportTypeL3VPN().getTransportType());
 686             } else {
 687                 setConfTransType("L3VPN", L3VPNTransportTypes.VxLAN.getTransportType());
 688                 LOG.trace("configuredTransportType is not set in the Config DS. VxLAN as default will be used.");
 689             }
 690         } else {
 691             LOG.trace("configuredTransportType is set as {}", getConfiguredTransportTypeL3VPN().getTransportType());
 692         }
 693         return getConfiguredTransportTypeL3VPN().getTransportType();
 694     }
 695
 696     public InstanceIdentifier<ConfTransportTypeL3vpn> getConfTransportTypeIdentifier() {
 697         return InstanceIdentifier.builder(ConfTransportTypeL3vpn.class).build();
 698     }
 699
 700     private ConfTransportTypeL3vpn createConfTransportType(String type) {
 701         ConfTransportTypeL3vpn confTransType;
 702         switch (type) {
 703             case ITMConstants.TUNNEL_TYPE_GRE:
 704                 confTransType = new ConfTransportTypeL3vpnBuilder().setTransportType(TunnelTypeGre.class).build();
 705                 LOG.trace("Setting the confTransportType to GRE.");
 706                 break;
 707             case ITMConstants.TUNNEL_TYPE_VXLAN:
 708                 confTransType = new ConfTransportTypeL3vpnBuilder().setTransportType(TunnelTypeVxlan.class).build();
 709                 LOG.trace("Setting the confTransportType to VxLAN.");
 710                 break;
 711             default:
 712                 LOG.trace("Invalid transport type {} passed to Config DS ", type);
 713                 confTransType = null;
 714                 break;
 715         }
 716         return confTransType;
 717     }
 718
 719     public Class<? extends TunnelTypeBase> getReqTunType(String transportType) {
 720         switch (transportType) {
 721             case "VXLAN":
 722                 return TunnelTypeVxlan.class;
 723             case "GRE":
 724                 return TunnelTypeGre.class;
 725             default:
 726                 return TunnelTypeMplsOverGre.class;
 727         }
 728     }
 729
 730     public String getTransportTypeStr(String tunType) {
 731         if (tunType.equals(TunnelTypeVxlan.class.toString())) {
 732             return ITMConstants.TUNNEL_TYPE_VXLAN;
 733         } else if (tunType.equals(TunnelTypeGre.class.toString())) {
 734             return ITMConstants.TUNNEL_TYPE_GRE;
 735         } else if (tunType.equals(TunnelTypeMplsOverGre.class.toString())) {
 736             return ITMConstants.TUNNEL_TYPE_MPLSoGRE;
 737         } else {
 738             return ITMConstants.TUNNEL_TYPE_INVALID;
 739         }
 740     }
 741
 742     @Override
 743     @PreDestroy
 744     public void close() {
 745         LOG.info("{} close", getClass().getSimpleName());
 746     }
 747
 748     // TODO Clean up the exception handling
 749     @SuppressWarnings("checkstyle:IllegalCatch")
 750     @Nullable
 751     private String getTunnelRemoteNextHopPointer(Uint64 remoteDpnId, String nextHopIp,
 752                                                  Class<? extends TunnelTypeBase> tunnelType) {
 753         if (nextHopIp != null && !nextHopIp.isEmpty()) {
 754             try {
 755                 // here use the config for tunnel type param
 756                 return getTunnelInterfaceName(remoteDpnId,
 757                     org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder
 758                         .getDefaultInstance(nextHopIp), tunnelType);
 759             } catch (Exception ex) {
 760                 LOG.error("Error while retrieving nexthop pointer for nexthop {} remoteDpn {}",
 761                         nextHopIp, remoteDpnId, ex);
 762             }
 763         }
 764
 765         return null;
 766     }
 767
 768     private String getExtPortRemoteNextHopPointer(Uint64 remoteDpnId, ElanInstance elanInstance) {
 769         return elanService.getExternalElanInterface(elanInstance.getElanInstanceName(), remoteDpnId);
 770     }
 771
 772     /**
 773      * Get the interface type associated with the type of ELAN used for routing
 774      * traffic to/from remote compute nodes.
 775      *
 776      * @param elanInstance The elan instance
 777      * @return L2vlan for flat/VLAN network type and Tunnel otherwise
 778      */
 779     private Class<? extends InterfaceType> getInterfaceType(ElanInstance elanInstance) {
 780         Class<? extends SegmentTypeBase> segmentType = elanInstance.getSegmentType();
 781         if (SegmentTypeFlat.class.equals(segmentType) || SegmentTypeVlan.class.equals(segmentType)) {
 782             return L2vlan.class;
 783         }
 784
 785         return Tunnel.class;
 786     }
 787
 788     private ElanInstance getElanInstanceForPrefix(Uint32 vpnId, String prefixIp) {
 789         ElanInstance elanInstance = null;
 790         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, prefixIp);
 791         if (prefix != null) {
 792             if (prefix.getNetworkId() != null) {
 793                 elanInstance = elanService.getElanInstance(prefix.getNetworkId().getValue());
 794             }
 795         }
 796
 797         return elanInstance;
 798     }
 799
 800     static class AdjacencyResult {
 801         private final String interfaceName;
 802         private final Class<? extends InterfaceType> interfaceType;
 803         private final String nextHopIp;
 804         private final String prefix;
 805
 806         AdjacencyResult(String interfaceName, Class<? extends InterfaceType> interfaceType, String nextHopIp,
 807                         String prefix) {
 808             this.interfaceName = interfaceName;
 809             this.interfaceType = interfaceType;
 810             this.nextHopIp = nextHopIp;
 811             this.prefix = prefix;
 812         }
 813
 814         public String getInterfaceName() {
 815             return interfaceName;
 816         }
 817
 818         public Class<? extends InterfaceType> getInterfaceType() {
 819             return interfaceType;
 820         }
 821
 822         public String getNextHopIp() {
 823             return nextHopIp;
 824         }
 825
 826         public String getPrefix() {
 827             return prefix;
 828         }
 829
 830         @Override
 831         public int hashCode() {
 832             final int prime = 31;
 833             int result = 1;
 834             result = prime * result + (interfaceName == null ? 0 : interfaceName.hashCode());
 835             return result;
 836         }
 837
 838         @Override
 839         public boolean equals(Object obj) {
 840             if (obj == null) {
 841                 return false;
 842             }
 843
 844             if (getClass() != obj.getClass()) {
 845                 return false;
 846             } else {
 847                 AdjacencyResult other = (AdjacencyResult) obj;
 848                 return interfaceName.equals(other.interfaceName);
 849             }
 850         }
 851     }
 852
 853     protected long setupLoadBalancingNextHop(Uint32 parentVpnId, Uint64 dpnId,
 854             String destPrefix, List<BucketInfo> localBucketInfo, List<BucketInfo> remoteBucketInfo) {
 855         long remoteGroupId = createNextHopPointer(getRemoteSelectGroupKey(parentVpnId, destPrefix));
 856         if (remoteGroupId == FibConstants.INVALID_GROUP_ID) {
 857             LOG.error("Unable to allocate/retrieve remote groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 858             return remoteGroupId;
 859         }
 860         long localGroupId =  createNextHopPointer(getLocalSelectGroupKey(parentVpnId, destPrefix));
 861         if (localGroupId == FibConstants.INVALID_GROUP_ID) {
 862             LOG.error("Unable to allocate/retrieve local groupId for vpnId {} , prefix {}",
 863                 parentVpnId, destPrefix);
 864             return remoteGroupId;
 865         }
 866         List<BucketInfo> combinedBucketInfo = new ArrayList<>();
 867         combinedBucketInfo.addAll(localBucketInfo);
 868         combinedBucketInfo.addAll(remoteBucketInfo);
 869         GroupEntity remoteGroupEntity = MDSALUtil.buildGroupEntity(
 870                 dpnId, remoteGroupId, destPrefix, GroupTypes.GroupSelect, combinedBucketInfo);
 871         GroupEntity localGroupEntity = MDSALUtil.buildGroupEntity(
 872                 dpnId, localGroupId, destPrefix, GroupTypes.GroupSelect, localBucketInfo);
 873         String jobKey = FibUtil.getCreateLocalNextHopJobKey(parentVpnId, dpnId, destPrefix);
 874         jobCoordinator.enqueueJob(jobKey, () -> {
 875             mdsalApiManager.syncInstallGroup(remoteGroupEntity);
 876             if (!localBucketInfo.isEmpty()) {
 877                 mdsalApiManager.syncInstallGroup(localGroupEntity);
 878             }
 879             if (LOG.isDebugEnabled()) {
 880                 LOG.debug("Finished installing GroupEntity with jobCoordinator key {} remoteGroupEntity.groupId {}"
 881                         + "localGroupEntity.groupId {}  groupEntity.groupType {}", jobKey,
 882                         remoteGroupEntity.getGroupId(), localGroupEntity.getGroupId(),
 883                         remoteGroupEntity.getGroupType());
 884             }
 885             // Delete local group(if exists) if there is no local info.
 886             // Local group has to be deleted if all VMs in a compute is deleted.
 887             if (localBucketInfo.isEmpty()) {
 888                 LOG.debug("Deleting local group {} since no local nhs present for "
 889                         + "prefix {}", localGroupEntity.getGroupId(), destPrefix);
 890                 mdsalApiManager.syncRemoveGroup(localGroupEntity);
 891             }
 892             return Collections.emptyList();
 893         });
 894         return remoteGroupId;
 895     }
 896
 897     protected void deleteLoadBalancingNextHop(Uint32 parentVpnId, Uint64 dpnId, String destPrefix) {
 898         long remoteGroupId = createNextHopPointer(getRemoteSelectGroupKey(parentVpnId, destPrefix));
 899         if (remoteGroupId == FibConstants.INVALID_GROUP_ID) {
 900             LOG.error("Unable to allocate/retrieve remote groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 901         }
 902         long localGroupId = createNextHopPointer(getLocalSelectGroupKey(parentVpnId, destPrefix));
 903         if (localGroupId == FibConstants.INVALID_GROUP_ID) {
 904             LOG.error("Unable to allocate/retrieve local groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 905         }
 906         GroupEntity remoteGroupEntity = MDSALUtil.buildGroupEntity(
 907                 dpnId, remoteGroupId, destPrefix, GroupTypes.GroupSelect, Collections.emptyList());
 908         GroupEntity localGroupEntity = MDSALUtil.buildGroupEntity(
 909                 dpnId, localGroupId, destPrefix, GroupTypes.GroupSelect, Collections.emptyList());
 910         String jobKey = FibUtil.getCreateLocalNextHopJobKey(parentVpnId, dpnId, destPrefix);
 911         jobCoordinator.enqueueJob(jobKey, () -> {
 912             mdsalApiManager.syncRemoveGroup(remoteGroupEntity);
 913             mdsalApiManager.syncRemoveGroup(localGroupEntity);
 914             if (LOG.isDebugEnabled()) {
 915                 LOG.debug("Finished removing GroupEntity with jobCoordinator key {} remoteGroupEntity.groupId {}"
 916                     + "localGroupEntity.groupId {}", jobKey, remoteGroupId, localGroupId);
 917             }
 918             return Collections.emptyList();
 919         });
 920     }
 921
 922     long createNextHopGroups(Uint32 vpnId, String rd, Uint64 dpnId, VrfEntry vrfEntry,
 923             @Nullable Routes routes, List<Routes> vpnExtraRoutes) {
 924         List<BucketInfo> localBucketInfo = new ArrayList<>();
 925         List<Routes> clonedVpnExtraRoutes  = new ArrayList<>(vpnExtraRoutes);
 926         if (clonedVpnExtraRoutes.contains(routes)) {
 927             localBucketInfo.addAll(getBucketsForLocalNexthop(vpnId, dpnId, vrfEntry, routes));
 928             clonedVpnExtraRoutes.remove(routes);
 929         }
 930         List<BucketInfo> remoteBucketInfo =
 931             new ArrayList<>(getBucketsForRemoteNexthop(vpnId, dpnId, vrfEntry, rd, clonedVpnExtraRoutes));
 932         return setupLoadBalancingNextHop(vpnId, dpnId,
 933             vrfEntry.getDestPrefix(), localBucketInfo, remoteBucketInfo);
 934     }
 935
 936     private List<BucketInfo> getBucketsForLocalNexthop(Uint32 vpnId, Uint64 dpnId,
 937             VrfEntry vrfEntry, Routes routes) {
 938         @Nullable List<String> nexthopIpList = routes.getNexthopIpList();
 939         if (LOG.isDebugEnabled()) {
 940             LOG.debug("NexthopManager.getBucketsForLocalNexthop invoked with vpnId {} dpnId {} "
 941                             + " vrfEntry.routePaths {}, routes.nexthopList {}", vpnId, dpnId, vrfEntry.getRoutePaths(),
 942                 nexthopIpList);
 943         }
 944         List<BucketInfo> listBucketInfo = new CopyOnWriteArrayList<>();
 945         if (nexthopIpList != null) {
 946             nexthopIpList.parallelStream().forEach(nextHopIp -> {
 947                 String localNextHopIP;
 948                 if (isIpv4Address(nextHopIp)) {
 949                     localNextHopIP = nextHopIp + NwConstants.IPV4PREFIX;
 950                 } else {
 951                     localNextHopIP = nextHopIp + NwConstants.IPV6PREFIX;
 952                 }
 953                 Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 954                 if (localNextHopInfo != null) {
 955                     long groupId = getLocalNextHopGroup(vpnId, localNextHopIP);
 956                     if (groupId == FibConstants.INVALID_GROUP_ID) {
 957                         LOG.error("Unable to allocate groupId for vpnId {} , prefix {} , interface {}", vpnId,
 958                             vrfEntry.getDestPrefix(), localNextHopInfo.getVpnInterfaceName());
 959                         return;
 960                     }
 961                     List<ActionInfo> actionsInfos =
 962                         Collections.singletonList(new ActionGroup(groupId));
 963                     BucketInfo bucket = new BucketInfo(actionsInfos);
 964                     bucket.setWeight(1);
 965                     listBucketInfo.add(bucket);
 966                 }
 967             });
 968         }
 969         LOG.trace("LOCAL: listbucket {}, vpnId {}, dpnId {}, routes {}", listBucketInfo, vpnId, dpnId, routes);
 970         return listBucketInfo;
 971     }
 972
 973     private List<BucketInfo> getBucketsForRemoteNexthop(Uint32 vpnId, Uint64 dpnId, VrfEntry vrfEntry, String rd,
 974             List<Routes> vpnExtraRoutes) {
 975         List<BucketInfo> listBucketInfo = new ArrayList<>();
 976         Map<String, List<ActionInfo>> egressActionMap = new HashMap<>();
 977         vpnExtraRoutes.stream().filter(vpnExtraRoute -> vpnExtraRoute.getNexthopIpList() != null).forEach(
 978             vpnExtraRoute -> vpnExtraRoute.getNexthopIpList().forEach(nextHopIp -> {
 979                 String nextHopPrefixIp;
 980                 if (isIpv4Address(nextHopIp)) {
 981                     nextHopPrefixIp = nextHopIp + NwConstants.IPV4PREFIX;
 982                 } else {
 983                     nextHopPrefixIp = nextHopIp + NwConstants.IPV6PREFIX;
 984                 }
 985                 List<String> tepIpAddresses = fibUtil.getNextHopAddresses(rd, nextHopPrefixIp);
 986                 if (tepIpAddresses.isEmpty()) {
 987                     return;
 988                 }
 989                 // There would be only one nexthop address for a VM ip which would be the tep Ip
 990                 String tepIp = tepIpAddresses.get(0);
 991                 AdjacencyResult adjacencyResult = getRemoteNextHopPointer(dpnId, vpnId,
 992                     vrfEntry.getDestPrefix(), tepIp, TunnelTypeVxlan.class);
 993                 if (adjacencyResult == null) {
 994                     return;
 995                 }
 996                 String egressInterface = adjacencyResult.getInterfaceName();
 997                 if (!FibUtil.isTunnelInterface(adjacencyResult)) {
 998                     return;
 999                 }
1000                 Class<? extends TunnelTypeBase> tunnelType =
1001                     VpnExtraRouteHelper.getTunnelType(itmManager, egressInterface);
1002                 StateTunnelList ifState = null;
1003                 try {
1004                     ifState = fibUtil.getTunnelState(egressInterface);
1005                     if (ifState == null || ifState.getOperState() != TunnelOperStatus.Up) {
1006                         LOG.trace("Tunnel is not up for interface {}", egressInterface);
1007                         return;
1008                     }
1009                 } catch (ReadFailedException e) {
1010                     LOG.error("getBucketsForRemoteNexthop: error in fetching tunnel state for interface {}",
1011                         egressInterface, e);
1012                     return;
1013                 }
1014                 if (!TunnelTypeVxlan.class.equals(tunnelType)) {
1015                     return;
1016                 }
1017                 Uint32 label = FibUtil.getLabelFromRoutePaths(vrfEntry).get();
1018                 Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, nextHopPrefixIp);
1019                 if (prefixInfo == null) {
1020                     LOG.error("No prefix info found for prefix {} in rd {} for VPN {}", nextHopPrefixIp, rd,
1021                         vpnId);
1022                     return;
1023                 }
1024                 Uint32 tunnelId;
1025                 if (FibUtil.isVxlanNetwork(prefixInfo.getNetworkType())) {
1026                     tunnelId = prefixInfo.getSegmentationId();
1027                 } else {
1028                     LOG.warn("Network is not of type VXLAN for prefix {}."
1029                         + "Going with default Lport Tag.", prefixInfo.toString());
1030                     tunnelId = label;
1031                 }
1032                 List<ActionInfo> actionInfos = new ArrayList<>();
1033                 actionInfos.add(new ActionSetFieldTunnelId(Uint64.valueOf(tunnelId.longValue())));
1034                 String ifName = prefixInfo.getVpnInterfaceName();
1035                 String vpnName = fibUtil.getVpnNameFromId(vpnId);
1036                 if (vpnName == null) {
1037                     return;
1038                 }
1039                 String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, nextHopPrefixIp);
1040                 actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
1041                     new MacAddress(macAddress)));
1042                 List<ActionInfo> egressActions;
1043                 if (egressActionMap.containsKey(egressInterface)) {
1044                     egressActions = egressActionMap.get(egressInterface);
1045                 } else {
1046                     egressActions = getEgressActionsForInterface(egressInterface, actionInfos.size(),
1047                             true, vpnId, vrfEntry.getDestPrefix());
1048                     if (egressActions.isEmpty()) {
1049                         LOG.error("Skipping getBucketsForRemoteNexthop: Empty list of egress actions received for "
1050                                         + "interface {} on dpn {} for vpn {} prefix {} nextHop {}", ifName, dpnId,
1051                                 vpnId, vrfEntry.getDestPrefix(), nextHopPrefixIp);
1052                     }
1053                     egressActionMap.put(egressInterface, egressActions);
1054                 }
1055                 if (egressActions.isEmpty()) {
1056                     LOG.error("Failed to retrieve egress action for prefix {} route-paths {}"
1057                             + " interface {}." + " Aborting remote FIB entry creation.",
1058                         vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), egressInterface);
1059                 }
1060                 actionInfos.addAll(egressActions);
1061                 BucketInfo bucket = new BucketInfo(actionInfos);
1062                 bucket.setWeight(1);
1063                 listBucketInfo.add(bucket);
1064             }));
1065         LOG.trace("LOCAL: listbucket {}, rd {}, dpnId {}, routes {}", listBucketInfo, rd, dpnId, vpnExtraRoutes);
1066         return listBucketInfo;
1067     }
1068
1069     public void createDcGwLoadBalancingGroup(Uint64 dpnId, String destinationIp,
1070                                              Class<? extends TunnelTypeBase> tunnelType) {
1071         jobCoordinator.enqueueJob(getJobKey(dpnId), () -> {
1072             List<ListenableFuture<Void>> futures = new ArrayList<>();
1073             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operationalTx -> {
1074                 synchronized (getDcGateWaySyncKey(destinationIp)) {
1075                     FibUtil.addL3vpnDcGateWay(destinationIp, operationalTx);
1076                 }
1077                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, configTx -> {
1078                     List<String> availableDcGws = getDcGwIps();
1079                     Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
1080                     int noOfDcGws = availableDcGws.size();
1081                     if (noOfDcGws == 1) {
1082                         LOG.trace("There are no enough DC GateWays {} present to program LB group", availableDcGws);
1083                         return;
1084                     }
1085                     if (availableDcGws.contains(destinationIp)) {
1086                         availableDcGws.remove(destinationIp);
1087                     }
1088                     availableDcGws.forEach(dcGwIp -> {
1089                         List<String> dcGws = Arrays.asList(dcGwIp, destinationIp);
1090                         Collections.sort(dcGws);
1091                         String groupIdKey = FibUtil.getGreLbGroupKey(dcGws);
1092                         Long groupId = createNextHopPointer(groupIdKey);
1093                         List<Bucket> listBucket = new ArrayList<>();
1094                         for (int index = 0; index < dcGws.size(); index++) {
1095                             if (isTunnelUp(dcGws.get(index), dpnId, tunnelType)) {
1096                                 listBucket.add(buildBucketForDcGwLbGroup(dcGws.get(index),
1097                                         dpnId, index, tunnelType, true));
1098                             }
1099                         }
1100                         Group group = MDSALUtil.buildGroup(groupId, groupIdKey, GroupTypes.GroupSelect,
1101                                         MDSALUtil.buildBucketLists(listBucket));
1102                         mdsalApiManager.addGroup(configTx, dpnId, group);
1103                         FibUtil.updateLbGroupInfo(dpnId, groupIdKey, groupId.toString(), operationalTx);
1104                         LOG.trace("LB group {} towards DC-GW installed on dpn {}. Group - {}",
1105                                 groupIdKey, dpnId, group);
1106                     });
1107                 }));
1108             }));
1109             return futures;
1110         }, RETRY_COUNT);
1111     }
1112
1113     private String getJobKey(Uint64 dpnId) {
1114         return new StringBuilder().append(NEXTHOPMANAGER_JOB_KEY_PREFIX).append(dpnId).toString();
1115     }
1116
1117     private String getDcGateWaySyncKey(String destinationIp) {
1118         String mutex = new StringBuilder().append("L3vpncDcGateWay").append(destinationIp).toString();
1119         return mutex.intern();
1120     }
1121
1122     private List<String> getDcGwIps() {
1123         InstanceIdentifier<DcGatewayIpList> dcGatewayIpListid =
1124                 InstanceIdentifier.builder(DcGatewayIpList.class).build();
1125         DcGatewayIpList dcGatewayIpListConfig =
1126                 MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, dcGatewayIpListid).orNull();
1127         if (dcGatewayIpListConfig == null) {
1128             return Collections.emptyList();
1129         }
1130         return dcGatewayIpListConfig.getDcGatewayIp()
1131                 .stream()
1132                 .filter(dcGwIp -> dcGwIp.getTunnnelType().equals(TunnelTypeMplsOverGre.class))
1133                 .map(dcGwIp -> dcGwIp.getIpAddress().stringValue()).sorted()
1134                 .collect(toList());
1135     }
1136
1137     private boolean isTunnelUp(String dcGwIp, Uint64 dpnId, Class<? extends TunnelTypeBase> tunnelType) {
1138         String tunnelName = getTunnelRemoteNextHopPointer(dpnId, dcGwIp, tunnelType);
1139         if (tunnelName != null) {
1140             InstanceIdentifier<StateTunnelList> tunnelStateId =
1141                     InstanceIdentifier.builder(TunnelsState.class).child(
1142                             StateTunnelList.class, new StateTunnelListKey(tunnelName)).build();
1143             return MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, tunnelStateId)
1144                     .toJavaUtil().map(StateTunnelList::getOperState)
1145                     .orElse(TunnelOperStatus.Down) == TunnelOperStatus.Up;
1146         }
1147         return false;
1148     }
1149
1150     private List<Action> getEgressActions(String interfaceName, int actionKey) {
1151         List<Action> actions = Collections.emptyList();
1152         try {
1153             GetEgressActionsForInterfaceInputBuilder egressAction =
1154                     new GetEgressActionsForInterfaceInputBuilder().setIntfName(interfaceName).setActionKey(actionKey);
1155             Future<RpcResult<GetEgressActionsForInterfaceOutput>> result =
1156                     odlInterfaceRpcService.getEgressActionsForInterface(egressAction.build());
1157             RpcResult<GetEgressActionsForInterfaceOutput> rpcResult = result.get();
1158             if (!rpcResult.isSuccessful()) {
1159                 LOG.warn("RPC Call to Get egress actions for interface {} returned with Errors {}",
1160                         interfaceName, rpcResult.getErrors());
1161             } else {
1162                 actions = rpcResult.getResult().nonnullAction();
1163             }
1164         } catch (InterruptedException | ExecutionException e) {
1165             LOG.warn("Exception when egress actions for interface {}", interfaceName, e);
1166         }
1167         return actions;
1168     }
1169
1170     /**
1171      * This method is invoked when the neighbor is removed from DS.
1172      * All the LB groups which point to the given destination will be deleted.
1173      */
1174     public void removeDcGwLoadBalancingGroup(Uint64 dpnId,
1175             String destinationIp) {
1176         jobCoordinator.enqueueJob(getJobKey(dpnId), () -> {
1177             List<String> availableDcGws = fibUtil.getL3VpnDcGateWays();
1178             if (availableDcGws.contains(destinationIp)) {
1179                 availableDcGws.remove(destinationIp);
1180             }
1181             List<ListenableFuture<Void>> futures = new ArrayList<>();
1182             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operationalTx -> {
1183                 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx -> {
1184                     availableDcGws.forEach(dcGwIp -> {
1185                         List<String> dcGws = Arrays.asList(dcGwIp, destinationIp);
1186                         Collections.sort(dcGws);
1187                         String nextHopKey = FibUtil.getGreLbGroupKey(dcGws);
1188                         Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
1189                         if (!optionalNextHops.isPresent()) {
1190                             return;
1191                         }
1192                         Nexthops nexthops = optionalNextHops.get();
1193                         final String groupId = nexthops.getGroupId();
1194                         final long groupIdValue = Long.parseLong(groupId);
1195                         Group group = MDSALUtil.buildGroup(groupIdValue, nextHopKey, GroupTypes.GroupSelect,
1196                                 MDSALUtil.buildBucketLists(Collections.emptyList()));
1197                         LOG.trace("Removed LB group {} on dpn {}", group, dpnId);
1198                         try {
1199                             mdsalApiManager.removeGroup(configTx, dpnId, group);
1200                         } catch (ExecutionException | InterruptedException e) {
1201                             LOG.error("Group removal failed for group {} with exception", groupId, e);
1202                         }
1203                         removeNextHopPointer(nextHopKey);
1204                         FibUtil.removeOrUpdateNextHopInfo(dpnId, nextHopKey, groupId, nexthops, operationalTx);
1205                     });
1206                     synchronized (getDcGateWaySyncKey(destinationIp)) {
1207                         FibUtil.removeL3vpnDcGateWay(destinationIp, operationalTx);
1208                     }
1209                 }));
1210             }));
1211             return futures;
1212         }, RETRY_COUNT);
1213     }
1214
1215     /**
1216      * This method is invoked when the tunnel status is deleted.
1217      * All the buckets which point to given destination will be marked down.
1218      */
1219     public void updateDcGwLoadBalancingGroup(Uint64 dpnId, String destinationIp,
1220             boolean isTunnelUp, Class<? extends TunnelTypeBase> tunnelType) {
1221         jobCoordinator.enqueueJob(getJobKey(dpnId), () -> {
1222             List<String> availableDcGws = fibUtil.getL3VpnDcGateWays();
1223             if (availableDcGws.contains(destinationIp)) {
1224                 availableDcGws.remove(destinationIp);
1225             }
1226             List<ListenableFuture<Void>> futures = new ArrayList<>();
1227             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx -> {
1228                 availableDcGws.forEach(dcGwIp -> {
1229                     List<String> dcGws = Arrays.asList(dcGwIp, destinationIp);
1230                     Collections.sort(dcGws);
1231                     String nextHopKey = FibUtil.getGreLbGroupKey(dcGws);
1232                     int bucketId = dcGws.indexOf(destinationIp);
1233                     Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
1234                     if (!optionalNextHops.isPresent()) {
1235                         return;
1236                     }
1237                     Nexthops nexthops = optionalNextHops.get();
1238                     final String groupId = nexthops.getGroupId();
1239                     final long groupIdValue = Long.parseLong(groupId);
1240                     Bucket bucket = buildBucketForDcGwLbGroup(destinationIp, dpnId, bucketId, tunnelType, isTunnelUp);
1241                     LOG.trace("updated bucket {} to group {} on dpn {}.", bucket, groupId, dpnId);
1242                     try {
1243                         mdsalApiManager.addBucket(configTx, dpnId, groupIdValue, bucket);
1244                     } catch (ExecutionException | InterruptedException e) {
1245                         LOG.error("Bucket addition failed for bucket {} with exception", bucketId, e);
1246                     }
1247                 });
1248             }));
1249             return futures;
1250         }, RETRY_COUNT);
1251     }
1252
1253     private Bucket buildBucketForDcGwLbGroup(String ipAddress, Uint64 dpnId,
1254             int index, Class<? extends TunnelTypeBase> tunnelType, boolean isTunnelUp) {
1255         List<Action> listAction = new ArrayList<>();
1256         // ActionKey 0 goes to mpls label.
1257         int actionKey = 1;
1258         listAction.add(new ActionPushMpls().buildAction());
1259         listAction.add(new ActionRegMove(actionKey++, FibConstants.NXM_REG_MAPPING
1260                 .get(index), 0, 19).buildAction());
1261         String tunnelInterfaceName = getTunnelInterfaceName(dpnId, IpAddressBuilder.getDefaultInstance(ipAddress),
1262             tunnelType);
1263         List<Action> egressActions = getEgressActions(tunnelInterfaceName, actionKey++);
1264         if (!egressActions.isEmpty()) {
1265             listAction.addAll(getEgressActions(tunnelInterfaceName, actionKey++));
1266         } else {
1267             // clear off actions if there is no egress actions.
1268             listAction = Collections.emptyList();
1269         }
1270         long watchPort = MDSALUtil.WATCH_PORT;
1271         if (!isTunnelUp) {
1272             watchPort = 0xFFFFFFFEL;
1273         }
1274         //OVS expects a non-zero weight value for load balancing to happen in select groups
1275         return MDSALUtil.buildBucket(listAction, SELECT_GROUP_WEIGHT, index,
1276                 watchPort, MDSALUtil.WATCH_GROUP);
1277     }
1278
1279     public void programDcGwLoadBalancingGroup(Uint64 dpnId, String destinationIp,
1280                                               int addRemoveOrUpdate, boolean isTunnelUp,
1281                                               Class<? extends TunnelTypeBase> tunnelType) {
1282         if (NwConstants.ADD_FLOW == addRemoveOrUpdate) {
1283             createDcGwLoadBalancingGroup(dpnId, destinationIp, tunnelType);
1284         } else if (NwConstants.DEL_FLOW == addRemoveOrUpdate) {
1285             removeDcGwLoadBalancingGroup(dpnId, destinationIp);
1286         } else if (NwConstants.MOD_FLOW == addRemoveOrUpdate) {
1287             updateDcGwLoadBalancingGroup(dpnId, destinationIp, isTunnelUp, tunnelType);
1288         }
1289     }
1290 }