fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/NexthopManager.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static java.util.stream.Collectors.toList;
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  14
  15 import com.google.common.base.Preconditions;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import java.util.ArrayList;
  18 import java.util.Arrays;
  19 import java.util.Collections;
  20 import java.util.HashMap;
  21 import java.util.List;
  22 import java.util.Locale;
  23 import java.util.Map;
  24 import java.util.Objects;
  25 import java.util.Optional;
  26 import java.util.concurrent.CopyOnWriteArrayList;
  27 import java.util.concurrent.ExecutionException;
  28 import java.util.concurrent.Future;
  29 import java.util.concurrent.TimeUnit;
  30 import java.util.concurrent.TimeoutException;
  31 import javax.annotation.PreDestroy;
  32 import javax.inject.Inject;
  33 import javax.inject.Singleton;
  34 import org.eclipse.jdt.annotation.Nullable;
  35 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  36 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  37 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  38 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  39 import org.opendaylight.genius.itm.globals.ITMConstants;
  40 import org.opendaylight.genius.mdsalutil.ActionInfo;
  41 import org.opendaylight.genius.mdsalutil.BucketInfo;
  42 import org.opendaylight.genius.mdsalutil.GroupEntity;
  43 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  44 import org.opendaylight.genius.mdsalutil.NwConstants;
  45 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  46 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  47 import org.opendaylight.genius.mdsalutil.actions.ActionOutput;
  48 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
  49 import org.opendaylight.genius.mdsalutil.actions.ActionPushVlan;
  50 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  51 import org.opendaylight.genius.mdsalutil.actions.ActionRegMove;
  52 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  53 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetSource;
  54 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  55 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldVlanVid;
  56 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  57 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  58 import org.opendaylight.mdsal.binding.api.DataBroker;
  59 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  60 import org.opendaylight.mdsal.common.api.ReadFailedException;
  61 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  62 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  63 import org.opendaylight.netvirt.fibmanager.api.L3VPNTransportTypes;
  64 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  65 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev170119.L2vlan;
  66 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev170119.Tunnel;
  67 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfaceType;
  69 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.OutputActionCase;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.PushVlanActionCase;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.SetFieldCase;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInput;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInputBuilder;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutput;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInput;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInputBuilder;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolOutput;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInput;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInputBuilder;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdOutput;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceInputBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceOutput;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelOperStatus;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelsState;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelListKey;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rev160406.DcGatewayIpList;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rev160406.dc.gateway.ip.list.DcGatewayIp;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelInputBuilder;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelOutput;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameInputBuilder;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameOutput;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.lockmanager.rev160413.LockManagerService;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupInput;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupInputBuilder;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupOutput;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.SalGroupService;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupId;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupRef;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.Buckets;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeRef;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeBase;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeFlat;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeVlan;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.L3nexthop;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthops;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthopsKey;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopKey;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacencies;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacenciesBuilder;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacenciesKey;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpn;
 131 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpnBuilder;
 132 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.lb.nexthops.Nexthops;
 133 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 134 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 135 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6;
 136 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.add.group.input.buckets.bucket.action.action.NxActionResubmitRpcAddGroupCase;
 137 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.nodes.node.table.flow.instructions.instruction.instruction.apply.actions._case.apply.actions.action.action.NxActionRegLoadNodesNodeTableFlowApplyActionsCase;
 138 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.nx.action.reg.load.grouping.NxRegLoad;
 139 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 140 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 141 import org.opendaylight.yangtools.yang.common.RpcResult;
 142 import org.opendaylight.yangtools.yang.common.Uint32;
 143 import org.opendaylight.yangtools.yang.common.Uint64;
 144 import org.slf4j.Logger;
 145 import org.slf4j.LoggerFactory;
 146
 147 @Singleton
 148 public class NexthopManager implements AutoCloseable {
 149     private static final Logger LOG = LoggerFactory.getLogger(NexthopManager.class);
 150     private static final String NEXTHOP_ID_POOL_NAME = "nextHopPointerPool";
 151     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 152     //  We set the total wait time for lock to be obtained at 9 seconds since GC pauses can be upto 8 seconds
 153     //in scale setups.
 154     private static final long WAIT_TIME_TO_ACQUIRE_LOCK = 9000L;
 155     private static final int SELECT_GROUP_WEIGHT = 1;
 156     private static final int RETRY_COUNT = 6;
 157
 158     private final DataBroker dataBroker;
 159     private final ManagedNewTransactionRunner txRunner;
 160     private final IMdsalApiManager mdsalApiManager;
 161     private final OdlInterfaceRpcService odlInterfaceRpcService;
 162     private final ItmRpcService itmManager;
 163     private final IdManagerService idManager;
 164     private final IElanService elanService;
 165     private final LockManagerService lockManager;
 166     private final SalGroupService salGroupService;
 167     private final JobCoordinator jobCoordinator;
 168     private final FibUtil fibUtil;
 169     private final IInterfaceManager interfaceManager;
 170     private volatile L3VPNTransportTypes configuredTransportTypeL3VPN = L3VPNTransportTypes.Invalid;
 171
 172     /**
 173      * Provides nexthop functions.
 174      * Creates group ID pool
 175      *
 176      * @param dataBroker       - dataBroker reference
 177      * @param mdsalApiManager  - mdsalApiManager reference
 178      * @param idManager        - idManager reference
 179      * @param odlInterfaceRpcService - odlInterfaceRpcService reference
 180      * @param itmManager       - itmManager reference
 181      */
 182     @Inject
 183     public NexthopManager(final DataBroker dataBroker,
 184                           final IMdsalApiManager mdsalApiManager,
 185                           final IdManagerService idManager,
 186                           final OdlInterfaceRpcService odlInterfaceRpcService,
 187                           final ItmRpcService itmManager,
 188                           final LockManagerService lockManager,
 189                           final IElanService elanService,
 190                           final SalGroupService salGroupService,
 191                           final JobCoordinator jobCoordinator,
 192                           final FibUtil fibUtil,
 193                           final IInterfaceManager interfaceManager) {
 194         this.dataBroker = dataBroker;
 195         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 196         this.mdsalApiManager = mdsalApiManager;
 197         this.idManager = idManager;
 198         this.odlInterfaceRpcService = odlInterfaceRpcService;
 199         this.itmManager = itmManager;
 200         this.elanService = elanService;
 201         this.salGroupService = salGroupService;
 202         this.jobCoordinator = jobCoordinator;
 203         this.fibUtil = fibUtil;
 204         this.lockManager = lockManager;
 205         this.interfaceManager = interfaceManager;
 206         createIdPool();
 207     }
 208
 209     private void createIdPool() {
 210         CreateIdPoolInput createPool = new CreateIdPoolInputBuilder()
 211             .setPoolName(NEXTHOP_ID_POOL_NAME)
 212             .setLow(150000L)
 213             .setHigh(175000L)
 214             .build();
 215         try {
 216             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPool);
 217             if (result != null && result.get().isSuccessful()) {
 218                 LOG.info("Created IdPool for NextHopPointerPool");
 219             }
 220         } catch (InterruptedException | ExecutionException e) {
 221             LOG.error("Failed to create idPool for NextHopPointerPool", e);
 222         }
 223     }
 224
 225     private String getNextHopKey(Uint32 vpnId, String ipAddress) {
 226         return "nexthop." + vpnId + ipAddress;
 227     }
 228
 229     String getRemoteSelectGroupKey(Uint32 vpnId, String ipAddress) {
 230         return "remote.ecmp.nexthop." + vpnId + ipAddress;
 231     }
 232
 233     String getLocalSelectGroupKey(Uint32 vpnId, String ipAddress) {
 234         return "local.ecmp.nexthop." + vpnId + ipAddress;
 235     }
 236
 237     public ItmRpcService getItmManager() {
 238         return itmManager;
 239     }
 240
 241     protected long createNextHopPointer(String nexthopKey) {
 242         AllocateIdInput getIdInput = new AllocateIdInputBuilder()
 243             .setPoolName(NEXTHOP_ID_POOL_NAME).setIdKey(nexthopKey)
 244             .build();
 245         //TODO: Proper error handling once IdManager code is complete
 246         try {
 247             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
 248             RpcResult<AllocateIdOutput> rpcResult = result.get();
 249             return rpcResult.getResult().getIdValue().toJava();
 250         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 251             // FIXME: NPEs should not be caught but rather their root cause should be eliminated
 252             LOG.trace("Failed to allocate {}", getIdInput, e);
 253         }
 254         return 0;
 255     }
 256
 257     protected void removeNextHopPointer(String nexthopKey) {
 258         ReleaseIdInput idInput = new ReleaseIdInputBuilder()
 259             .setPoolName(NEXTHOP_ID_POOL_NAME)
 260             .setIdKey(nexthopKey).build();
 261         try {
 262             RpcResult<ReleaseIdOutput> rpcResult = idManager.releaseId(idInput).get();
 263             if (!rpcResult.isSuccessful()) {
 264                 LOG.error("RPC Call to Get Unique Id for nexthopKey {} returned with Errors {}",
 265                         nexthopKey, rpcResult.getErrors());
 266             }
 267         } catch (InterruptedException | ExecutionException e) {
 268             LOG.warn("Exception when getting Unique Id for key {}", nexthopKey, e);
 269         }
 270     }
 271
 272     protected List<ActionInfo> getEgressActionsForInterface(final String ifName, int actionKey,
 273                                                             boolean isTunnelInterface,
 274                                                             Uint32 vpnId, String destIpPrefix) {
 275         List<Action> actions;
 276         try {
 277             if (isTunnelInterface && interfaceManager.isItmDirectTunnelsEnabled()) {
 278                 RpcResult<GetEgressActionsForTunnelOutput> rpcResult =
 279                         itmManager.getEgressActionsForTunnel(new GetEgressActionsForTunnelInputBuilder()
 280                                 .setIntfName(ifName).build()).get();
 281                 if (!rpcResult.isSuccessful()) {
 282                     LOG.error("RPC Call to Get egress tunnel actions for interface {} returned with Errors {}",
 283                             ifName, rpcResult.getErrors());
 284                     return Collections.emptyList();
 285                 } else {
 286                     actions = new ArrayList<Action>(rpcResult.getResult().nonnullAction().values());
 287                 }
 288             } else {
 289                 RpcResult<GetEgressActionsForInterfaceOutput> rpcResult = odlInterfaceRpcService
 290                         .getEgressActionsForInterface(new GetEgressActionsForInterfaceInputBuilder()
 291                                 .setIntfName(ifName).build()).get();
 292                 if (!rpcResult.isSuccessful()) {
 293                     LOG.error("RPC Call to Get egress vm actions for interface {} vpnId {} ipPrefix {} returned with "
 294                                     + "Errors {}", ifName, vpnId, destIpPrefix, rpcResult.getErrors());
 295                     return Collections.emptyList();
 296                 } else {
 297                     actions = new ArrayList<Action>(rpcResult.getResult().nonnullAction().values());
 298                 }
 299             }
 300             List<ActionInfo> listActionInfo = new ArrayList<>();
 301             for (Action action : actions) {
 302                 actionKey = action.key().getOrder() + actionKey;
 303                 org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action
 304                     actionClass = action.getAction();
 305                 if (actionClass instanceof OutputActionCase) {
 306                     listActionInfo.add(new ActionOutput(actionKey,
 307                         ((OutputActionCase) actionClass).getOutputAction().getOutputNodeConnector()));
 308                 } else if (actionClass instanceof PushVlanActionCase) {
 309                     listActionInfo.add(new ActionPushVlan(actionKey));
 310                 } else if (actionClass instanceof SetFieldCase) {
 311                     if (((SetFieldCase) actionClass).getSetField().getVlanMatch() != null) {
 312                         int vlanVid = ((SetFieldCase) actionClass).getSetField().getVlanMatch()
 313                             .getVlanId().getVlanId().getValue().toJava();
 314                         listActionInfo.add(new ActionSetFieldVlanVid(actionKey, vlanVid));
 315                     }
 316                 } else if (actionClass instanceof NxActionResubmitRpcAddGroupCase) {
 317                     Short tableId = ((NxActionResubmitRpcAddGroupCase) actionClass).getNxResubmit().getTable().toJava();
 318                     listActionInfo.add(new ActionNxResubmit(actionKey, tableId));
 319                 } else if (actionClass instanceof NxActionRegLoadNodesNodeTableFlowApplyActionsCase) {
 320                     NxRegLoad nxRegLoad =
 321                         ((NxActionRegLoadNodesNodeTableFlowApplyActionsCase) actionClass).getNxRegLoad();
 322                     listActionInfo.add(new ActionRegLoad(actionKey, NxmNxReg6.class,
 323                         nxRegLoad.getDst().getStart().toJava(), nxRegLoad.getDst().getEnd().toJava(),
 324                         nxRegLoad.getValue().longValue()));
 325                 }
 326             }
 327             return listActionInfo;
 328         } catch (InterruptedException | ExecutionException | NullPointerException e) {
 329             LOG.error("Exception when egress actions for interface {} isTunnel {} vpnId {} ipPrefix {}", ifName,
 330                     isTunnelInterface, vpnId, destIpPrefix, e);
 331         }
 332         LOG.warn("Exception when egress actions for interface {}", ifName);
 333         return Collections.emptyList();
 334     }
 335
 336     @Nullable
 337     protected String getTunnelInterfaceName(Uint64 srcDpId, Uint64 dstDpId) {
 338         Class<? extends TunnelTypeBase> tunType = getReqTunType(getReqTransType().toUpperCase(Locale.getDefault()));
 339         Future<RpcResult<GetTunnelInterfaceNameOutput>> result;
 340         try {
 341             result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 342                 .setSourceDpid(srcDpId)
 343                 .setDestinationDpid(dstDpId)
 344                 .setTunnelType(tunType)
 345                 .build());
 346             RpcResult<GetTunnelInterfaceNameOutput> rpcResult = result.get();
 347             if (!rpcResult.isSuccessful()) {
 348                 LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
 349             } else {
 350                 return rpcResult.getResult().getInterfaceName();
 351             }
 352         } catch (InterruptedException | ExecutionException e) {
 353             LOG.warn("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstDpId, e);
 354         }
 355         return null;
 356     }
 357
 358     @Nullable
 359     protected String getTunnelInterfaceName(Uint64 srcDpId, org.opendaylight.yang.gen.v1.urn.ietf.params
 360         .xml.ns.yang.ietf.inet.types.rev130715.IpAddress dstIp, Class<? extends TunnelTypeBase> tunnelType) {
 361         Future<RpcResult<GetInternalOrExternalInterfaceNameOutput>> result;
 362         try {
 363             LOG.debug("Trying to fetch tunnel interface name for source dpn {} destIp {} tunType {}", srcDpId,
 364                     dstIp.stringValue(), tunnelType.getName());
 365             result = itmManager.getInternalOrExternalInterfaceName(new GetInternalOrExternalInterfaceNameInputBuilder()
 366                 .setSourceDpid(srcDpId)
 367                 .setDestinationIp(dstIp)
 368                 .setTunnelType(tunnelType)
 369                 .build());
 370             RpcResult<GetInternalOrExternalInterfaceNameOutput> rpcResult = result.get();
 371             if (!rpcResult.isSuccessful()) {
 372                 LOG.warn("RPC Call to getTunnelInterfaceName returned with Errors {}", rpcResult.getErrors());
 373             } else {
 374                 return rpcResult.getResult().getInterfaceName();
 375             }
 376         } catch (InterruptedException | ExecutionException e) {
 377             LOG.error("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstIp, e);
 378         }
 379         return null;
 380     }
 381
 382
 383     public long getLocalNextHopGroup(Uint32 vpnId,
 384             String ipNextHopAddress) {
 385         long groupId = createNextHopPointer(getNextHopKey(vpnId, ipNextHopAddress));
 386         if (groupId == FibConstants.INVALID_GROUP_ID) {
 387             LOG.error("Unable to allocate groupId for vpnId {} , prefix {}", vpnId, ipNextHopAddress);
 388         }
 389         return groupId;
 390     }
 391
 392     public long getLocalSelectGroup(Uint32 vpnId,
 393             String ipNextHopAddress) {
 394         long groupId = createNextHopPointer(getLocalSelectGroupKey(vpnId, ipNextHopAddress));
 395         if (groupId == FibConstants.INVALID_GROUP_ID) {
 396             LOG.error("Unable to allocate groupId for vpnId {} , prefix {}", vpnId, ipNextHopAddress);
 397         }
 398         return groupId;
 399     }
 400
 401     public long createLocalNextHop(Uint32 vpnId, Uint64 dpnId, String ifName,
 402                                    String primaryIpAddress, String currDestIpPrefix,
 403                                    String gwMacAddress, Uint32 parentVpnId) {
 404         //For VPN Imported routes, getting VPN Instance name using parentVpnId
 405         String vpnName = parentVpnId != null ? fibUtil.getVpnNameFromId(parentVpnId) : fibUtil.getVpnNameFromId(vpnId);
 406         if (vpnName == null) {
 407             return 0;
 408         }
 409         String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, primaryIpAddress);
 410
 411         long groupId = createNextHopPointer(getNextHopKey(vpnId, primaryIpAddress));
 412         if (groupId == 0) {
 413             LOG.error("Unable to allocate groupId for vpnId {} , IntfName {}, primaryIpAddress {} curIpPrefix {}",
 414                     vpnId, ifName, primaryIpAddress, currDestIpPrefix);
 415             return groupId;
 416         }
 417         String nextHopLockStr = vpnId + primaryIpAddress;
 418         String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, currDestIpPrefix);
 419         jobCoordinator.enqueueJob(jobKey, () -> {
 420             try {
 421                 if (FibUtil.lockCluster(lockManager, nextHopLockStr, WAIT_TIME_TO_ACQUIRE_LOCK)) {
 422                     VpnNexthop nexthop = getVpnNexthop(vpnId, primaryIpAddress);
 423                     LOG.trace("nexthop: {} retrieved for vpnId {}, prefix {}, ifName {} on dpn {}", nexthop, vpnId,
 424                             primaryIpAddress, ifName, dpnId);
 425                     if (nexthop == null) {
 426                         String encMacAddress = macAddress == null
 427                                 ? fibUtil.getMacAddressFromPrefix(ifName, vpnName, primaryIpAddress) : macAddress;
 428                         List<ActionInfo> listActionInfo = new ArrayList<>();
 429                         int actionKey = 0;
 430                         // MAC re-write
 431                         if (encMacAddress != null) {
 432                             if (gwMacAddress != null) {
 433                                 LOG.trace("The Local NextHop Group Source Mac {} for VpnInterface {} on VPN {}",
 434                                         gwMacAddress, ifName, vpnId);
 435                                 listActionInfo.add(new ActionSetFieldEthernetSource(actionKey++,
 436                                         new MacAddress(gwMacAddress)));
 437                             }
 438                             listActionInfo.add(new ActionSetFieldEthernetDestination(actionKey++,
 439                                     new MacAddress(encMacAddress)));
 440                             // listActionInfo.add(0, new ActionPopMpls());
 441                         } else {
 442                             LOG.error("mac address for new local nexthop group {} is null for vpnId {}, prefix {}, "
 443                                     + "ifName {} on dpn {}", groupId, vpnId, primaryIpAddress, ifName, dpnId);
 444                         }
 445                         List<ActionInfo> nhActionInfoList = getEgressActionsForInterface(ifName, actionKey, false,
 446                                 vpnId, currDestIpPrefix);
 447                         if (nhActionInfoList.isEmpty()) {
 448                             LOG.error("createLocalNextHop: Skipping, Empty list of egress actions received for "
 449                                     + "interface {} on dpn {} for vpn {} prefix {}", ifName, dpnId, vpnId,
 450                                     currDestIpPrefix);
 451                         }
 452                         listActionInfo.addAll(nhActionInfoList);
 453                         BucketInfo bucket = new BucketInfo(listActionInfo);
 454                         List<BucketInfo> listBucketInfo = new ArrayList<>();
 455                         listBucketInfo.add(bucket);
 456                         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, groupId, primaryIpAddress,
 457                                 GroupTypes.GroupAll, listBucketInfo);
 458                         LOG.trace("Install LNH Group: id {}, mac address {}, interface {} for prefix {}", groupId,
 459                                 encMacAddress, ifName, primaryIpAddress);
 460                         //Try to install group directly on the DPN bypassing the FRM, in order to avoid waiting for the
 461                         // group to get installed before programming the flows
 462                         installGroupOnDpn(groupId, dpnId, primaryIpAddress, listBucketInfo,
 463                                 getNextHopKey(vpnId, primaryIpAddress), GroupTypes.GroupAll);
 464                         // install Group
 465                         mdsalApiManager.syncInstallGroup(groupEntity);
 466                         // update MD-SAL DS
 467                         addVpnNexthopToDS(dpnId, vpnId, primaryIpAddress, currDestIpPrefix, groupId);
 468
 469                     } else {
 470                         // Ignore adding new prefix , if it already exists
 471                         Map<IpAdjacenciesKey, IpAdjacencies> keyIpAdjacenciesMap = nexthop.getIpAdjacencies();
 472                         IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currDestIpPrefix).build();
 473                         if (keyIpAdjacenciesMap != null && keyIpAdjacenciesMap.values().contains(prefix)) {
 474                             LOG.trace("Prefix {} is already present in l3nextHop {} ", currDestIpPrefix, nexthop);
 475                         } else {
 476                             IpAdjacenciesBuilder ipPrefixesBuilder =
 477                                     new IpAdjacenciesBuilder().withKey(new IpAdjacenciesKey(currDestIpPrefix));
 478                             LOG.trace("Updating prefix {} to vpnNextHop {} Operational DS", currDestIpPrefix, nexthop);
 479                             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
 480                                     getVpnNextHopIpPrefixIdentifier(vpnId, primaryIpAddress, currDestIpPrefix),
 481                                     ipPrefixesBuilder.build());
 482                         }
 483                     }
 484                 }
 485             } finally {
 486                 FibUtil.unlockCluster(lockManager, nextHopLockStr);
 487             }
 488             return Collections.emptyList();
 489         });
 490         return groupId;
 491     }
 492
 493     private void installGroupOnDpn(long groupId, Uint64 dpnId, String groupName, List<BucketInfo> bucketsInfo,
 494                                      String nextHopKey, GroupTypes groupType) {
 495         NodeRef nodeRef = FibUtil.buildNodeRef(dpnId);
 496         Buckets buckets = FibUtil.buildBuckets(bucketsInfo);
 497         GroupRef groupRef = new GroupRef(FibUtil.buildGroupInstanceIdentifier(groupId, dpnId));
 498         AddGroupInput input = new AddGroupInputBuilder().setNode(nodeRef).setGroupId(new GroupId(groupId))
 499                 .setBuckets(buckets).setGroupRef(groupRef).setGroupType(groupType)
 500                 .setGroupName(groupName).build();
 501         Future<RpcResult<AddGroupOutput>> groupStats = salGroupService.addGroup(input);
 502         RpcResult<AddGroupOutput> rpcResult = null;
 503         try {
 504             rpcResult = groupStats.get(WAIT_TIME_FOR_SYNC_INSTALL, TimeUnit.MILLISECONDS);
 505             if (rpcResult != null && rpcResult.isSuccessful()) {
 506                 LOG.info("installGroupOnDpn: Group {} with key {} has been successfully installed directly on dpn {}.",
 507                         groupId, nextHopKey, dpnId);
 508             } else {
 509                 LOG.error("installGroupOnDpn: Unable to install group {} with key {} directly on dpn {} due to {}.",
 510                         groupId, nextHopKey, dpnId, rpcResult != null ? rpcResult.getErrors() : null);
 511             }
 512         } catch (InterruptedException | ExecutionException e) {
 513             LOG.error("installGroupOnDpn: Error while installing group {} directly on dpn {}", groupId, dpnId);
 514         } catch (TimeoutException e) {
 515             LOG.error("installGroupOnDpn: Group {} installation on dpn {} timed out.", groupId, dpnId);
 516         }
 517     }
 518
 519     protected void addVpnNexthopToDS(Uint64 dpnId, Uint32 vpnId, String primaryIpAddr,
 520                                      String currIpAddr, long egressPointer) {
 521         InstanceIdentifierBuilder<VpnNexthops> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
 522             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId));
 523
 524         List<IpAdjacencies> ipPrefixesList = new ArrayList<>();
 525         IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currIpAddr).build();
 526         ipPrefixesList.add(prefix);
 527         // Add nexthop to vpn node
 528         VpnNexthop nh = new VpnNexthopBuilder()
 529             .withKey(new VpnNexthopKey(primaryIpAddr))
 530             .setDpnId(dpnId)
 531             .setIpAdjacencies(ipPrefixesList)
 532             .setEgressPointer(egressPointer).build();
 533
 534         InstanceIdentifier<VpnNexthop> id1 = idBuilder
 535             .child(VpnNexthop.class, new VpnNexthopKey(primaryIpAddr)).build();
 536         LOG.trace("Adding vpnnextHop {} to Operational DS", nh);
 537         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id1, nh);
 538
 539     }
 540
 541     protected InstanceIdentifier<IpAdjacencies> getVpnNextHopIpPrefixIdentifier(Uint32 vpnId, String primaryIpAddress,
 542                                                                                 String ipPrefix) {
 543         InstanceIdentifier<IpAdjacencies> id = InstanceIdentifier.builder(L3nexthop.class)
 544                 .child(VpnNexthops.class, new VpnNexthopsKey(vpnId))
 545                 .child(VpnNexthop.class, new VpnNexthopKey(primaryIpAddress))
 546                 .child(IpAdjacencies.class, new IpAdjacenciesKey(ipPrefix)).build();
 547         return id;
 548     }
 549
 550     @Nullable
 551     protected VpnNexthop getVpnNexthop(Uint32 vpnId, String ipAddress) {
 552
 553         // check if vpn node is there
 554         InstanceIdentifierBuilder<VpnNexthops> idBuilder =
 555             InstanceIdentifier.builder(L3nexthop.class).child(VpnNexthops.class,
 556                 new VpnNexthopsKey(vpnId));
 557         InstanceIdentifier<VpnNexthops> id = idBuilder.build();
 558         Optional<VpnNexthops> vpnNexthops;
 559         try {
 560             vpnNexthops = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 561                     LogicalDatastoreType.OPERATIONAL, id);
 562         } catch (ExecutionException | InterruptedException e) {
 563             LOG.error("getVpnNexthop: Exception while reading VpnNexthops DS for the address {} vpn {}", ipAddress,
 564                     vpnId, e);
 565             return null;
 566         }
 567         if (vpnNexthops.isPresent()) {
 568             // get keyVpnNexthopMap list for vpn
 569             Map<VpnNexthopKey, VpnNexthop> keyVpnNexthopMap = vpnNexthops.get().nonnullVpnNexthop();
 570             for (VpnNexthop nexthop : keyVpnNexthopMap.values()) {
 571                 if (Objects.equals(nexthop.getIpAddress(), ipAddress)) {
 572                     // return nexthop
 573                     LOG.trace("VpnNextHop : {}", nexthop);
 574                     return nexthop;
 575                 }
 576             }
 577             // return null if not found
 578         }
 579         return null;
 580     }
 581
 582     @Nullable
 583     public AdjacencyResult getRemoteNextHopPointer(Uint64 remoteDpnId, Uint32 vpnId, String prefixIp,
 584             @Nullable String nextHopIp, Class<? extends TunnelTypeBase> tunnelType) {
 585         String egressIfName = null;
 586         LOG.trace("getRemoteNextHopPointer: input [remoteDpnId {}, vpnId {}, prefixIp {}, nextHopIp {} ]", remoteDpnId,
 587             vpnId, prefixIp, nextHopIp);
 588
 589         Class<? extends InterfaceType> egressIfType;
 590         ElanInstance elanInstance = getElanInstanceForPrefix(vpnId, prefixIp);
 591         if (elanInstance != null) {
 592             egressIfType = getInterfaceType(elanInstance);
 593         } else {
 594             LOG.warn("Failed to determine network type for prefixIp {} using tunnel", prefixIp);
 595             egressIfType = Tunnel.class;
 596         }
 597
 598         if (Tunnel.class.equals(egressIfType)) {
 599             egressIfName = getTunnelRemoteNextHopPointer(remoteDpnId, nextHopIp, tunnelType);
 600         } else {
 601             egressIfName = getExtPortRemoteNextHopPointer(remoteDpnId, elanInstance);
 602         }
 603
 604         LOG.trace("NextHop pointer for prefixIp {} vpnId {} dpnId {} is {}", prefixIp, vpnId, remoteDpnId,
 605             egressIfName);
 606         return egressIfName != null ? new AdjacencyResult(egressIfName, egressIfType, nextHopIp,
 607                 prefixIp) : null;
 608     }
 609
 610     private void removeVpnNexthopFromDS(Uint32 vpnId, String ipPrefix) {
 611
 612         InstanceIdentifierBuilder<VpnNexthop> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
 613             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId))
 614             .child(VpnNexthop.class, new VpnNexthopKey(ipPrefix));
 615         InstanceIdentifier<VpnNexthop> id = idBuilder.build();
 616         // remove from DS
 617         LOG.trace("Removing vpn next hop from datastore : {}", id);
 618         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 619     }
 620
 621     public void removeLocalNextHop(Uint64 dpnId, Uint32 vpnId, String primaryIpAddress, String currDestIpPrefix) {
 622         String nextHopLockStr = vpnId + primaryIpAddress;
 623         try {
 624             if (FibUtil.lockCluster(lockManager, nextHopLockStr, WAIT_TIME_TO_ACQUIRE_LOCK)) {
 625                 VpnNexthop nh = getVpnNexthop(vpnId, primaryIpAddress);
 626                 if (nh != null) {
 627                     List<IpAdjacencies> prefixesList = new ArrayList<IpAdjacencies>(nh.nonnullIpAdjacencies().values());
 628                     IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currDestIpPrefix).build();
 629                     prefixesList.remove(prefix);
 630                     if (prefixesList.isEmpty()) { //remove the group only if there are no more flows using this group
 631                         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, nh.getEgressPointer().toJava(),
 632                                 primaryIpAddress, GroupTypes.GroupAll, Collections.emptyList());
 633                         // remove Group ...
 634                         mdsalApiManager.removeGroup(groupEntity);
 635                         //update MD-SAL DS
 636                         removeVpnNexthopFromDS(vpnId, primaryIpAddress);
 637                         //release groupId
 638                         removeNextHopPointer(getNextHopKey(vpnId, primaryIpAddress));
 639                         LOG.debug("Local Next hop {} for {} {} on dpn {} successfully deleted",
 640                                 nh.getEgressPointer(), vpnId, primaryIpAddress, dpnId);
 641                     } else {
 642                         //remove the currIpPrefx from IpPrefixList of the vpnNexthop
 643                         LOG.trace("Removing the prefix {} from vpnNextHop {} Operational DS", currDestIpPrefix, nh);
 644                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
 645                                 getVpnNextHopIpPrefixIdentifier(vpnId, primaryIpAddress, currDestIpPrefix));
 646                     }
 647                 } else {
 648                     //throw error
 649                     LOG.error("Local NextHop for VpnId {} curIpPrefix {} on dpn {} primaryIpAddress {} not deleted",
 650                             vpnId, currDestIpPrefix, dpnId, primaryIpAddress);
 651                 }
 652             }
 653         } finally {
 654             FibUtil.unlockCluster(lockManager, nextHopLockStr);
 655         }
 656     }
 657
 658     public void setConfTransType(String service, String transportType) {
 659
 660         if (!service.equalsIgnoreCase("L3VPN")) {
 661             LOG.error("Incorrect service {} provided for setting the transport type.", service);
 662             return;
 663         }
 664
 665         L3VPNTransportTypes transType = L3VPNTransportTypes.validateTransportType(transportType
 666                 .toUpperCase(Locale.getDefault()));
 667
 668         if (transType != L3VPNTransportTypes.Invalid) {
 669             configuredTransportTypeL3VPN = transType;
 670         }
 671     }
 672
 673     public void writeConfTransTypeConfigDS() {
 674         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, getConfTransportTypeIdentifier(),
 675             createConfTransportType(configuredTransportTypeL3VPN.getTransportType()));
 676     }
 677
 678     public L3VPNTransportTypes getConfiguredTransportTypeL3VPN() {
 679         return this.configuredTransportTypeL3VPN;
 680     }
 681
 682     public String getReqTransType() {
 683         if (configuredTransportTypeL3VPN == L3VPNTransportTypes.Invalid) {
 684             /*
 685              * Restart scenario, Read from the ConfigDS.
 686              * if the value is Unset, cache value as VxLAN.
 687              */
 688             LOG.trace("configureTransportType is not yet set.");
 689             Optional<ConfTransportTypeL3vpn> configuredTransTypeFromConfig;
 690             try {
 691                 configuredTransTypeFromConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 692                         LogicalDatastoreType.CONFIGURATION, getConfTransportTypeIdentifier());
 693             } catch (ExecutionException | InterruptedException e) {
 694                 LOG.error("getReqTransType: Exception while reading ConfTransportTypeL3vpn DS", e);
 695                 return null;
 696             }
 697             if (configuredTransTypeFromConfig.isPresent()) {
 698                 if (TunnelTypeGre.class.equals(configuredTransTypeFromConfig.get().getTransportType())) {
 699                     configuredTransportTypeL3VPN = L3VPNTransportTypes.GRE;
 700                 } else {
 701                     configuredTransportTypeL3VPN = L3VPNTransportTypes.VxLAN;
 702                 }
 703                 LOG.trace("configuredTransportType set from config DS to {}",
 704                     getConfiguredTransportTypeL3VPN().getTransportType());
 705             } else {
 706                 setConfTransType("L3VPN", L3VPNTransportTypes.VxLAN.getTransportType());
 707                 LOG.trace("configuredTransportType is not set in the Config DS. VxLAN as default will be used.");
 708             }
 709         } else {
 710             LOG.trace("configuredTransportType is set as {}", getConfiguredTransportTypeL3VPN().getTransportType());
 711         }
 712         return getConfiguredTransportTypeL3VPN().getTransportType();
 713     }
 714
 715     public InstanceIdentifier<ConfTransportTypeL3vpn> getConfTransportTypeIdentifier() {
 716         return InstanceIdentifier.builder(ConfTransportTypeL3vpn.class).build();
 717     }
 718
 719     private ConfTransportTypeL3vpn createConfTransportType(String type) {
 720         ConfTransportTypeL3vpn confTransType;
 721         switch (type) {
 722             case ITMConstants.TUNNEL_TYPE_GRE:
 723                 confTransType = new ConfTransportTypeL3vpnBuilder().setTransportType(TunnelTypeGre.class).build();
 724                 LOG.trace("Setting the confTransportType to GRE.");
 725                 break;
 726             case ITMConstants.TUNNEL_TYPE_VXLAN:
 727                 confTransType = new ConfTransportTypeL3vpnBuilder().setTransportType(TunnelTypeVxlan.class).build();
 728                 LOG.trace("Setting the confTransportType to VxLAN.");
 729                 break;
 730             default:
 731                 LOG.trace("Invalid transport type {} passed to Config DS ", type);
 732                 confTransType = null;
 733                 break;
 734         }
 735         return confTransType;
 736     }
 737
 738     public Class<? extends TunnelTypeBase> getReqTunType(String transportType) {
 739         switch (transportType) {
 740             case "VXLAN":
 741                 return TunnelTypeVxlan.class;
 742             case "GRE":
 743                 return TunnelTypeGre.class;
 744             default:
 745                 return TunnelTypeMplsOverGre.class;
 746         }
 747     }
 748
 749     public String getTransportTypeStr(String tunType) {
 750         if (tunType.equals(TunnelTypeVxlan.class.toString())) {
 751             return ITMConstants.TUNNEL_TYPE_VXLAN;
 752         } else if (tunType.equals(TunnelTypeGre.class.toString())) {
 753             return ITMConstants.TUNNEL_TYPE_GRE;
 754         } else if (tunType.equals(TunnelTypeMplsOverGre.class.toString())) {
 755             return ITMConstants.TUNNEL_TYPE_MPLSoGRE;
 756         } else {
 757             return ITMConstants.TUNNEL_TYPE_INVALID;
 758         }
 759     }
 760
 761     @Override
 762     @PreDestroy
 763     public void close() {
 764         LOG.info("{} close", getClass().getSimpleName());
 765     }
 766
 767     // TODO Clean up the exception handling
 768     @SuppressWarnings("checkstyle:IllegalCatch")
 769     @Nullable
 770     private String getTunnelRemoteNextHopPointer(Uint64 remoteDpnId, String nextHopIp,
 771                                                  Class<? extends TunnelTypeBase> tunnelType) {
 772         if (nextHopIp != null && !nextHopIp.isEmpty()) {
 773             try {
 774                 // here use the config for tunnel type param
 775                 return getTunnelInterfaceName(remoteDpnId,
 776                     org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder
 777                         .getDefaultInstance(nextHopIp), tunnelType);
 778             } catch (Exception ex) {
 779                 LOG.error("Error while retrieving nexthop pointer for nexthop {} remoteDpn {}",
 780                         nextHopIp, remoteDpnId, ex);
 781             }
 782         }
 783
 784         return null;
 785     }
 786
 787     private String getExtPortRemoteNextHopPointer(Uint64 remoteDpnId, ElanInstance elanInstance) {
 788         return elanService.getExternalElanInterface(elanInstance.getElanInstanceName(), remoteDpnId);
 789     }
 790
 791     /**
 792      * Get the interface type associated with the type of ELAN used for routing
 793      * traffic to/from remote compute nodes.
 794      *
 795      * @param elanInstance The elan instance
 796      * @return L2vlan for flat/VLAN network type and Tunnel otherwise
 797      */
 798     private Class<? extends InterfaceType> getInterfaceType(ElanInstance elanInstance) {
 799         Class<? extends SegmentTypeBase> segmentType = elanInstance.getSegmentType();
 800         if (SegmentTypeFlat.class.equals(segmentType) || SegmentTypeVlan.class.equals(segmentType)) {
 801             return L2vlan.class;
 802         }
 803
 804         return Tunnel.class;
 805     }
 806
 807     private ElanInstance getElanInstanceForPrefix(Uint32 vpnId, String prefixIp) {
 808         ElanInstance elanInstance = null;
 809         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, prefixIp);
 810         if (prefix != null) {
 811             if (prefix.getNetworkId() != null) {
 812                 elanInstance = elanService.getElanInstance(prefix.getNetworkId().getValue());
 813             }
 814         }
 815
 816         return elanInstance;
 817     }
 818
 819     static class AdjacencyResult {
 820         private final String interfaceName;
 821         private final Class<? extends InterfaceType> interfaceType;
 822         private final String nextHopIp;
 823         private final String prefix;
 824
 825         AdjacencyResult(String interfaceName, Class<? extends InterfaceType> interfaceType, String nextHopIp,
 826                         String prefix) {
 827             this.interfaceName = interfaceName;
 828             this.interfaceType = interfaceType;
 829             this.nextHopIp = nextHopIp;
 830             this.prefix = prefix;
 831         }
 832
 833         public String getInterfaceName() {
 834             return interfaceName;
 835         }
 836
 837         public Class<? extends InterfaceType> getInterfaceType() {
 838             return interfaceType;
 839         }
 840
 841         public String getNextHopIp() {
 842             return nextHopIp;
 843         }
 844
 845         public String getPrefix() {
 846             return prefix;
 847         }
 848
 849         @Override
 850         public int hashCode() {
 851             final int prime = 31;
 852             int result = 1;
 853             result = prime * result + (interfaceName == null ? 0 : interfaceName.hashCode());
 854             return result;
 855         }
 856
 857         @Override
 858         public boolean equals(Object obj) {
 859             if (obj == null) {
 860                 return false;
 861             }
 862
 863             if (getClass() != obj.getClass()) {
 864                 return false;
 865             } else {
 866                 AdjacencyResult other = (AdjacencyResult) obj;
 867                 return interfaceName.equals(other.interfaceName);
 868             }
 869         }
 870     }
 871
 872     protected long setupLoadBalancingNextHop(Uint32 parentVpnId, Uint64 dpnId,
 873             String destPrefix, List<BucketInfo> localBucketInfo, List<BucketInfo> remoteBucketInfo) {
 874         long remoteGroupId = createNextHopPointer(getRemoteSelectGroupKey(parentVpnId, destPrefix));
 875         if (remoteGroupId == FibConstants.INVALID_GROUP_ID) {
 876             LOG.error("Unable to allocate/retrieve remote groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 877             return remoteGroupId;
 878         }
 879         long localGroupId =  createNextHopPointer(getLocalSelectGroupKey(parentVpnId, destPrefix));
 880         if (localGroupId == FibConstants.INVALID_GROUP_ID) {
 881             LOG.error("Unable to allocate/retrieve local groupId for vpnId {} , prefix {}",
 882                 parentVpnId, destPrefix);
 883             return remoteGroupId;
 884         }
 885         List<BucketInfo> combinedBucketInfo = new ArrayList<>();
 886         combinedBucketInfo.addAll(localBucketInfo);
 887         combinedBucketInfo.addAll(remoteBucketInfo);
 888         GroupEntity remoteGroupEntity = MDSALUtil.buildGroupEntity(
 889                 dpnId, remoteGroupId, destPrefix, GroupTypes.GroupSelect, combinedBucketInfo);
 890         GroupEntity localGroupEntity = MDSALUtil.buildGroupEntity(
 891                 dpnId, localGroupId, destPrefix, GroupTypes.GroupSelect, localBucketInfo);
 892         String jobKey = FibUtil.getCreateLocalNextHopJobKey(parentVpnId, dpnId, destPrefix);
 893         jobCoordinator.enqueueJob(jobKey, () -> {
 894             mdsalApiManager.syncInstallGroup(remoteGroupEntity);
 895             if (!localBucketInfo.isEmpty()) {
 896                 mdsalApiManager.syncInstallGroup(localGroupEntity);
 897             }
 898             if (LOG.isDebugEnabled()) {
 899                 LOG.debug("Finished installing GroupEntity with jobCoordinator key {} remoteGroupEntity.groupId {}"
 900                         + "localGroupEntity.groupId {}  groupEntity.groupType {}", jobKey,
 901                         remoteGroupEntity.getGroupId(), localGroupEntity.getGroupId(),
 902                         remoteGroupEntity.getGroupType());
 903             }
 904             // Delete local group(if exists) if there is no local info.
 905             // Local group has to be deleted if all VMs in a compute is deleted.
 906             if (localBucketInfo.isEmpty()) {
 907                 LOG.debug("Deleting local group {} since no local nhs present for "
 908                         + "prefix {}", localGroupEntity.getGroupId(), destPrefix);
 909                 mdsalApiManager.syncRemoveGroup(localGroupEntity);
 910             }
 911             return Collections.emptyList();
 912         });
 913         return remoteGroupId;
 914     }
 915
 916     protected void deleteLoadBalancingNextHop(Uint32 parentVpnId, Uint64 dpnId, String destPrefix) {
 917         long remoteGroupId = createNextHopPointer(getRemoteSelectGroupKey(parentVpnId, destPrefix));
 918         if (remoteGroupId == FibConstants.INVALID_GROUP_ID) {
 919             LOG.error("Unable to allocate/retrieve remote groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 920         }
 921         long localGroupId = createNextHopPointer(getLocalSelectGroupKey(parentVpnId, destPrefix));
 922         if (localGroupId == FibConstants.INVALID_GROUP_ID) {
 923             LOG.error("Unable to allocate/retrieve local groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 924         }
 925         GroupEntity remoteGroupEntity = MDSALUtil.buildGroupEntity(
 926                 dpnId, remoteGroupId, destPrefix, GroupTypes.GroupSelect, Collections.emptyList());
 927         GroupEntity localGroupEntity = MDSALUtil.buildGroupEntity(
 928                 dpnId, localGroupId, destPrefix, GroupTypes.GroupSelect, Collections.emptyList());
 929         String jobKey = FibUtil.getCreateLocalNextHopJobKey(parentVpnId, dpnId, destPrefix);
 930         jobCoordinator.enqueueJob(jobKey, () -> {
 931             mdsalApiManager.syncRemoveGroup(remoteGroupEntity);
 932             mdsalApiManager.syncRemoveGroup(localGroupEntity);
 933             if (LOG.isDebugEnabled()) {
 934                 LOG.debug("Finished removing GroupEntity with jobCoordinator key {} remoteGroupEntity.groupId {}"
 935                     + "localGroupEntity.groupId {}", jobKey, remoteGroupId, localGroupId);
 936             }
 937             return Collections.emptyList();
 938         });
 939     }
 940
 941     long createNextHopGroups(Uint32 vpnId, String rd, Uint64 dpnId, VrfEntry vrfEntry,
 942             @Nullable Routes routes, List<Routes> vpnExtraRoutes) {
 943         List<BucketInfo> localBucketInfo = new ArrayList<>();
 944         List<Routes> clonedVpnExtraRoutes  = new ArrayList<>(vpnExtraRoutes);
 945         if (clonedVpnExtraRoutes.contains(routes)) {
 946             localBucketInfo.addAll(getBucketsForLocalNexthop(vpnId, dpnId, vrfEntry, routes));
 947             clonedVpnExtraRoutes.remove(routes);
 948         }
 949         List<BucketInfo> remoteBucketInfo =
 950             new ArrayList<>(getBucketsForRemoteNexthop(vpnId, dpnId, vrfEntry, rd, clonedVpnExtraRoutes));
 951         return setupLoadBalancingNextHop(vpnId, dpnId,
 952             vrfEntry.getDestPrefix(), localBucketInfo, remoteBucketInfo);
 953     }
 954
 955     private List<BucketInfo> getBucketsForLocalNexthop(Uint32 vpnId, Uint64 dpnId,
 956             VrfEntry vrfEntry, Routes routes) {
 957         @Nullable List<String> nexthopIpList = routes.getNexthopIpList();
 958         if (LOG.isDebugEnabled()) {
 959             LOG.debug("NexthopManager.getBucketsForLocalNexthop invoked with vpnId {} dpnId {} "
 960                             + " vrfEntry.routePaths {}, routes.nexthopList {}", vpnId, dpnId, vrfEntry.getRoutePaths(),
 961                 nexthopIpList);
 962         }
 963         List<BucketInfo> listBucketInfo = new CopyOnWriteArrayList<>();
 964         if (nexthopIpList != null) {
 965             nexthopIpList.parallelStream().forEach(nextHopIp -> {
 966                 String localNextHopIP;
 967                 if (isIpv4Address(nextHopIp)) {
 968                     localNextHopIP = nextHopIp + NwConstants.IPV4PREFIX;
 969                 } else {
 970                     localNextHopIP = nextHopIp + NwConstants.IPV6PREFIX;
 971                 }
 972                 Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 973                 if (localNextHopInfo != null) {
 974                     long groupId = getLocalNextHopGroup(vpnId, localNextHopIP);
 975                     if (groupId == FibConstants.INVALID_GROUP_ID) {
 976                         LOG.error("Unable to allocate groupId for vpnId {} , prefix {} , interface {}", vpnId,
 977                             vrfEntry.getDestPrefix(), localNextHopInfo.getVpnInterfaceName());
 978                         return;
 979                     }
 980                     List<ActionInfo> actionsInfos =
 981                         Collections.singletonList(new ActionGroup(groupId));
 982                     BucketInfo bucket = new BucketInfo(actionsInfos);
 983                     bucket.setWeight(1);
 984                     listBucketInfo.add(bucket);
 985                 }
 986             });
 987         }
 988         LOG.trace("LOCAL: listbucket {}, vpnId {}, dpnId {}, routes {}", listBucketInfo, vpnId, dpnId, routes);
 989         return listBucketInfo;
 990     }
 991
 992     private List<BucketInfo> getBucketsForRemoteNexthop(Uint32 vpnId, Uint64 dpnId, VrfEntry vrfEntry, String rd,
 993             List<Routes> vpnExtraRoutes) {
 994         List<BucketInfo> listBucketInfo = new ArrayList<>();
 995         Map<String, List<ActionInfo>> egressActionMap = new HashMap<>();
 996         vpnExtraRoutes.stream().filter(vpnExtraRoute -> vpnExtraRoute.getNexthopIpList() != null).forEach(
 997             vpnExtraRoute -> vpnExtraRoute.getNexthopIpList().forEach(nextHopIp -> {
 998                 String nextHopPrefixIp;
 999                 if (isIpv4Address(nextHopIp)) {
1000                     nextHopPrefixIp = nextHopIp + NwConstants.IPV4PREFIX;
1001                 } else {
1002                     nextHopPrefixIp = nextHopIp + NwConstants.IPV6PREFIX;
1003                 }
1004                 List<String> tepIpAddresses = fibUtil.getNextHopAddresses(rd, nextHopPrefixIp);
1005                 if (tepIpAddresses.isEmpty()) {
1006                     return;
1007                 }
1008                 // There would be only one nexthop address for a VM ip which would be the tep Ip
1009                 String tepIp = tepIpAddresses.get(0);
1010                 AdjacencyResult adjacencyResult = getRemoteNextHopPointer(dpnId, vpnId,
1011                     vrfEntry.getDestPrefix(), tepIp, TunnelTypeVxlan.class);
1012                 if (adjacencyResult == null) {
1013                     return;
1014                 }
1015                 String egressInterface = adjacencyResult.getInterfaceName();
1016                 if (!FibUtil.isTunnelInterface(adjacencyResult)) {
1017                     return;
1018                 }
1019                 Class<? extends TunnelTypeBase> tunnelType =
1020                     VpnExtraRouteHelper.getTunnelType(itmManager, egressInterface);
1021                 StateTunnelList ifState = null;
1022                 try {
1023                     ifState = fibUtil.getTunnelState(egressInterface);
1024                     if (ifState == null || ifState.getOperState() != TunnelOperStatus.Up) {
1025                         LOG.trace("Tunnel is not up for interface {}", egressInterface);
1026                         return;
1027                     }
1028                 } catch (ReadFailedException e) {
1029                     LOG.error("getBucketsForRemoteNexthop: error in fetching tunnel state for interface {}",
1030                         egressInterface, e);
1031                     return;
1032                 }
1033                 if (!TunnelTypeVxlan.class.equals(tunnelType)) {
1034                     return;
1035                 }
1036                 Uint32 label = FibUtil.getLabelFromRoutePaths(vrfEntry).get();
1037                 Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, nextHopPrefixIp);
1038                 if (prefixInfo == null) {
1039                     LOG.error("No prefix info found for prefix {} in rd {} for VPN {}", nextHopPrefixIp, rd,
1040                         vpnId);
1041                     return;
1042                 }
1043                 Uint32 tunnelId;
1044                 if (FibUtil.isVxlanNetwork(prefixInfo.getNetworkType())) {
1045                     tunnelId = prefixInfo.getSegmentationId();
1046                 } else {
1047                     LOG.warn("Network is not of type VXLAN for prefix {}."
1048                         + "Going with default Lport Tag.", prefixInfo.toString());
1049                     tunnelId = label;
1050                 }
1051                 List<ActionInfo> actionInfos = new ArrayList<>();
1052                 actionInfos.add(new ActionSetFieldTunnelId(Uint64.valueOf(tunnelId.longValue())));
1053                 String ifName = prefixInfo.getVpnInterfaceName();
1054                 String vpnName = fibUtil.getVpnNameFromId(vpnId);
1055                 if (vpnName == null) {
1056                     return;
1057                 }
1058                 String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, nextHopPrefixIp);
1059                 actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
1060                     new MacAddress(macAddress)));
1061                 List<ActionInfo> egressActions;
1062                 if (egressActionMap.containsKey(egressInterface)) {
1063                     egressActions = egressActionMap.get(egressInterface);
1064                 } else {
1065                     egressActions = getEgressActionsForInterface(egressInterface, actionInfos.size(),
1066                             true, vpnId, vrfEntry.getDestPrefix());
1067                     if (egressActions.isEmpty()) {
1068                         LOG.error("Skipping getBucketsForRemoteNexthop: Empty list of egress actions received for "
1069                                         + "interface {} on dpn {} for vpn {} prefix {} nextHop {}", ifName, dpnId,
1070                                 vpnId, vrfEntry.getDestPrefix(), nextHopPrefixIp);
1071                     }
1072                     egressActionMap.put(egressInterface, egressActions);
1073                 }
1074                 if (egressActions.isEmpty()) {
1075                     LOG.error("Failed to retrieve egress action for prefix {} route-paths {}"
1076                             + " interface {}." + " Aborting remote FIB entry creation.",
1077                         vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), egressInterface);
1078                 }
1079                 actionInfos.addAll(egressActions);
1080                 BucketInfo bucket = new BucketInfo(actionInfos);
1081                 bucket.setWeight(1);
1082                 listBucketInfo.add(bucket);
1083             }));
1084         LOG.trace("LOCAL: listbucket {}, rd {}, dpnId {}, routes {}", listBucketInfo, rd, dpnId, vpnExtraRoutes);
1085         return listBucketInfo;
1086     }
1087
1088     public void createDcGwLoadBalancingGroup(Uint64 dpnId, String destinationIp,
1089                                              Class<? extends TunnelTypeBase> tunnelType) {
1090         jobCoordinator.enqueueJob(FibHelper.getJobKeyForDcGwLoadBalancingGroup(dpnId), () -> {
1091             List<ListenableFuture<Void>> futures = new ArrayList<>();
1092             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operationalTx -> {
1093                 synchronized (getDcGateWaySyncKey(destinationIp)) {
1094                     FibUtil.addL3vpnDcGateWay(destinationIp, operationalTx);
1095                 }
1096                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, configTx -> {
1097                     List<String> availableDcGws = getDcGwIps();
1098                     Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
1099                     int noOfDcGws = availableDcGws.size();
1100                     if (noOfDcGws == 1) {
1101                         LOG.trace("There are no enough DC GateWays {} present to program LB group", availableDcGws);
1102                         return;
1103                     }
1104                     if (availableDcGws.contains(destinationIp)) {
1105                         availableDcGws.remove(destinationIp);
1106                     }
1107                     availableDcGws.forEach(dcGwIp -> {
1108                         List<String> dcGws = Arrays.asList(dcGwIp, destinationIp);
1109                         Collections.sort(dcGws);
1110                         String groupIdKey = FibUtil.getGreLbGroupKey(dcGws);
1111                         Long groupId = createNextHopPointer(groupIdKey);
1112                         List<Bucket> listBucket = new ArrayList<>();
1113                         for (int index = 0; index < dcGws.size(); index++) {
1114                             if (isTunnelUp(dcGws.get(index), dpnId, tunnelType)) {
1115                                 listBucket.add(buildBucketForDcGwLbGroup(dcGws.get(index),
1116                                         dpnId, index, tunnelType, true));
1117                             }
1118                         }
1119                         Group group = MDSALUtil.buildGroup(groupId, groupIdKey, GroupTypes.GroupSelect,
1120                                         MDSALUtil.buildBucketLists(listBucket));
1121                         mdsalApiManager.addGroup(configTx, dpnId, group);
1122                         FibUtil.updateLbGroupInfo(dpnId, groupIdKey, groupId.toString(), operationalTx);
1123                         LOG.trace("LB group {} towards DC-GW installed on dpn {}. Group - {}",
1124                                 groupIdKey, dpnId, group);
1125                     });
1126                 }));
1127             }));
1128             return futures;
1129         }, RETRY_COUNT);
1130     }
1131
1132     private String getDcGateWaySyncKey(String destinationIp) {
1133         String mutex = new StringBuilder().append("L3vpncDcGateWay").append(destinationIp).toString();
1134         return mutex.intern();
1135     }
1136
1137     private List<String> getDcGwIps() {
1138         InstanceIdentifier<DcGatewayIpList> dcGatewayIpListid =
1139                 InstanceIdentifier.builder(DcGatewayIpList.class).build();
1140         DcGatewayIpList dcGatewayIpListConfig;
1141         try {
1142             dcGatewayIpListConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1143                     LogicalDatastoreType.CONFIGURATION, dcGatewayIpListid).orElse(null);
1144         } catch (ExecutionException | InterruptedException e) {
1145             LOG.error("getDcGwIps: Exception while reading DcGatewayIpList DS", e);
1146             return Collections.emptyList();
1147         }
1148         if (dcGatewayIpListConfig == null) {
1149             return Collections.emptyList();
1150         }
1151         return new ArrayList<DcGatewayIp>(dcGatewayIpListConfig.getDcGatewayIp().values())
1152                 .stream()
1153                 .filter(dcGwIp -> dcGwIp.getTunnnelType().equals(TunnelTypeMplsOverGre.class))
1154                 .map(dcGwIp -> dcGwIp.getIpAddress().stringValue()).sorted()
1155                 .collect(toList());
1156     }
1157
1158     private boolean isTunnelUp(String dcGwIp, Uint64 dpnId, Class<? extends TunnelTypeBase> tunnelType) {
1159         String tunnelName = getTunnelRemoteNextHopPointer(dpnId, dcGwIp, tunnelType);
1160         if (tunnelName != null) {
1161             InstanceIdentifier<StateTunnelList> tunnelStateId =
1162                     InstanceIdentifier.builder(TunnelsState.class).child(
1163                             StateTunnelList.class, new StateTunnelListKey(tunnelName)).build();
1164             try {
1165                 return SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
1166                         tunnelStateId).map(StateTunnelList::getOperState)
1167                         .orElse(TunnelOperStatus.Down) == TunnelOperStatus.Up;
1168             } catch (ExecutionException | InterruptedException e) {
1169                 LOG.error("isTunnelUp: Exception while reading StateTunnelList DS for tunnel {} tunnelType {}",
1170                         tunnelName, tunnelType, e);
1171                 return false;
1172             }
1173         }
1174         return false;
1175     }
1176
1177     private List<Action> getEgressActions(String interfaceName, int actionKey) {
1178         List<Action> actions = Collections.emptyList();
1179         try {
1180             GetEgressActionsForInterfaceInputBuilder egressAction =
1181                     new GetEgressActionsForInterfaceInputBuilder().setIntfName(interfaceName).setActionKey(actionKey);
1182             Future<RpcResult<GetEgressActionsForInterfaceOutput>> result =
1183                     odlInterfaceRpcService.getEgressActionsForInterface(egressAction.build());
1184             RpcResult<GetEgressActionsForInterfaceOutput> rpcResult = result.get();
1185             if (!rpcResult.isSuccessful()) {
1186                 LOG.warn("RPC Call to Get egress actions for interface {} returned with Errors {}",
1187                         interfaceName, rpcResult.getErrors());
1188             } else {
1189                 actions = new ArrayList<Action>(rpcResult.getResult().nonnullAction().values());
1190             }
1191         } catch (InterruptedException | ExecutionException e) {
1192             LOG.warn("Exception when egress actions for interface {}", interfaceName, e);
1193         }
1194         return actions;
1195     }
1196
1197     /**
1198      * This method is invoked when the neighbor is removed from DS.
1199      * All the LB groups which point to the given destination will be deleted.
1200      */
1201     public void removeDcGwLoadBalancingGroup(Uint64 dpnId,
1202             String destinationIp) {
1203         jobCoordinator.enqueueJob(FibHelper.getJobKeyForDcGwLoadBalancingGroup(dpnId), () -> {
1204             List<String> availableDcGws = fibUtil.getL3VpnDcGateWays();
1205             if (availableDcGws.contains(destinationIp)) {
1206                 availableDcGws.remove(destinationIp);
1207             }
1208             List<ListenableFuture<Void>> futures = new ArrayList<>();
1209             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operationalTx -> {
1210                 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx -> {
1211                     availableDcGws.forEach(dcGwIp -> {
1212                         List<String> dcGws = Arrays.asList(dcGwIp, destinationIp);
1213                         Collections.sort(dcGws);
1214                         String nextHopKey = FibUtil.getGreLbGroupKey(dcGws);
1215                         Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
1216                         if (!optionalNextHops.isPresent()) {
1217                             return;
1218                         }
1219                         Nexthops nexthops = optionalNextHops.get();
1220                         final String groupId = nexthops.getGroupId();
1221                         final long groupIdValue = Long.parseLong(groupId);
1222                         Group group = MDSALUtil.buildGroup(groupIdValue, nextHopKey, GroupTypes.GroupSelect,
1223                                 MDSALUtil.buildBucketLists(Collections.emptyList()));
1224                         LOG.trace("Removed LB group {} on dpn {}", group, dpnId);
1225                         try {
1226                             mdsalApiManager.removeGroup(configTx, dpnId, group);
1227                         } catch (ExecutionException | InterruptedException e) {
1228                             LOG.error("Group removal failed for group {} with exception", groupId, e);
1229                         }
1230                         removeNextHopPointer(nextHopKey);
1231                         FibUtil.removeOrUpdateNextHopInfo(dpnId, nextHopKey, groupId, nexthops, operationalTx);
1232                     });
1233                     synchronized (getDcGateWaySyncKey(destinationIp)) {
1234                         FibUtil.removeL3vpnDcGateWay(destinationIp, operationalTx);
1235                     }
1236                 }));
1237             }));
1238             return futures;
1239         }, RETRY_COUNT);
1240     }
1241
1242     /**
1243      * This method is invoked when the tunnel status is deleted.
1244      * All the buckets which point to given destination will be marked down.
1245      */
1246     public void updateDcGwLoadBalancingGroup(Uint64 dpnId, String destinationIp,
1247             boolean isTunnelUp, Class<? extends TunnelTypeBase> tunnelType) {
1248         jobCoordinator.enqueueJob(FibHelper.getJobKeyForDcGwLoadBalancingGroup(dpnId), () -> {
1249             List<String> availableDcGws = fibUtil.getL3VpnDcGateWays();
1250             if (availableDcGws.contains(destinationIp)) {
1251                 availableDcGws.remove(destinationIp);
1252             }
1253             List<ListenableFuture<Void>> futures = new ArrayList<>();
1254             futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, configTx -> {
1255                 availableDcGws.forEach(dcGwIp -> {
1256                     List<String> dcGws = Arrays.asList(dcGwIp, destinationIp);
1257                     Collections.sort(dcGws);
1258                     String nextHopKey = FibUtil.getGreLbGroupKey(dcGws);
1259                     int bucketId = dcGws.indexOf(destinationIp);
1260                     Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
1261                     if (!optionalNextHops.isPresent()) {
1262                         return;
1263                     }
1264                     Nexthops nexthops = optionalNextHops.get();
1265                     final String groupId = nexthops.getGroupId();
1266                     final long groupIdValue = Long.parseLong(groupId);
1267                     Bucket bucket = buildBucketForDcGwLbGroup(destinationIp, dpnId, bucketId, tunnelType, isTunnelUp);
1268                     LOG.trace("updated bucket {} to group {} on dpn {}.", bucket, groupId, dpnId);
1269                     try {
1270                         mdsalApiManager.addBucket(configTx, dpnId, groupIdValue, bucket);
1271                     } catch (ExecutionException | InterruptedException e) {
1272                         LOG.error("Bucket addition failed for bucket {} with exception", bucketId, e);
1273                     }
1274                 });
1275             }));
1276             return futures;
1277         }, RETRY_COUNT);
1278     }
1279
1280     private Bucket buildBucketForDcGwLbGroup(String ipAddress, Uint64 dpnId,
1281             int index, Class<? extends TunnelTypeBase> tunnelType, boolean isTunnelUp) {
1282         List<Action> listAction = new ArrayList<>();
1283         // ActionKey 0 goes to mpls label.
1284         int actionKey = 1;
1285         listAction.add(new ActionPushMpls().buildAction());
1286         listAction.add(new ActionRegMove(actionKey++, FibConstants.NXM_REG_MAPPING
1287                 .get(index), 0, 19).buildAction());
1288         String tunnelInterfaceName = getTunnelInterfaceName(dpnId, IpAddressBuilder.getDefaultInstance(ipAddress),
1289             tunnelType);
1290         List<Action> egressActions = getEgressActions(tunnelInterfaceName, actionKey++);
1291         if (!egressActions.isEmpty()) {
1292             listAction.addAll(getEgressActions(tunnelInterfaceName, actionKey++));
1293         } else {
1294             // clear off actions if there is no egress actions.
1295             listAction = Collections.emptyList();
1296         }
1297         long watchPort = MDSALUtil.WATCH_PORT;
1298         if (!isTunnelUp) {
1299             watchPort = 0xFFFFFFFEL;
1300         }
1301         //OVS expects a non-zero weight value for load balancing to happen in select groups
1302         return MDSALUtil.buildBucket(listAction, SELECT_GROUP_WEIGHT, index,
1303                 watchPort, MDSALUtil.WATCH_GROUP);
1304     }
1305
1306     public void programDcGwLoadBalancingGroup(Uint64 dpnId, String destinationIp,
1307                                               int addRemoveOrUpdate, boolean isTunnelUp,
1308                                               Class<? extends TunnelTypeBase> tunnelType) {
1309         if (NwConstants.ADD_FLOW == addRemoveOrUpdate) {
1310             createDcGwLoadBalancingGroup(dpnId, destinationIp, tunnelType);
1311         } else if (NwConstants.DEL_FLOW == addRemoveOrUpdate) {
1312             removeDcGwLoadBalancingGroup(dpnId, destinationIp);
1313         } else if (NwConstants.MOD_FLOW == addRemoveOrUpdate) {
1314             updateDcGwLoadBalancingGroup(dpnId, destinationIp, isTunnelUp, tunnelType);
1315         }
1316     }
1317 }