fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/NexthopManager.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  11 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  12 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  13
  14 import com.google.common.base.Optional;
  15 import com.google.common.base.Preconditions;
  16 import java.math.BigInteger;
  17 import java.util.ArrayList;
  18 import java.util.Collections;
  19 import java.util.HashMap;
  20 import java.util.List;
  21 import java.util.Locale;
  22 import java.util.Map;
  23 import java.util.concurrent.CopyOnWriteArrayList;
  24 import java.util.concurrent.ExecutionException;
  25 import java.util.concurrent.Future;
  26 import javax.annotation.PreDestroy;
  27 import javax.inject.Inject;
  28 import javax.inject.Singleton;
  29 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  30 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  31 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  32 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  33 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  34 import org.opendaylight.genius.itm.globals.ITMConstants;
  35 import org.opendaylight.genius.mdsalutil.ActionInfo;
  36 import org.opendaylight.genius.mdsalutil.BucketInfo;
  37 import org.opendaylight.genius.mdsalutil.GroupEntity;
  38 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  39 import org.opendaylight.genius.mdsalutil.NwConstants;
  40 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  41 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  42 import org.opendaylight.genius.mdsalutil.actions.ActionOutput;
  43 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
  44 import org.opendaylight.genius.mdsalutil.actions.ActionPushVlan;
  45 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  46 import org.opendaylight.genius.mdsalutil.actions.ActionRegMove;
  47 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  48 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetSource;
  49 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  50 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldVlanVid;
  51 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  52 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  53 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  54 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  55 import org.opendaylight.netvirt.fibmanager.api.L3VPNTransportTypes;
  56 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  57 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev140508.L2vlan;
  58 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev140508.Tunnel;
  59 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
  60 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfaceType;
  61 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.OutputActionCase;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.PushVlanActionCase;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.SetFieldCase;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInput;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInputBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutput;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInput;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInputBuilder;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolOutput;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInput;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInputBuilder;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdOutput;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceInputBuilder;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceOutput;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelOperStatus;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelsState;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelListKey;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelInputBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelOutput;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameInputBuilder;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameOutput;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.lockmanager.rev160413.LockManagerService;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupInput;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupInputBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupOutput;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.SalGroupService;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupId;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupRef;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.Buckets;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeRef;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeBase;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeFlat;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeVlan;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.L3nexthop;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthops;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthopsKey;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopKey;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacencies;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacenciesBuilder;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacenciesKey;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpn;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpnBuilder;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.dpid.l3vpn.lb.nexthops.DpnLbNexthops;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.lb.nexthops.Nexthops;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.add.group.input.buckets.bucket.action.action.NxActionResubmitRpcAddGroupCase;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.nodes.node.table.flow.instructions.instruction.instruction.apply.actions._case.apply.actions.action.action.NxActionRegLoadNodesNodeTableFlowApplyActionsCase;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.nx.action.reg.load.grouping.NxRegLoad;
 130 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 131 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 132 import org.opendaylight.yangtools.yang.common.RpcResult;
 133 import org.slf4j.Logger;
 134 import org.slf4j.LoggerFactory;
 135
 136 @Singleton
 137 public class NexthopManager implements AutoCloseable {
 138     private static final Logger LOG = LoggerFactory.getLogger(NexthopManager.class);
 139     private static final String NEXTHOP_ID_POOL_NAME = "nextHopPointerPool";
 140     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 141     private static final long WAIT_TIME_TO_ACQUIRE_LOCK = 3000L;
 142     private static final int SELECT_GROUP_WEIGHT = 1;
 143
 144     private final DataBroker dataBroker;
 145     private final ManagedNewTransactionRunner txRunner;
 146     private final IMdsalApiManager mdsalApiManager;
 147     private final OdlInterfaceRpcService odlInterfaceRpcService;
 148     private final ItmRpcService itmManager;
 149     private final IdManagerService idManager;
 150     private final IElanService elanService;
 151     private final LockManagerService lockManager;
 152     private final SalGroupService salGroupService;
 153     private final JobCoordinator jobCoordinator;
 154     private final FibUtil fibUtil;
 155     private final IInterfaceManager interfaceManager;
 156     private volatile L3VPNTransportTypes configuredTransportTypeL3VPN = L3VPNTransportTypes.Invalid;
 157
 158     /**
 159      * Provides nexthop functions.
 160      * Creates group ID pool
 161      *
 162      * @param dataBroker       - dataBroker reference
 163      * @param mdsalApiManager  - mdsalApiManager reference
 164      * @param idManager        - idManager reference
 165      * @param odlInterfaceRpcService - odlInterfaceRpcService reference
 166      * @param itmManager       - itmManager reference
 167      */
 168     @Inject
 169     public NexthopManager(final DataBroker dataBroker,
 170                           final IMdsalApiManager mdsalApiManager,
 171                           final IdManagerService idManager,
 172                           final OdlInterfaceRpcService odlInterfaceRpcService,
 173                           final ItmRpcService itmManager,
 174                           final LockManagerService lockManager,
 175                           final IElanService elanService,
 176                           final SalGroupService salGroupService,
 177                           final JobCoordinator jobCoordinator,
 178                           final FibUtil fibUtil,
 179                           final IInterfaceManager interfaceManager) {
 180         this.dataBroker = dataBroker;
 181         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 182         this.mdsalApiManager = mdsalApiManager;
 183         this.idManager = idManager;
 184         this.odlInterfaceRpcService = odlInterfaceRpcService;
 185         this.itmManager = itmManager;
 186         this.elanService = elanService;
 187         this.salGroupService = salGroupService;
 188         this.jobCoordinator = jobCoordinator;
 189         this.fibUtil = fibUtil;
 190         this.lockManager = lockManager;
 191         this.interfaceManager = interfaceManager;
 192         createIdPool();
 193     }
 194
 195     private void createIdPool() {
 196         CreateIdPoolInput createPool = new CreateIdPoolInputBuilder()
 197             .setPoolName(NEXTHOP_ID_POOL_NAME)
 198             .setLow(150000L)
 199             .setHigh(175000L)
 200             .build();
 201         try {
 202             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPool);
 203             if (result != null && result.get().isSuccessful()) {
 204                 LOG.info("Created IdPool for NextHopPointerPool");
 205             }
 206         } catch (InterruptedException | ExecutionException e) {
 207             LOG.error("Failed to create idPool for NextHopPointerPool", e);
 208         }
 209     }
 210
 211     private String getNextHopKey(long vpnId, String ipAddress) {
 212         return "nexthop." + vpnId + ipAddress;
 213     }
 214
 215     public ItmRpcService getItmManager() {
 216         return itmManager;
 217     }
 218
 219     protected long createNextHopPointer(String nexthopKey) {
 220         AllocateIdInput getIdInput = new AllocateIdInputBuilder()
 221             .setPoolName(NEXTHOP_ID_POOL_NAME).setIdKey(nexthopKey)
 222             .build();
 223         //TODO: Proper error handling once IdManager code is complete
 224         try {
 225             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
 226             RpcResult<AllocateIdOutput> rpcResult = result.get();
 227             return rpcResult.getResult().getIdValue();
 228         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 229             LOG.trace("", e);
 230         }
 231         return 0;
 232     }
 233
 234     protected void removeNextHopPointer(String nexthopKey) {
 235         ReleaseIdInput idInput = new ReleaseIdInputBuilder()
 236             .setPoolName(NEXTHOP_ID_POOL_NAME)
 237             .setIdKey(nexthopKey).build();
 238         try {
 239             RpcResult<ReleaseIdOutput> rpcResult = idManager.releaseId(idInput).get();
 240             if (!rpcResult.isSuccessful()) {
 241                 LOG.error("RPC Call to Get Unique Id for nexthopKey {} returned with Errors {}",
 242                         nexthopKey, rpcResult.getErrors());
 243             }
 244         } catch (InterruptedException | ExecutionException e) {
 245             LOG.warn("Exception when getting Unique Id for key {}", nexthopKey, e);
 246         }
 247     }
 248
 249     protected List<ActionInfo> getEgressActionsForInterface(final String ifName, int actionKey,
 250                                                             boolean isTunnelInterface) {
 251         List<Action> actions;
 252         try {
 253             if (isTunnelInterface && interfaceManager.isItmDirectTunnelsEnabled()) {
 254                 RpcResult<GetEgressActionsForTunnelOutput> rpcResult =
 255                         itmManager.getEgressActionsForTunnel(new GetEgressActionsForTunnelInputBuilder()
 256                                 .setIntfName(ifName).build()).get();
 257                 if (!rpcResult.isSuccessful()) {
 258                     LOG.error("RPC Call to Get egress tunnel actions for interface {} returned with Errors {}",
 259                             ifName, rpcResult.getErrors());
 260                     return Collections.emptyList();
 261                 } else {
 262                     actions = rpcResult.getResult().getAction();
 263                 }
 264             } else {
 265                 RpcResult<GetEgressActionsForInterfaceOutput> rpcResult = odlInterfaceRpcService
 266                         .getEgressActionsForInterface(new GetEgressActionsForInterfaceInputBuilder()
 267                                 .setIntfName(ifName).build()).get();
 268                 if (!rpcResult.isSuccessful()) {
 269                     LOG.error("RPC Call to Get egress vm actions for interface {} returned with Errors {}",
 270                             ifName, rpcResult.getErrors());
 271                     return Collections.emptyList();
 272                 } else {
 273                     actions = rpcResult.getResult().getAction();
 274                 }
 275             }
 276             List<ActionInfo> listActionInfo = new ArrayList<>();
 277             for (Action action : actions) {
 278                 actionKey = action.key().getOrder() + actionKey;
 279                 org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action
 280                     actionClass = action.getAction();
 281                 if (actionClass instanceof OutputActionCase) {
 282                     listActionInfo.add(new ActionOutput(actionKey,
 283                         ((OutputActionCase) actionClass).getOutputAction().getOutputNodeConnector()));
 284                 } else if (actionClass instanceof PushVlanActionCase) {
 285                     listActionInfo.add(new ActionPushVlan(actionKey));
 286                 } else if (actionClass instanceof SetFieldCase) {
 287                     if (((SetFieldCase) actionClass).getSetField().getVlanMatch() != null) {
 288                         int vlanVid = ((SetFieldCase) actionClass).getSetField().getVlanMatch()
 289                             .getVlanId().getVlanId().getValue();
 290                         listActionInfo.add(new ActionSetFieldVlanVid(actionKey, vlanVid));
 291                     }
 292                 } else if (actionClass instanceof NxActionResubmitRpcAddGroupCase) {
 293                     Short tableId = ((NxActionResubmitRpcAddGroupCase) actionClass).getNxResubmit().getTable();
 294                     listActionInfo.add(new ActionNxResubmit(actionKey, tableId));
 295                 } else if (actionClass instanceof NxActionRegLoadNodesNodeTableFlowApplyActionsCase) {
 296                     NxRegLoad nxRegLoad =
 297                         ((NxActionRegLoadNodesNodeTableFlowApplyActionsCase) actionClass).getNxRegLoad();
 298                     listActionInfo.add(new ActionRegLoad(actionKey, NxmNxReg6.class,
 299                         nxRegLoad.getDst().getStart(), nxRegLoad.getDst().getEnd(),
 300                         nxRegLoad.getValue().longValue()));
 301                 }
 302             }
 303             return listActionInfo;
 304         } catch (InterruptedException | ExecutionException e) {
 305             LOG.warn("Exception when egress actions for interface {}", ifName, e);
 306         }
 307         LOG.warn("Exception when egress actions for interface {}", ifName);
 308         return Collections.emptyList();
 309     }
 310
 311     protected String getTunnelInterfaceName(BigInteger srcDpId, BigInteger dstDpId) {
 312         Class<? extends TunnelTypeBase> tunType = getReqTunType(getReqTransType().toUpperCase(Locale.getDefault()));
 313         Future<RpcResult<GetTunnelInterfaceNameOutput>> result;
 314         try {
 315             result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 316                 .setSourceDpid(srcDpId)
 317                 .setDestinationDpid(dstDpId)
 318                 .setTunnelType(tunType)
 319                 .build());
 320             RpcResult<GetTunnelInterfaceNameOutput> rpcResult = result.get();
 321             if (!rpcResult.isSuccessful()) {
 322                 LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
 323             } else {
 324                 return rpcResult.getResult().getInterfaceName();
 325             }
 326         } catch (InterruptedException | ExecutionException e) {
 327             LOG.warn("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstDpId, e);
 328         }
 329         return null;
 330     }
 331
 332     protected String getTunnelInterfaceName(BigInteger srcDpId, org.opendaylight.yang.gen.v1.urn.ietf.params
 333         .xml.ns.yang.ietf.inet.types.rev130715.IpAddress dstIp, Class<? extends TunnelTypeBase> tunnelType) {
 334         Future<RpcResult<GetInternalOrExternalInterfaceNameOutput>> result;
 335         try {
 336             LOG.debug("Trying to fetch tunnel interface name for source dpn {} destIp {} tunType {}", srcDpId,
 337                     dstIp.stringValue(), tunnelType.getName());
 338             result = itmManager.getInternalOrExternalInterfaceName(new GetInternalOrExternalInterfaceNameInputBuilder()
 339                 .setSourceDpid(srcDpId)
 340                 .setDestinationIp(dstIp)
 341                 .setTunnelType(tunnelType)
 342                 .build());
 343             RpcResult<GetInternalOrExternalInterfaceNameOutput> rpcResult = result.get();
 344             if (!rpcResult.isSuccessful()) {
 345                 LOG.warn("RPC Call to getTunnelInterfaceName returned with Errors {}", rpcResult.getErrors());
 346             } else {
 347                 return rpcResult.getResult().getInterfaceName();
 348             }
 349         } catch (InterruptedException | ExecutionException e) {
 350             LOG.warn("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstIp, e);
 351         }
 352         return null;
 353     }
 354
 355
 356     public long getLocalNextHopGroup(long vpnId,
 357             String ipNextHopAddress) {
 358         long groupId = createNextHopPointer(getNextHopKey(vpnId, ipNextHopAddress));
 359         if (groupId == FibConstants.INVALID_GROUP_ID) {
 360             LOG.error("Unable to allocate groupId for vpnId {} , prefix {}", vpnId, ipNextHopAddress);
 361         }
 362         return groupId;
 363     }
 364
 365     public long createLocalNextHop(long vpnId, BigInteger dpnId, String ifName,
 366                                    String primaryIpAddress, String currDestIpPrefix,
 367                                    String gwMacAddress) {
 368         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 369         if (vpnName == null) {
 370             return 0;
 371         }
 372         String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, primaryIpAddress);
 373
 374         long groupId = createNextHopPointer(getNextHopKey(vpnId, primaryIpAddress));
 375         if (groupId == 0) {
 376             LOG.error("Unable to allocate groupId for vpnId {} , IntfName {}, primaryIpAddress {} curIpPrefix {}",
 377                     vpnId, ifName, primaryIpAddress, currDestIpPrefix);
 378             return groupId;
 379         }
 380         String nextHopLockStr = vpnId + primaryIpAddress;
 381         String jobKey = FibUtil.getCreateLocalNextHopJobKey(vpnId, dpnId, currDestIpPrefix);
 382         jobCoordinator.enqueueJob(jobKey, () -> {
 383             try {
 384                 if (FibUtil.lockCluster(lockManager, nextHopLockStr, WAIT_TIME_TO_ACQUIRE_LOCK)) {
 385                     VpnNexthop nexthop = getVpnNexthop(vpnId, primaryIpAddress);
 386                     LOG.trace("nexthop: {} retrieved for vpnId {}, prefix {}, ifName {} on dpn {}", nexthop, vpnId,
 387                             primaryIpAddress, ifName, dpnId);
 388                     if (nexthop == null) {
 389                         String encMacAddress = macAddress == null
 390                                 ? fibUtil.getMacAddressFromPrefix(ifName, vpnName, primaryIpAddress) : macAddress;
 391                         List<BucketInfo> listBucketInfo = new ArrayList<>();
 392                         List<ActionInfo> listActionInfo = new ArrayList<>();
 393                         int actionKey = 0;
 394                         // MAC re-write
 395                         if (encMacAddress != null) {
 396                             if (gwMacAddress != null) {
 397                                 LOG.trace("The Local NextHop Group Source Mac {} for VpnInterface {} on VPN {}",
 398                                         gwMacAddress, ifName, vpnId);
 399                                 listActionInfo.add(new ActionSetFieldEthernetSource(actionKey++,
 400                                         new MacAddress(gwMacAddress)));
 401                             }
 402                             listActionInfo.add(new ActionSetFieldEthernetDestination(actionKey++,
 403                                     new MacAddress(encMacAddress)));
 404                             // listActionInfo.add(0, new ActionPopMpls());
 405                         } else {
 406                             // FIXME: Log message here.
 407                             LOG.debug("mac address for new local nexthop is null");
 408                         }
 409                         listActionInfo.addAll(getEgressActionsForInterface(ifName, actionKey, false));
 410                         BucketInfo bucket = new BucketInfo(listActionInfo);
 411
 412                         listBucketInfo.add(bucket);
 413                         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, groupId, primaryIpAddress,
 414                                 GroupTypes.GroupAll, listBucketInfo);
 415                         LOG.trace("Install LNH Group: id {}, mac address {}, interface {} for prefix {}", groupId,
 416                                 encMacAddress, ifName, primaryIpAddress);
 417                         //Try to install group directly on the DPN bypassing the FRM, in order to avoid waiting for the
 418                         // group to get installed before programming the flows
 419                         installGroupOnDpn(groupId, dpnId, primaryIpAddress, listBucketInfo,
 420                                 getNextHopKey(vpnId, primaryIpAddress), GroupTypes.GroupAll);
 421                         // install Group
 422                         mdsalApiManager.syncInstallGroup(groupEntity);
 423                         // update MD-SAL DS
 424                         addVpnNexthopToDS(dpnId, vpnId, primaryIpAddress, currDestIpPrefix, groupId);
 425
 426                     } else {
 427                         // Ignore adding new prefix , if it already exists
 428                         List<IpAdjacencies> prefixesList = nexthop.getIpAdjacencies();
 429                         IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currDestIpPrefix).build();
 430                         if (prefixesList != null && prefixesList.contains(prefix)) {
 431                             LOG.trace("Prefix {} is already present in l3nextHop {} ", currDestIpPrefix, nexthop);
 432                         } else {
 433                             IpAdjacenciesBuilder ipPrefixesBuilder =
 434                                     new IpAdjacenciesBuilder().withKey(new IpAdjacenciesKey(currDestIpPrefix));
 435                             LOG.trace("Updating prefix {} to vpnNextHop {} Operational DS", currDestIpPrefix, nexthop);
 436                             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
 437                                     getVpnNextHopIpPrefixIdentifier(vpnId, primaryIpAddress, currDestIpPrefix),
 438                                     ipPrefixesBuilder.build());
 439                         }
 440                     }
 441                 }
 442             } finally {
 443                 FibUtil.unlockCluster(lockManager, nextHopLockStr);
 444             }
 445             return Collections.emptyList();
 446         });
 447         return groupId;
 448     }
 449
 450     private void installGroupOnDpn(long groupId, BigInteger dpnId, String groupName, List<BucketInfo> bucketsInfo,
 451                                      String nextHopKey, GroupTypes groupType) {
 452         NodeRef nodeRef = FibUtil.buildNodeRef(dpnId);
 453         Buckets buckets = FibUtil.buildBuckets(bucketsInfo);
 454         GroupRef groupRef = new GroupRef(FibUtil.buildGroupInstanceIdentifier(groupId, dpnId));
 455         AddGroupInput input = new AddGroupInputBuilder().setNode(nodeRef).setGroupId(new GroupId(groupId))
 456                 .setBuckets(buckets).setGroupRef(groupRef).setGroupType(groupType)
 457                 .setGroupName(groupName).build();
 458         Future<RpcResult<AddGroupOutput>> groupStats = salGroupService.addGroup(input);
 459         RpcResult<AddGroupOutput> rpcResult = null;
 460         try {
 461             rpcResult = groupStats.get();
 462             if (rpcResult != null && rpcResult.isSuccessful()) {
 463                 LOG.info("Group {} with key {} has been successfully installed directly on dpn {}.", groupId,
 464                         nextHopKey, dpnId);
 465             } else {
 466                 LOG.error("Unable to install group {} with key {} directly on dpn {} due to {}.", groupId, nextHopKey,
 467                         dpnId, rpcResult != null ? rpcResult.getErrors() : null);
 468             }
 469         } catch (InterruptedException | ExecutionException e) {
 470             LOG.error("Error while installing group {} directly on dpn {}", groupId, dpnId);
 471         }
 472     }
 473
 474     protected void addVpnNexthopToDS(BigInteger dpnId, long vpnId, String primaryIpAddr,
 475                                      String currIpAddr, long egressPointer) {
 476         InstanceIdentifierBuilder<VpnNexthops> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
 477             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId));
 478
 479         List<IpAdjacencies> ipPrefixesList = new ArrayList<>();
 480         IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currIpAddr).build();
 481         ipPrefixesList.add(prefix);
 482         // Add nexthop to vpn node
 483         VpnNexthop nh = new VpnNexthopBuilder()
 484             .withKey(new VpnNexthopKey(primaryIpAddr))
 485             .setDpnId(dpnId)
 486             .setIpAdjacencies(ipPrefixesList)
 487             .setEgressPointer(egressPointer).build();
 488
 489         InstanceIdentifier<VpnNexthop> id1 = idBuilder
 490             .child(VpnNexthop.class, new VpnNexthopKey(primaryIpAddr)).build();
 491         LOG.trace("Adding vpnnextHop {} to Operational DS", nh);
 492         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id1, nh);
 493
 494     }
 495
 496     protected InstanceIdentifier<IpAdjacencies> getVpnNextHopIpPrefixIdentifier(long vpnId, String primaryIpAddress,
 497                                                                                 String ipPrefix) {
 498         InstanceIdentifier<IpAdjacencies> id = InstanceIdentifier.builder(L3nexthop.class)
 499                 .child(VpnNexthops.class, new VpnNexthopsKey(vpnId))
 500                 .child(VpnNexthop.class, new VpnNexthopKey(primaryIpAddress))
 501                 .child(IpAdjacencies.class, new IpAdjacenciesKey(ipPrefix)).build();
 502         return id;
 503     }
 504
 505     protected VpnNexthop getVpnNexthop(long vpnId, String ipAddress) {
 506
 507         // check if vpn node is there
 508         InstanceIdentifierBuilder<VpnNexthops> idBuilder =
 509             InstanceIdentifier.builder(L3nexthop.class).child(VpnNexthops.class,
 510                 new VpnNexthopsKey(vpnId));
 511         InstanceIdentifier<VpnNexthops> id = idBuilder.build();
 512         Optional<VpnNexthops> vpnNexthops = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 513         if (vpnNexthops.isPresent()) {
 514             // get nexthops list for vpn
 515             List<VpnNexthop> nexthops = vpnNexthops.get().getVpnNexthop();
 516             for (VpnNexthop nexthop : nexthops) {
 517                 if (nexthop.getIpAddress().equals(ipAddress)) {
 518                     // return nexthop
 519                     LOG.trace("VpnNextHop : {}", nexthop);
 520                     return nexthop;
 521                 }
 522             }
 523             // return null if not found
 524         }
 525         return null;
 526     }
 527
 528     public AdjacencyResult getRemoteNextHopPointer(BigInteger remoteDpnId, long vpnId, String prefixIp,
 529             String nextHopIp, Class<? extends TunnelTypeBase> tunnelType) {
 530         String egressIfName = null;
 531         LOG.trace("getRemoteNextHopPointer: input [remoteDpnId {}, vpnId {}, prefixIp {}, nextHopIp {} ]", remoteDpnId,
 532             vpnId, prefixIp, nextHopIp);
 533
 534         Class<? extends InterfaceType> egressIfType;
 535         ElanInstance elanInstance = getElanInstanceForPrefix(vpnId, prefixIp);
 536         if (elanInstance != null) {
 537             egressIfType = getInterfaceType(elanInstance);
 538         } else {
 539             LOG.warn("Failed to determine network type for prefixIp {} using tunnel", prefixIp);
 540             egressIfType = Tunnel.class;
 541         }
 542
 543         if (Tunnel.class.equals(egressIfType)) {
 544             egressIfName = getTunnelRemoteNextHopPointer(remoteDpnId, nextHopIp, tunnelType);
 545         } else {
 546             egressIfName = getExtPortRemoteNextHopPointer(remoteDpnId, elanInstance);
 547         }
 548
 549         LOG.trace("NextHop pointer for prefixIp {} vpnId {} dpnId {} is {}", prefixIp, vpnId, remoteDpnId,
 550             egressIfName);
 551         return egressIfName != null ? new AdjacencyResult(egressIfName, egressIfType, nextHopIp,
 552                 prefixIp) : null;
 553     }
 554
 555     public BigInteger getDpnForPrefix(long vpnId, String prefixIp) {
 556         VpnNexthop vpnNexthop = getVpnNexthop(vpnId, prefixIp);
 557         BigInteger localDpnId = vpnNexthop == null ? null : vpnNexthop.getDpnId();
 558         return localDpnId;
 559     }
 560
 561     private void removeVpnNexthopFromDS(long vpnId, String ipPrefix) {
 562
 563         InstanceIdentifierBuilder<VpnNexthop> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
 564             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId))
 565             .child(VpnNexthop.class, new VpnNexthopKey(ipPrefix));
 566         InstanceIdentifier<VpnNexthop> id = idBuilder.build();
 567         // remove from DS
 568         LOG.trace("Removing vpn next hop from datastore : {}", id);
 569         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 570     }
 571
 572     public void removeLocalNextHop(BigInteger dpnId, Long vpnId, String primaryIpAddress, String currDestIpPrefix) {
 573         String nextHopLockStr = vpnId + primaryIpAddress;
 574         try {
 575             if (FibUtil.lockCluster(lockManager, nextHopLockStr, WAIT_TIME_TO_ACQUIRE_LOCK)) {
 576                 VpnNexthop nh = getVpnNexthop(vpnId, primaryIpAddress);
 577                 if (nh != null) {
 578                     List<IpAdjacencies> prefixesList = nh.getIpAdjacencies();
 579                     IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currDestIpPrefix).build();
 580                     prefixesList.remove(prefix);
 581                     if (prefixesList.isEmpty()) { //remove the group only if there are no more flows using this group
 582                         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, nh.getEgressPointer(),
 583                                 primaryIpAddress, GroupTypes.GroupAll, Collections.EMPTY_LIST);
 584                         // remove Group ...
 585                         mdsalApiManager.removeGroup(groupEntity);
 586                         //update MD-SAL DS
 587                         removeVpnNexthopFromDS(vpnId, primaryIpAddress);
 588                         //release groupId
 589                         removeNextHopPointer(getNextHopKey(vpnId, primaryIpAddress));
 590                         LOG.debug("Local Next hop {} for {} {} on dpn {} successfully deleted",
 591                                 nh.getEgressPointer(), vpnId, primaryIpAddress, dpnId);
 592                     } else {
 593                         //remove the currIpPrefx from IpPrefixList of the vpnNexthop
 594                         LOG.trace("Removing the prefix {} from vpnNextHop {} Operational DS", currDestIpPrefix, nh);
 595                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
 596                                 getVpnNextHopIpPrefixIdentifier(vpnId, primaryIpAddress, currDestIpPrefix));
 597                     }
 598                 } else {
 599                     //throw error
 600                     LOG.error("Local NextHop for VpnId {} curIpPrefix {} on dpn {} primaryIpAddress {} not deleted",
 601                             vpnId, currDestIpPrefix, dpnId, primaryIpAddress);
 602                 }
 603             }
 604         } finally {
 605             FibUtil.unlockCluster(lockManager, nextHopLockStr);
 606         }
 607     }
 608
 609     public void setConfTransType(String service, String transportType) {
 610
 611         if (!service.equalsIgnoreCase("L3VPN")) {
 612             LOG.error("Incorrect service {} provided for setting the transport type.", service);
 613             return;
 614         }
 615
 616         L3VPNTransportTypes transType = L3VPNTransportTypes.validateTransportType(transportType
 617                 .toUpperCase(Locale.getDefault()));
 618
 619         if (transType != L3VPNTransportTypes.Invalid) {
 620             configuredTransportTypeL3VPN = transType;
 621         }
 622     }
 623
 624     public void writeConfTransTypeConfigDS() {
 625         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, getConfTransportTypeIdentifier(),
 626             createConfTransportType(configuredTransportTypeL3VPN.getTransportType()));
 627     }
 628
 629     public L3VPNTransportTypes getConfiguredTransportTypeL3VPN() {
 630         return this.configuredTransportTypeL3VPN;
 631     }
 632
 633     public String getReqTransType() {
 634         if (configuredTransportTypeL3VPN == L3VPNTransportTypes.Invalid) {
 635             /*
 636              * Restart scenario, Read from the ConfigDS.
 637              * if the value is Unset, cache value as VxLAN.
 638              */
 639             LOG.trace("configureTransportType is not yet set.");
 640             Optional<ConfTransportTypeL3vpn> configuredTransTypeFromConfig =
 641                 MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, getConfTransportTypeIdentifier());
 642
 643             if (configuredTransTypeFromConfig.isPresent()) {
 644                 if (configuredTransTypeFromConfig.get().getTransportType().equals(TunnelTypeGre.class)) {
 645                     configuredTransportTypeL3VPN = L3VPNTransportTypes.GRE;
 646                 } else {
 647                     configuredTransportTypeL3VPN = L3VPNTransportTypes.VxLAN;
 648                 }
 649                 LOG.trace("configuredTransportType set from config DS to {}",
 650                     getConfiguredTransportTypeL3VPN().getTransportType());
 651             } else {
 652                 setConfTransType("L3VPN", L3VPNTransportTypes.VxLAN.getTransportType());
 653                 LOG.trace("configuredTransportType is not set in the Config DS. VxLAN as default will be used.");
 654             }
 655         } else {
 656             LOG.trace("configuredTransportType is set as {}", getConfiguredTransportTypeL3VPN().getTransportType());
 657         }
 658         return getConfiguredTransportTypeL3VPN().getTransportType();
 659     }
 660
 661     public InstanceIdentifier<ConfTransportTypeL3vpn> getConfTransportTypeIdentifier() {
 662         return InstanceIdentifier.builder(ConfTransportTypeL3vpn.class).build();
 663     }
 664
 665     private ConfTransportTypeL3vpn createConfTransportType(String type) {
 666         ConfTransportTypeL3vpn confTransType;
 667         switch (type) {
 668             case ITMConstants.TUNNEL_TYPE_GRE:
 669                 confTransType = new ConfTransportTypeL3vpnBuilder().setTransportType(TunnelTypeGre.class).build();
 670                 LOG.trace("Setting the confTransportType to GRE.");
 671                 break;
 672             case ITMConstants.TUNNEL_TYPE_VXLAN:
 673                 confTransType = new ConfTransportTypeL3vpnBuilder().setTransportType(TunnelTypeVxlan.class).build();
 674                 LOG.trace("Setting the confTransportType to VxLAN.");
 675                 break;
 676             default:
 677                 LOG.trace("Invalid transport type {} passed to Config DS ", type);
 678                 confTransType = null;
 679                 break;
 680         }
 681         return confTransType;
 682     }
 683
 684     public Class<? extends TunnelTypeBase> getReqTunType(String transportType) {
 685         switch (transportType) {
 686             case "VXLAN":
 687                 return TunnelTypeVxlan.class;
 688             case "GRE":
 689                 return TunnelTypeGre.class;
 690             default:
 691                 return TunnelTypeMplsOverGre.class;
 692         }
 693     }
 694
 695     public String getTransportTypeStr(String tunType) {
 696         if (tunType.equals(TunnelTypeVxlan.class.toString())) {
 697             return ITMConstants.TUNNEL_TYPE_VXLAN;
 698         } else if (tunType.equals(TunnelTypeGre.class.toString())) {
 699             return ITMConstants.TUNNEL_TYPE_GRE;
 700         } else if (tunType.equals(TunnelTypeMplsOverGre.class.toString())) {
 701             return ITMConstants.TUNNEL_TYPE_MPLSoGRE;
 702         } else {
 703             return ITMConstants.TUNNEL_TYPE_INVALID;
 704         }
 705     }
 706
 707     @Override
 708     @PreDestroy
 709     public void close() {
 710         LOG.info("{} close", getClass().getSimpleName());
 711     }
 712
 713     // TODO Clean up the exception handling
 714     @SuppressWarnings("checkstyle:IllegalCatch")
 715     private String getTunnelRemoteNextHopPointer(BigInteger remoteDpnId, String nextHopIp,
 716                                                  Class<? extends TunnelTypeBase> tunnelType) {
 717         if (nextHopIp != null && !nextHopIp.isEmpty()) {
 718             try {
 719                 // here use the config for tunnel type param
 720                 return getTunnelInterfaceName(remoteDpnId,
 721                     org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder
 722                         .getDefaultInstance(nextHopIp), tunnelType);
 723             } catch (Exception ex) {
 724                 LOG.error("Error while retrieving nexthop pointer for nexthop {} remoteDpn {}",
 725                         nextHopIp, remoteDpnId, ex);
 726             }
 727         }
 728
 729         return null;
 730     }
 731
 732     private String getExtPortRemoteNextHopPointer(BigInteger remoteDpnId, ElanInstance elanInstance) {
 733         return elanService.getExternalElanInterface(elanInstance.getElanInstanceName(), remoteDpnId);
 734     }
 735
 736     /**
 737      * Get the interface type associated with the type of ELAN used for routing
 738      * traffic to/from remote compute nodes.
 739      *
 740      * @param elanInstance The elan instance
 741      * @return L2vlan for flat/VLAN network type and Tunnel otherwise
 742      */
 743     private Class<? extends InterfaceType> getInterfaceType(ElanInstance elanInstance) {
 744         Class<? extends SegmentTypeBase> segmentType = elanInstance.getSegmentType();
 745         if (SegmentTypeFlat.class.equals(segmentType) || SegmentTypeVlan.class.equals(segmentType)) {
 746             return L2vlan.class;
 747         }
 748
 749         return Tunnel.class;
 750     }
 751
 752     private ElanInstance getElanInstanceForPrefix(long vpnId, String prefixIp) {
 753         ElanInstance elanInstance = null;
 754         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, prefixIp);
 755         if (prefix != null) {
 756             if (prefix.getNetworkId() != null) {
 757                 elanInstance = elanService.getElanInstance(prefix.getNetworkId().getValue());
 758             }
 759         }
 760
 761         return elanInstance;
 762     }
 763
 764     static class AdjacencyResult {
 765         private final String interfaceName;
 766         private final Class<? extends InterfaceType> interfaceType;
 767         private final String nextHopIp;
 768         private final String prefix;
 769
 770         AdjacencyResult(String interfaceName, Class<? extends InterfaceType> interfaceType, String nextHopIp,
 771                         String prefix) {
 772             this.interfaceName = interfaceName;
 773             this.interfaceType = interfaceType;
 774             this.nextHopIp = nextHopIp;
 775             this.prefix = prefix;
 776         }
 777
 778         public String getInterfaceName() {
 779             return interfaceName;
 780         }
 781
 782         public Class<? extends InterfaceType> getInterfaceType() {
 783             return interfaceType;
 784         }
 785
 786         public String getNextHopIp() {
 787             return nextHopIp;
 788         }
 789
 790         public String getPrefix() {
 791             return prefix;
 792         }
 793
 794         @Override
 795         public int hashCode() {
 796             final int prime = 31;
 797             int result = 1;
 798             result = prime * result + (interfaceName == null ? 0 : interfaceName.hashCode());
 799             return result;
 800         }
 801
 802         @Override
 803         public boolean equals(Object obj) {
 804             if (obj == null) {
 805                 return false;
 806             }
 807
 808             boolean result = false;
 809             if (getClass() != obj.getClass()) {
 810                 return result;
 811             } else {
 812                 AdjacencyResult other = (AdjacencyResult) obj;
 813                 result = interfaceName.equals(other.interfaceName);
 814             }
 815             return result;
 816         }
 817     }
 818
 819     protected long setupLoadBalancingNextHop(Long parentVpnId, BigInteger dpnId,
 820             String destPrefix, List<BucketInfo> listBucketInfo, boolean addOrRemove) {
 821         long groupId = createNextHopPointer(getNextHopKey(parentVpnId, destPrefix));
 822         if (groupId == FibConstants.INVALID_GROUP_ID) {
 823             LOG.error("Unable to allocate/retrieve groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 824             return groupId;
 825         }
 826         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(
 827                 dpnId, groupId, destPrefix, GroupTypes.GroupSelect, listBucketInfo);
 828         String jobKey = FibUtil.getCreateLocalNextHopJobKey(parentVpnId, dpnId, destPrefix);
 829         jobCoordinator.enqueueJob(jobKey, () -> {
 830             if (addOrRemove) {
 831                 mdsalApiManager.syncInstallGroup(groupEntity);
 832                 if (LOG.isDebugEnabled()) {
 833                     LOG.debug("Finished installing GroupEntity with jobCoordinator key {} groupEntity.groupId {}"
 834                             + "  groupEntity.groupType {}", jobKey, groupEntity.getGroupId(),
 835                             groupEntity.getGroupType());
 836                 }
 837             } else {
 838                 mdsalApiManager.removeGroup(groupEntity);
 839                 if (LOG.isDebugEnabled()) {
 840                     LOG.debug("Finished removing GroupEntity with jobCoordinator key {} groupEntity.groupId {}"
 841                             + "  groupEntity.groupType {}", jobKey, groupEntity.getGroupId(),
 842                             groupEntity.getGroupType());
 843                 }
 844             }
 845             return Collections.emptyList();
 846         });
 847         return groupId;
 848     }
 849
 850     long createNextHopGroups(Long vpnId, String rd, BigInteger dpnId, VrfEntry vrfEntry,
 851             Routes routes, List<Routes> vpnExtraRoutes) {
 852         List<BucketInfo> listBucketInfo = new ArrayList<>();
 853         List<Routes> clonedVpnExtraRoutes  = new ArrayList<>(vpnExtraRoutes);
 854         if (clonedVpnExtraRoutes.contains(routes)) {
 855             listBucketInfo.addAll(getBucketsForLocalNexthop(vpnId, dpnId, vrfEntry, routes));
 856             clonedVpnExtraRoutes.remove(routes);
 857         }
 858         listBucketInfo.addAll(getBucketsForRemoteNexthop(vpnId, dpnId, vrfEntry, rd, clonedVpnExtraRoutes));
 859         return setupLoadBalancingNextHop(vpnId, dpnId, vrfEntry.getDestPrefix(), listBucketInfo,true);
 860     }
 861
 862     private List<BucketInfo> getBucketsForLocalNexthop(Long vpnId, BigInteger dpnId,
 863             VrfEntry vrfEntry, Routes routes) {
 864         if (LOG.isDebugEnabled()) {
 865             LOG.debug("NexthopManager.getBucketsForLocalNexthop invoked with vpnId {} dpnId {} "
 866                             + " vrfEntry.routePaths {}, routes.nexthopList {}", vpnId, dpnId, vrfEntry.getRoutePaths(),
 867                     routes.getNexthopIpList());
 868         }
 869         List<BucketInfo> listBucketInfo = new CopyOnWriteArrayList<>();
 870         routes.getNexthopIpList().parallelStream().forEach(nextHopIp -> {
 871             String localNextHopIP;
 872             if (isIpv4Address(nextHopIp)) {
 873                 localNextHopIP = nextHopIp + NwConstants.IPV4PREFIX;
 874             } else {
 875                 localNextHopIP = nextHopIp + NwConstants.IPV6PREFIX;
 876             }
 877             Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 878             if (localNextHopInfo != null) {
 879                 long groupId = getLocalNextHopGroup(vpnId, localNextHopIP);
 880                 if (groupId == FibConstants.INVALID_GROUP_ID) {
 881                     LOG.error("Unable to allocate groupId for vpnId {} , prefix {} , interface {}", vpnId,
 882                             vrfEntry.getDestPrefix(), localNextHopInfo.getVpnInterfaceName());
 883                     return;
 884                 }
 885                 List<ActionInfo> actionsInfos =
 886                         Collections.singletonList(new ActionGroup(groupId));
 887                 BucketInfo bucket = new BucketInfo(actionsInfos);
 888                 bucket.setWeight(1);
 889                 listBucketInfo.add(bucket);
 890             }
 891         });
 892         LOG.trace("LOCAL: listbucket {}, vpnId {}, dpnId {}, routes {}", listBucketInfo, vpnId, dpnId, routes);
 893         return listBucketInfo;
 894     }
 895
 896     private List<BucketInfo> getBucketsForRemoteNexthop(Long vpnId, BigInteger dpnId, VrfEntry vrfEntry, String rd,
 897             List<Routes> vpnExtraRoutes) {
 898         List<BucketInfo> listBucketInfo = new ArrayList<>();
 899         Map<String, List<ActionInfo>> egressActionMap = new HashMap<>();
 900         vpnExtraRoutes.forEach(vpnExtraRoute -> vpnExtraRoute.getNexthopIpList().forEach(nextHopIp -> {
 901             String nextHopPrefixIp;
 902             if (isIpv4Address(nextHopIp)) {
 903                 nextHopPrefixIp = nextHopIp + NwConstants.IPV4PREFIX;
 904             } else {
 905                 nextHopPrefixIp = nextHopIp + NwConstants.IPV6PREFIX;
 906             }
 907             List<String> tepIpAddresses = fibUtil.getNextHopAddresses(rd, nextHopPrefixIp);
 908             if (tepIpAddresses.isEmpty()) {
 909                 return;
 910             }
 911             // There would be only one nexthop address for a VM ip which would be the tep Ip
 912             String tepIp = tepIpAddresses.get(0);
 913             AdjacencyResult adjacencyResult = getRemoteNextHopPointer(dpnId, vpnId,
 914                     vrfEntry.getDestPrefix(), tepIp, TunnelTypeVxlan.class);
 915             if (adjacencyResult == null) {
 916                 return;
 917             }
 918             String egressInterface = adjacencyResult.getInterfaceName();
 919             if (!FibUtil.isTunnelInterface(adjacencyResult)) {
 920                 return;
 921             }
 922             Class<? extends TunnelTypeBase> tunnelType = VpnExtraRouteHelper.getTunnelType(itmManager, egressInterface);
 923             StateTunnelList ifState = fibUtil.getTunnelState(egressInterface);
 924             if (ifState == null || ifState.getOperState() != TunnelOperStatus.Up) {
 925                 LOG.trace("Tunnel not up {}", egressInterface);
 926                 return;
 927             }
 928             if (!TunnelTypeVxlan.class.equals(tunnelType)) {
 929                 return;
 930             }
 931             Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).get();
 932             Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, nextHopPrefixIp);
 933             if (prefixInfo == null) {
 934                 LOG.error("No prefix info found for prefix {} in rd {} for VPN {}", nextHopPrefixIp, rd,
 935                         vpnId);
 936                 return;
 937             }
 938             BigInteger tunnelId;
 939             if (FibUtil.isVxlanNetwork(prefixInfo.getNetworkType())) {
 940                 tunnelId = BigInteger.valueOf(prefixInfo.getSegmentationId());
 941             } else {
 942                 LOG.warn("Network is not of type VXLAN for prefix {}."
 943                         + "Going with default Lport Tag.", prefixInfo.toString());
 944                 tunnelId = BigInteger.valueOf(label);
 945             }
 946             List<ActionInfo> actionInfos = new ArrayList<>();
 947             actionInfos.add(new ActionSetFieldTunnelId(tunnelId));
 948             String ifName = prefixInfo.getVpnInterfaceName();
 949             String vpnName = fibUtil.getVpnNameFromId(vpnId);
 950             if (vpnName == null) {
 951                 return;
 952             }
 953             String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, nextHopPrefixIp);
 954             actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
 955                     new MacAddress(macAddress)));
 956             List<ActionInfo> egressActions;
 957             if (egressActionMap.containsKey(egressInterface)) {
 958                 egressActions = egressActionMap.get(egressInterface);
 959             } else {
 960                 egressActions = getEgressActionsForInterface(egressInterface, actionInfos.size(), true);
 961                 egressActionMap.put(egressInterface, egressActions);
 962             }
 963             if (egressActions.isEmpty()) {
 964                 LOG.error("Failed to retrieve egress action for prefix {} route-paths {}"
 965                         + " interface {}." + " Aborting remote FIB entry creation.",
 966                         vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), egressInterface);
 967             }
 968             actionInfos.addAll(egressActions);
 969             BucketInfo bucket = new BucketInfo(actionInfos);
 970             bucket.setWeight(1);
 971             listBucketInfo.add(bucket);
 972         }));
 973         LOG.trace("LOCAL: listbucket {}, rd {}, dpnId {}, routes {}", listBucketInfo, rd, dpnId, vpnExtraRoutes);
 974         return listBucketInfo;
 975     }
 976
 977     public void createDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId, String destinationIp,
 978                                              Class<? extends TunnelTypeBase> tunnelType) {
 979         Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
 980         int noOfDcGws = availableDcGws.size();
 981         if (noOfDcGws == 1) {
 982             LOG.trace("There are no enough DC GateWays {} present to program LB group", availableDcGws);
 983             return;
 984         }
 985         // TODO : Place the logic to construct all possible DC-GW combination here.
 986         String groupIdKey = FibUtil.getGreLbGroupKey(availableDcGws);
 987         Long groupId = createNextHopPointer(groupIdKey);
 988         List<Bucket> listBucket = new ArrayList<>();
 989         for (int index = 0; index < noOfDcGws; index++) {
 990             if (isTunnelUp(availableDcGws.get(index), dpnId, tunnelType)) {
 991                 listBucket.add(buildBucketForDcGwLbGroup(availableDcGws.get(index), dpnId, index, tunnelType));
 992             }
 993         }
 994         Group group = MDSALUtil.buildGroup(groupId, groupIdKey, GroupTypes.GroupSelect,
 995                         MDSALUtil.buildBucketLists(listBucket));
 996         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 997             confTx -> mdsalApiManager.addGroup(confTx, dpnId, group)), LOG, "Error adding load-balancing group");
 998         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
 999             operTx -> FibUtil.updateLbGroupInfo(dpnId, destinationIp, groupIdKey, groupId.toString(), operTx)), LOG,
1000             "Error updating load-balancing group info");
1001         LOG.trace("LB group {} towards DC-GW installed on dpn {}. Group - {}", groupIdKey, dpnId, group);
1002     }
1003
1004     private boolean isTunnelUp(String dcGwIp, BigInteger dpnId, Class<? extends TunnelTypeBase> tunnelType) {
1005         String tunnelName = getTunnelRemoteNextHopPointer(dpnId, dcGwIp, tunnelType);
1006         if (tunnelName != null) {
1007             InstanceIdentifier<StateTunnelList> tunnelStateId =
1008                     InstanceIdentifier.builder(TunnelsState.class).child(
1009                             StateTunnelList.class, new StateTunnelListKey(tunnelName)).build();
1010             return MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, tunnelStateId)
1011                     .toJavaUtil().map(StateTunnelList::getOperState)
1012                     .orElse(TunnelOperStatus.Down) == TunnelOperStatus.Up;
1013         }
1014         return false;
1015     }
1016
1017     private List<Action> getEgressActions(String interfaceName, int actionKey) {
1018         List<Action> actions = Collections.emptyList();
1019         try {
1020             GetEgressActionsForInterfaceInputBuilder egressAction =
1021                     new GetEgressActionsForInterfaceInputBuilder().setIntfName(interfaceName).setActionKey(actionKey);
1022             Future<RpcResult<GetEgressActionsForInterfaceOutput>> result =
1023                     odlInterfaceRpcService.getEgressActionsForInterface(egressAction.build());
1024             RpcResult<GetEgressActionsForInterfaceOutput> rpcResult = result.get();
1025             if (!rpcResult.isSuccessful()) {
1026                 LOG.warn("RPC Call to Get egress actions for interface {} returned with Errors {}",
1027                         interfaceName, rpcResult.getErrors());
1028             } else {
1029                 actions = rpcResult.getResult().getAction();
1030             }
1031         } catch (InterruptedException | ExecutionException e) {
1032             LOG.warn("Exception when egress actions for interface {}", interfaceName, e);
1033         }
1034         return actions;
1035     }
1036
1037     /**
1038      * This method is invoked when the tunnel state is removed from DS.
1039      * If the there is just one DC-GW left in configuration then the LB groups can be deleted.
1040      * Otherwise, the groups are just updated.
1041      */
1042     public void removeOrUpdateDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId,
1043             String destinationIp) {
1044         Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
1045         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
1046             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
1047                 int noOfDcGws = availableDcGws.size();
1048                 // If availableDcGws does not contain the destination Ip it means this is a configuration delete.
1049                 if (!availableDcGws.contains(destinationIp)) {
1050                     availableDcGws.add(destinationIp);
1051                     Collections.sort(availableDcGws);
1052                 }
1053                 // TODO : Place the logic to construct all possible DC-GW combination here.
1054                 int bucketId = availableDcGws.indexOf(destinationIp);
1055                 Optional<DpnLbNexthops> dpnLbNextHops = fibUtil.getDpnLbNexthops(dpnId, destinationIp);
1056                 if (!dpnLbNextHops.isPresent()) {
1057                     return;
1058                 }
1059                 List<String> nextHopKeys = dpnLbNextHops.get().getNexthopKey();
1060                 for (String nextHopKey : nextHopKeys) {
1061                     Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
1062                     if (!optionalNextHops.isPresent()) {
1063                         return;
1064                     }
1065                     Nexthops nexthops = optionalNextHops.get();
1066                     final String groupId = nexthops.getGroupId();
1067                     final long groupIdValue = Long.parseLong(groupId);
1068                     if (noOfDcGws > 1) {
1069                         mdsalApiManager.removeBucket(confTx, dpnId, groupIdValue, bucketId);
1070                     } else {
1071                         LOG.trace("Removed LB group {} on dpn {}", groupIdValue, dpnId);
1072                         mdsalApiManager.removeGroup(confTx, dpnId, groupIdValue);
1073                         removeNextHopPointer(nextHopKey);
1074                     }
1075                     // When the DC-GW is removed from configuration.
1076                     if (noOfDcGws != availableDcGws.size()) {
1077                         FibUtil.removeOrUpdateNextHopInfo(dpnId, nextHopKey, groupId, nexthops, operTx);
1078                     }
1079                 }
1080                 FibUtil.removeDpnIdToNextHopInfo(destinationIp, dpnId, operTx);
1081             }), LOG, "Error removing or updating load-balancing group");
1082         }), LOG, "Error removing or updating load-balancing group");
1083     }
1084
1085     /**
1086      * This method is invoked when the tunnel status is updated.
1087      * The bucket is directly removed/added based on the operational status of the tunnel.
1088      */
1089     public void updateDcGwLoadBalancingGroup(List<String> availableDcGws,
1090             BigInteger dpnId, String destinationIp, boolean isTunnelUp, Class<? extends TunnelTypeBase> tunnelType) {
1091         Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
1092         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
1093             // TODO : Place the logic to construct all possible DC-GW combination here.
1094             int bucketId = availableDcGws.indexOf(destinationIp);
1095             Optional<DpnLbNexthops> dpnLbNextHops = fibUtil.getDpnLbNexthops(dpnId, destinationIp);
1096             if (!dpnLbNextHops.isPresent()) {
1097                 return;
1098             }
1099             List<String> nextHopKeys = dpnLbNextHops.get().getNexthopKey();
1100             for (String nextHopKey : nextHopKeys) {
1101                 Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
1102                 if (!optionalNextHops.isPresent()) {
1103                     return;
1104                 }
1105                 Nexthops nexthops = optionalNextHops.get();
1106                 final String groupId = nexthops.getGroupId();
1107                 final long groupIdValue = Long.parseLong(groupId);
1108                 if (isTunnelUp) {
1109                     Bucket bucket = buildBucketForDcGwLbGroup(destinationIp, dpnId, bucketId, tunnelType);
1110                     LOG.trace("Added bucket {} to group {} on dpn {}.", bucket, groupId, dpnId);
1111                     mdsalApiManager.addBucket(confTx, dpnId, groupIdValue, bucket);
1112                 } else {
1113                     LOG.trace("Removed bucketId {} from group {} on dpn {}.", bucketId, groupId, dpnId);
1114                     mdsalApiManager.removeBucket(confTx, dpnId, groupIdValue, bucketId);
1115                 }
1116             }
1117         }), LOG, "Error updating load-balancing group");
1118     }
1119
1120     private Bucket buildBucketForDcGwLbGroup(String ipAddress, BigInteger dpnId, int index,
1121                                              Class<? extends TunnelTypeBase> tunnelType) {
1122         List<Action> listAction = new ArrayList<>();
1123         // ActionKey 0 goes to mpls label.
1124         int actionKey = 1;
1125         listAction.add(new ActionPushMpls().buildAction());
1126         listAction.add(new ActionRegMove(actionKey++, FibConstants.NXM_REG_MAPPING
1127                 .get(index), 0, 19).buildAction());
1128         String tunnelInterfaceName = getTunnelInterfaceName(dpnId, IpAddressBuilder.getDefaultInstance(ipAddress),
1129             tunnelType);
1130         List<Action> egressActions = getEgressActions(tunnelInterfaceName, actionKey++);
1131         if (!egressActions.isEmpty()) {
1132             listAction.addAll(getEgressActions(tunnelInterfaceName, actionKey++));
1133         } else {
1134             // clear off actions if there is no egress actions.
1135             listAction = Collections.emptyList();
1136         }
1137         //OVS expects a non-zero weight value for load balancing to happen in select groups
1138         return MDSALUtil.buildBucket(listAction, SELECT_GROUP_WEIGHT, index,
1139                 MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP);
1140     }
1141
1142     public void programDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId, String destinationIp,
1143                                               int addRemoveOrUpdate, boolean isTunnelUp,
1144                                               Class<? extends TunnelTypeBase> tunnelType) {
1145         if (NwConstants.ADD_FLOW == addRemoveOrUpdate) {
1146             createDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp, tunnelType);
1147         } else if (NwConstants.DEL_FLOW == addRemoveOrUpdate) {
1148             removeOrUpdateDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp);
1149         } else if (NwConstants.MOD_FLOW == addRemoveOrUpdate) {
1150             updateDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp, isTunnelUp, tunnelType);
1151         }
1152     }
1153 }