fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/NexthopManager.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  11
  12 import com.google.common.base.Optional;
  13 import com.google.common.base.Preconditions;
  14 import java.math.BigInteger;
  15 import java.util.ArrayList;
  16 import java.util.Collections;
  17 import java.util.HashMap;
  18 import java.util.List;
  19 import java.util.Locale;
  20 import java.util.Map;
  21 import java.util.concurrent.CopyOnWriteArrayList;
  22 import java.util.concurrent.ExecutionException;
  23 import java.util.concurrent.Future;
  24 import javax.annotation.PreDestroy;
  25 import javax.inject.Inject;
  26 import javax.inject.Singleton;
  27 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  28 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  29 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  30 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  31 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  32 import org.opendaylight.genius.itm.globals.ITMConstants;
  33 import org.opendaylight.genius.mdsalutil.ActionInfo;
  34 import org.opendaylight.genius.mdsalutil.BucketInfo;
  35 import org.opendaylight.genius.mdsalutil.GroupEntity;
  36 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  37 import org.opendaylight.genius.mdsalutil.NwConstants;
  38 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  39 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  40 import org.opendaylight.genius.mdsalutil.actions.ActionOutput;
  41 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
  42 import org.opendaylight.genius.mdsalutil.actions.ActionPushVlan;
  43 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  44 import org.opendaylight.genius.mdsalutil.actions.ActionRegMove;
  45 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  46 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetSource;
  47 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  48 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldVlanVid;
  49 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  50 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  51 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  52 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  53 import org.opendaylight.netvirt.fibmanager.api.L3VPNTransportTypes;
  54 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  55 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev140508.L2vlan;
  56 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev140508.Tunnel;
  57 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  58 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfaceType;
  59 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  60 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.OperStatus;
  61 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  62 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.OutputActionCase;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.PushVlanActionCase;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.SetFieldCase;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInput;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInputBuilder;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutput;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInput;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInputBuilder;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolOutput;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInput;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInputBuilder;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdOutput;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceInputBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceOutput;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelOperStatus;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelsState;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelListKey;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetEgressActionsForTunnelInputBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameInputBuilder;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameOutput;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.lockmanager.rev160413.LockManagerService;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupInput;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupInputBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupOutput;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.SalGroupService;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupId;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupRef;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.Buckets;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeRef;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeBase;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeFlat;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeVlan;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.L3nexthop;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthops;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthopsKey;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopKey;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacencies;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacenciesBuilder;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.vpnnexthop.IpAdjacenciesKey;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpn;
 121 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpnBuilder;
 122 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.dpid.l3vpn.lb.nexthops.DpnLbNexthops;
 123 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.lb.nexthops.Nexthops;
 124 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 125 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 126 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
 127 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6;
 128 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.add.group.input.buckets.bucket.action.action.NxActionResubmitRpcAddGroupCase;
 129 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.nodes.node.table.flow.instructions.instruction.instruction.apply.actions._case.apply.actions.action.action.NxActionRegLoadNodesNodeTableFlowApplyActionsCase;
 130 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.nx.action.reg.load.grouping.NxRegLoad;
 131 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 132 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 133 import org.opendaylight.yangtools.yang.common.RpcResult;
 134 import org.slf4j.Logger;
 135 import org.slf4j.LoggerFactory;
 136
 137 @Singleton
 138 public class NexthopManager implements AutoCloseable {
 139     private static final Logger LOG = LoggerFactory.getLogger(NexthopManager.class);
 140     private static final String NEXTHOP_ID_POOL_NAME = "nextHopPointerPool";
 141     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 142     private static final long WAIT_TIME_TO_ACQUIRE_LOCK = 3000L;
 143
 144     private final DataBroker dataBroker;
 145     private final ManagedNewTransactionRunner txRunner;
 146     private final IMdsalApiManager mdsalApiManager;
 147     private final OdlInterfaceRpcService odlInterfaceRpcService;
 148     private final ItmRpcService itmManager;
 149     private final IdManagerService idManager;
 150     private final IElanService elanService;
 151     private final LockManagerService lockManager;
 152     private final SalGroupService salGroupService;
 153     private final JobCoordinator jobCoordinator;
 154     private final FibUtil fibUtil;
 155     private final IInterfaceManager interfaceManager;
 156     private volatile L3VPNTransportTypes configuredTransportTypeL3VPN = L3VPNTransportTypes.Invalid;
 157
 158     /**
 159      * Provides nexthop functions.
 160      * Creates group ID pool
 161      *
 162      * @param dataBroker       - dataBroker reference
 163      * @param mdsalApiManager  - mdsalApiManager reference
 164      * @param idManager        - idManager reference
 165      * @param odlInterfaceRpcService - odlInterfaceRpcService reference
 166      * @param itmManager       - itmManager reference
 167      */
 168     @Inject
 169     public NexthopManager(final DataBroker dataBroker,
 170                           final IMdsalApiManager mdsalApiManager,
 171                           final IdManagerService idManager,
 172                           final OdlInterfaceRpcService odlInterfaceRpcService,
 173                           final ItmRpcService itmManager,
 174                           final LockManagerService lockManager,
 175                           final IElanService elanService,
 176                           final SalGroupService salGroupService,
 177                           final JobCoordinator jobCoordinator,
 178                           final FibUtil fibUtil,
 179                           final IInterfaceManager interfaceManager) {
 180         this.dataBroker = dataBroker;
 181         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 182         this.mdsalApiManager = mdsalApiManager;
 183         this.idManager = idManager;
 184         this.odlInterfaceRpcService = odlInterfaceRpcService;
 185         this.itmManager = itmManager;
 186         this.elanService = elanService;
 187         this.salGroupService = salGroupService;
 188         this.jobCoordinator = jobCoordinator;
 189         this.fibUtil = fibUtil;
 190         this.lockManager = lockManager;
 191         this.interfaceManager = interfaceManager;
 192         createIdPool();
 193     }
 194
 195     private void createIdPool() {
 196         CreateIdPoolInput createPool = new CreateIdPoolInputBuilder()
 197             .setPoolName(NEXTHOP_ID_POOL_NAME)
 198             .setLow(150000L)
 199             .setHigh(175000L)
 200             .build();
 201         try {
 202             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPool);
 203             if (result != null && result.get().isSuccessful()) {
 204                 LOG.info("Created IdPool for NextHopPointerPool");
 205             }
 206         } catch (InterruptedException | ExecutionException e) {
 207             LOG.error("Failed to create idPool for NextHopPointerPool", e);
 208         }
 209     }
 210
 211     private String getNextHopKey(long vpnId, String ipAddress) {
 212         return "nexthop." + vpnId + ipAddress;
 213     }
 214
 215     public ItmRpcService getItmManager() {
 216         return itmManager;
 217     }
 218
 219     protected long createNextHopPointer(String nexthopKey) {
 220         AllocateIdInput getIdInput = new AllocateIdInputBuilder()
 221             .setPoolName(NEXTHOP_ID_POOL_NAME).setIdKey(nexthopKey)
 222             .build();
 223         //TODO: Proper error handling once IdManager code is complete
 224         try {
 225             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
 226             RpcResult<AllocateIdOutput> rpcResult = result.get();
 227             return rpcResult.getResult().getIdValue();
 228         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 229             LOG.trace("", e);
 230         }
 231         return 0;
 232     }
 233
 234     protected void removeNextHopPointer(String nexthopKey) {
 235         ReleaseIdInput idInput = new ReleaseIdInputBuilder()
 236             .setPoolName(NEXTHOP_ID_POOL_NAME)
 237             .setIdKey(nexthopKey).build();
 238         try {
 239             RpcResult<ReleaseIdOutput> rpcResult = idManager.releaseId(idInput).get();
 240             if (!rpcResult.isSuccessful()) {
 241                 LOG.error("RPC Call to Get Unique Id for nexthopKey {} returned with Errors {}",
 242                         nexthopKey, rpcResult.getErrors());
 243             }
 244         } catch (InterruptedException | ExecutionException e) {
 245             LOG.warn("Exception when getting Unique Id for key {}", nexthopKey, e);
 246         }
 247     }
 248
 249     protected List<ActionInfo> getEgressActionsForInterface(final String ifName, int actionKey,
 250                                                             boolean isTunnelInterface) {
 251         RpcResult rpcResult;
 252         List<Action> actions;
 253         try {
 254             if (isTunnelInterface && interfaceManager.isItmDirectTunnelsEnabled()) {
 255                 rpcResult = itmManager.getEgressActionsForTunnel(
 256                         new GetEgressActionsForTunnelInputBuilder().setIntfName(ifName).build()).get();
 257                 if (!rpcResult.isSuccessful()) {
 258                     LOG.error("RPC Call to Get egress tunnel actions for interface {} returned with Errors {}",
 259                             ifName, rpcResult.getErrors());
 260                     return Collections.emptyList();
 261                 } else {
 262                     GetEgressActionsForInterfaceOutput output =
 263                             (GetEgressActionsForInterfaceOutput)rpcResult.getResult();
 264                     actions = output.getAction();
 265                 }
 266             } else {
 267                 rpcResult = odlInterfaceRpcService.getEgressActionsForInterface(
 268                         new GetEgressActionsForInterfaceInputBuilder().setIntfName(ifName).build()).get();
 269                 if (!rpcResult.isSuccessful()) {
 270                     LOG.error("RPC Call to Get egress vm actions for interface {} returned with Errors {}",
 271                             ifName, rpcResult.getErrors());
 272                     return Collections.emptyList();
 273                 } else {
 274                     GetEgressActionsForInterfaceOutput output =
 275                             (GetEgressActionsForInterfaceOutput)rpcResult.getResult();
 276                     actions = output.getAction();
 277                 }
 278             }
 279             List<ActionInfo> listActionInfo = new ArrayList<>();
 280             for (Action action : actions) {
 281                 actionKey = action.getKey().getOrder() + actionKey;
 282                 org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action
 283                     actionClass = action.getAction();
 284                 if (actionClass instanceof OutputActionCase) {
 285                     listActionInfo.add(new ActionOutput(actionKey,
 286                         ((OutputActionCase) actionClass).getOutputAction().getOutputNodeConnector()));
 287                 } else if (actionClass instanceof PushVlanActionCase) {
 288                     listActionInfo.add(new ActionPushVlan(actionKey));
 289                 } else if (actionClass instanceof SetFieldCase) {
 290                     if (((SetFieldCase) actionClass).getSetField().getVlanMatch() != null) {
 291                         int vlanVid = ((SetFieldCase) actionClass).getSetField().getVlanMatch()
 292                             .getVlanId().getVlanId().getValue();
 293                         listActionInfo.add(new ActionSetFieldVlanVid(actionKey, vlanVid));
 294                     }
 295                 } else if (actionClass instanceof NxActionResubmitRpcAddGroupCase) {
 296                     Short tableId = ((NxActionResubmitRpcAddGroupCase) actionClass).getNxResubmit().getTable();
 297                     listActionInfo.add(new ActionNxResubmit(actionKey, tableId));
 298                 } else if (actionClass instanceof NxActionRegLoadNodesNodeTableFlowApplyActionsCase) {
 299                     NxRegLoad nxRegLoad =
 300                         ((NxActionRegLoadNodesNodeTableFlowApplyActionsCase) actionClass).getNxRegLoad();
 301                     listActionInfo.add(new ActionRegLoad(actionKey, NxmNxReg6.class,
 302                         nxRegLoad.getDst().getStart(), nxRegLoad.getDst().getEnd(),
 303                         nxRegLoad.getValue().longValue()));
 304                 }
 305             }
 306             return listActionInfo;
 307         } catch (InterruptedException | ExecutionException e) {
 308             LOG.warn("Exception when egress actions for interface {}", ifName, e);
 309         }
 310         LOG.warn("Exception when egress actions for interface {}", ifName);
 311         return Collections.emptyList();
 312     }
 313
 314     protected String getTunnelInterfaceName(BigInteger srcDpId, BigInteger dstDpId) {
 315         Class<? extends TunnelTypeBase> tunType = getReqTunType(getReqTransType().toUpperCase(Locale.getDefault()));
 316         Future<RpcResult<GetTunnelInterfaceNameOutput>> result;
 317         try {
 318             result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 319                 .setSourceDpid(srcDpId)
 320                 .setDestinationDpid(dstDpId)
 321                 .setTunnelType(tunType)
 322                 .build());
 323             RpcResult<GetTunnelInterfaceNameOutput> rpcResult = result.get();
 324             if (!rpcResult.isSuccessful()) {
 325                 LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
 326             } else {
 327                 return rpcResult.getResult().getInterfaceName();
 328             }
 329         } catch (InterruptedException | ExecutionException e) {
 330             LOG.warn("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstDpId, e);
 331         }
 332         return null;
 333     }
 334
 335     protected String getTunnelInterfaceName(BigInteger srcDpId, org.opendaylight.yang.gen.v1.urn.ietf.params
 336         .xml.ns.yang.ietf.inet.types.rev130715.IpAddress dstIp) {
 337         Class<? extends TunnelTypeBase> tunType = getReqTunType(getReqTransType().toUpperCase(Locale.getDefault()));
 338         Future<RpcResult<GetInternalOrExternalInterfaceNameOutput>> result;
 339         try {
 340             result = itmManager.getInternalOrExternalInterfaceName(new GetInternalOrExternalInterfaceNameInputBuilder()
 341                 .setSourceDpid(srcDpId)
 342                 .setDestinationIp(dstIp)
 343                 .setTunnelType(tunType)
 344                 .build());
 345             RpcResult<GetInternalOrExternalInterfaceNameOutput> rpcResult = result.get();
 346             if (!rpcResult.isSuccessful()) {
 347                 LOG.warn("RPC Call to getTunnelInterfaceName returned with Errors {}", rpcResult.getErrors());
 348             } else {
 349                 return rpcResult.getResult().getInterfaceName();
 350             }
 351         } catch (InterruptedException | ExecutionException e) {
 352             LOG.warn("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstIp, e);
 353         }
 354         return null;
 355     }
 356
 357
 358     public long getLocalNextHopGroup(long vpnId,
 359             String ipNextHopAddress) {
 360         long groupId = createNextHopPointer(getNextHopKey(vpnId, ipNextHopAddress));
 361         if (groupId == FibConstants.INVALID_GROUP_ID) {
 362             LOG.error("Unable to allocate groupId for vpnId {} , prefix {}", vpnId, ipNextHopAddress);
 363         }
 364         return groupId;
 365     }
 366
 367     public long createLocalNextHop(long vpnId, BigInteger dpnId, String ifName,
 368                                    String primaryIpAddress, String currDestIpPrefix,
 369                                    String gwMacAddress, String jobKey) {
 370         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 371         if (vpnName == null) {
 372             return 0;
 373         }
 374         String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, primaryIpAddress);
 375
 376         long groupId = createNextHopPointer(getNextHopKey(vpnId, primaryIpAddress));
 377         if (groupId == 0) {
 378             LOG.error("Unable to allocate groupId for vpnId {} , IntfName {}, primaryIpAddress {} curIpPrefix {}",
 379                     vpnId, ifName, primaryIpAddress, currDestIpPrefix);
 380             return groupId;
 381         }
 382         String nextHopLockStr = vpnId + primaryIpAddress;
 383         jobCoordinator.enqueueJob(jobKey, () -> {
 384             try {
 385                 if (FibUtil.lockCluster(lockManager, nextHopLockStr, WAIT_TIME_TO_ACQUIRE_LOCK)) {
 386                     VpnNexthop nexthop = getVpnNexthop(vpnId, primaryIpAddress);
 387                     LOG.trace("nexthop: {} retrieved for vpnId {}, prefix {}, ifName {} on dpn {}", nexthop, vpnId,
 388                             primaryIpAddress, ifName, dpnId);
 389                     if (nexthop == null) {
 390                         String encMacAddress = macAddress == null
 391                                 ? fibUtil.getMacAddressFromPrefix(ifName, vpnName, primaryIpAddress) : macAddress;
 392                         List<BucketInfo> listBucketInfo = new ArrayList<>();
 393                         List<ActionInfo> listActionInfo = new ArrayList<>();
 394                         int actionKey = 0;
 395                         // MAC re-write
 396                         if (encMacAddress != null) {
 397                             if (gwMacAddress != null) {
 398                                 LOG.trace("The Local NextHop Group Source Mac {} for VpnInterface {} on VPN {}",
 399                                         gwMacAddress, ifName, vpnId);
 400                                 listActionInfo.add(new ActionSetFieldEthernetSource(actionKey++,
 401                                         new MacAddress(gwMacAddress)));
 402                             }
 403                             listActionInfo.add(new ActionSetFieldEthernetDestination(actionKey++,
 404                                     new MacAddress(encMacAddress)));
 405                             // listActionInfo.add(0, new ActionPopMpls());
 406                         } else {
 407                             // FIXME: Log message here.
 408                             LOG.debug("mac address for new local nexthop is null");
 409                         }
 410                         listActionInfo.addAll(getEgressActionsForInterface(ifName, actionKey, false));
 411                         BucketInfo bucket = new BucketInfo(listActionInfo);
 412
 413                         listBucketInfo.add(bucket);
 414                         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, groupId, primaryIpAddress,
 415                                 GroupTypes.GroupAll, listBucketInfo);
 416                         LOG.trace("Install LNH Group: id {}, mac address {}, interface {} for prefix {}", groupId,
 417                                 encMacAddress, ifName, primaryIpAddress);
 418                         //Try to install group directly on the DPN bypassing the FRM, in order to avoid waiting for the
 419                         // group to get installed before programming the flows
 420                         installGroupOnDpn(groupId, dpnId, primaryIpAddress, listBucketInfo,
 421                                 getNextHopKey(vpnId, primaryIpAddress), GroupTypes.GroupAll);
 422                         // install Group
 423                         mdsalApiManager.syncInstallGroup(groupEntity);
 424                         // update MD-SAL DS
 425                         addVpnNexthopToDS(dpnId, vpnId, primaryIpAddress, currDestIpPrefix, groupId);
 426
 427                     } else {
 428                         // Ignore adding new prefix , if it already exists
 429                         List<IpAdjacencies> prefixesList = nexthop.getIpAdjacencies();
 430                         IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currDestIpPrefix).build();
 431                         if (prefixesList != null && prefixesList.contains(prefix)) {
 432                             LOG.trace("Prefix {} is already present in l3nextHop {} ", currDestIpPrefix, nexthop);
 433                         } else {
 434                             IpAdjacenciesBuilder ipPrefixesBuilder =
 435                                     new IpAdjacenciesBuilder().setKey(new IpAdjacenciesKey(currDestIpPrefix));
 436                             LOG.trace("Updating prefix {} to vpnNextHop {} Operational DS", currDestIpPrefix, nexthop);
 437                             MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL,
 438                                     getVpnNextHopIpPrefixIdentifier(vpnId, primaryIpAddress, currDestIpPrefix),
 439                                     ipPrefixesBuilder.build());
 440                         }
 441                     }
 442                 }
 443             } finally {
 444                 FibUtil.unlockCluster(lockManager, nextHopLockStr);
 445             }
 446             return Collections.emptyList();
 447         });
 448         return groupId;
 449     }
 450
 451     private void installGroupOnDpn(long groupId, BigInteger dpnId, String groupName, List<BucketInfo> bucketsInfo,
 452                                      String nextHopKey, GroupTypes groupType) {
 453         NodeRef nodeRef = FibUtil.buildNodeRef(dpnId);
 454         Buckets buckets = FibUtil.buildBuckets(bucketsInfo);
 455         GroupRef groupRef = new GroupRef(FibUtil.buildGroupInstanceIdentifier(groupId, dpnId));
 456         AddGroupInput input = new AddGroupInputBuilder().setNode(nodeRef).setGroupId(new GroupId(groupId))
 457                 .setBuckets(buckets).setGroupRef(groupRef).setGroupType(groupType)
 458                 .setGroupName(groupName).build();
 459         Future<RpcResult<AddGroupOutput>> groupStats = salGroupService.addGroup(input);
 460         RpcResult<AddGroupOutput> rpcResult = null;
 461         try {
 462             rpcResult = groupStats.get();
 463             if (rpcResult != null && rpcResult.isSuccessful()) {
 464                 LOG.info("Group {} with key {} has been successfully installed directly on dpn {}.", groupId,
 465                         nextHopKey, dpnId);
 466             } else {
 467                 LOG.error("Unable to install group {} with key {} directly on dpn {} due to {}.", groupId, nextHopKey,
 468                         dpnId, rpcResult != null ? rpcResult.getErrors() : null);
 469             }
 470         } catch (InterruptedException | ExecutionException e) {
 471             LOG.error("Error while installing group {} directly on dpn {}", groupId, dpnId);
 472         }
 473     }
 474
 475     protected void addVpnNexthopToDS(BigInteger dpnId, long vpnId, String primaryIpAddr,
 476                                      String currIpAddr, long egressPointer) {
 477         InstanceIdentifierBuilder<VpnNexthops> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
 478             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId));
 479
 480         List<IpAdjacencies> ipPrefixesList = new ArrayList<>();
 481         IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currIpAddr).build();
 482         ipPrefixesList.add(prefix);
 483         // Add nexthop to vpn node
 484         VpnNexthop nh = new VpnNexthopBuilder()
 485             .setKey(new VpnNexthopKey(primaryIpAddr))
 486             .setDpnId(dpnId)
 487             .setIpAdjacencies(ipPrefixesList)
 488             .setEgressPointer(egressPointer).build();
 489
 490         InstanceIdentifier<VpnNexthop> id1 = idBuilder
 491             .child(VpnNexthop.class, new VpnNexthopKey(primaryIpAddr)).build();
 492         LOG.trace("Adding vpnnextHop {} to Operational DS", nh);
 493         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id1, nh);
 494
 495     }
 496
 497     protected InstanceIdentifier<IpAdjacencies> getVpnNextHopIpPrefixIdentifier(long vpnId, String primaryIpAddress,
 498                                                                                 String ipPrefix) {
 499         InstanceIdentifier<IpAdjacencies> id = InstanceIdentifier.builder(L3nexthop.class)
 500                 .child(VpnNexthops.class, new VpnNexthopsKey(vpnId))
 501                 .child(VpnNexthop.class, new VpnNexthopKey(primaryIpAddress))
 502                 .child(IpAdjacencies.class, new IpAdjacenciesKey(ipPrefix)).build();
 503         return id;
 504     }
 505
 506     protected VpnNexthop getVpnNexthop(long vpnId, String ipAddress) {
 507
 508         // check if vpn node is there
 509         InstanceIdentifierBuilder<VpnNexthops> idBuilder =
 510             InstanceIdentifier.builder(L3nexthop.class).child(VpnNexthops.class,
 511                 new VpnNexthopsKey(vpnId));
 512         InstanceIdentifier<VpnNexthops> id = idBuilder.build();
 513         Optional<VpnNexthops> vpnNexthops = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 514         if (vpnNexthops.isPresent()) {
 515             // get nexthops list for vpn
 516             List<VpnNexthop> nexthops = vpnNexthops.get().getVpnNexthop();
 517             for (VpnNexthop nexthop : nexthops) {
 518                 if (nexthop.getIpAddress().equals(ipAddress)) {
 519                     // return nexthop
 520                     LOG.trace("VpnNextHop : {}", nexthop);
 521                     return nexthop;
 522                 }
 523             }
 524             // return null if not found
 525         }
 526         return null;
 527     }
 528
 529     public AdjacencyResult getRemoteNextHopPointer(BigInteger remoteDpnId, long vpnId, String prefixIp,
 530             String nextHopIp) {
 531         String egressIfName = null;
 532         LOG.trace("getRemoteNextHopPointer: input [remoteDpnId {}, vpnId {}, prefixIp {}, nextHopIp {} ]", remoteDpnId,
 533             vpnId, prefixIp, nextHopIp);
 534
 535         Class<? extends InterfaceType> egressIfType;
 536         ElanInstance elanInstance = getElanInstanceForPrefix(vpnId, prefixIp);
 537         if (elanInstance != null) {
 538             egressIfType = getInterfaceType(elanInstance);
 539         } else {
 540             LOG.warn("Failed to determine network type for prefixIp {} using tunnel", prefixIp);
 541             egressIfType = Tunnel.class;
 542         }
 543
 544         if (Tunnel.class.equals(egressIfType)) {
 545             egressIfName = getTunnelRemoteNextHopPointer(remoteDpnId, nextHopIp);
 546         } else {
 547             egressIfName = getExtPortRemoteNextHopPointer(remoteDpnId, elanInstance);
 548         }
 549
 550         LOG.trace("NextHop pointer for prefixIp {} vpnId {} dpnId {} is {}", prefixIp, vpnId, remoteDpnId,
 551             egressIfName);
 552         return egressIfName != null ? new AdjacencyResult(egressIfName, egressIfType, nextHopIp,
 553                 prefixIp) : null;
 554     }
 555
 556     public BigInteger getDpnForPrefix(long vpnId, String prefixIp) {
 557         VpnNexthop vpnNexthop = getVpnNexthop(vpnId, prefixIp);
 558         BigInteger localDpnId = vpnNexthop == null ? null : vpnNexthop.getDpnId();
 559         return localDpnId;
 560     }
 561
 562     private void removeVpnNexthopFromDS(long vpnId, String ipPrefix) {
 563
 564         InstanceIdentifierBuilder<VpnNexthop> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
 565             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId))
 566             .child(VpnNexthop.class, new VpnNexthopKey(ipPrefix));
 567         InstanceIdentifier<VpnNexthop> id = idBuilder.build();
 568         // remove from DS
 569         LOG.trace("Removing vpn next hop from datastore : {}", id);
 570         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 571     }
 572
 573     public void removeLocalNextHop(BigInteger dpnId, Long vpnId, String primaryIpAddress, String currDestIpPrefix) {
 574         String nextHopLockStr = vpnId + primaryIpAddress;
 575         try {
 576             if (FibUtil.lockCluster(lockManager, nextHopLockStr, WAIT_TIME_TO_ACQUIRE_LOCK)) {
 577                 VpnNexthop nh = getVpnNexthop(vpnId, primaryIpAddress);
 578                 if (nh != null) {
 579                     List<IpAdjacencies> prefixesList = nh.getIpAdjacencies();
 580                     IpAdjacencies prefix = new IpAdjacenciesBuilder().setIpAdjacency(currDestIpPrefix).build();
 581                     prefixesList.remove(prefix);
 582                     if (prefixesList.isEmpty()) { //remove the group only if there are no more flows using this group
 583                         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, nh.getEgressPointer(),
 584                                 primaryIpAddress, GroupTypes.GroupAll, Collections.EMPTY_LIST);
 585                         // remove Group ...
 586                         mdsalApiManager.removeGroup(groupEntity);
 587                         //update MD-SAL DS
 588                         removeVpnNexthopFromDS(vpnId, primaryIpAddress);
 589                         //release groupId
 590                         removeNextHopPointer(getNextHopKey(vpnId, primaryIpAddress));
 591                         LOG.debug("Local Next hop {} for {} {} on dpn {} successfully deleted",
 592                                 nh.getEgressPointer(), vpnId, primaryIpAddress, dpnId);
 593                     } else {
 594                         //remove the currIpPrefx from IpPrefixList of the vpnNexthop
 595                         LOG.trace("Removing the prefix {} from vpnNextHop {} Operational DS", currDestIpPrefix, nh);
 596                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
 597                                 getVpnNextHopIpPrefixIdentifier(vpnId, primaryIpAddress, currDestIpPrefix));
 598                     }
 599                 } else {
 600                     //throw error
 601                     LOG.error("Local NextHop for VpnId {} curIpPrefix {} on dpn {} primaryIpAddress {} not deleted",
 602                             vpnId, currDestIpPrefix, dpnId, primaryIpAddress);
 603                 }
 604             }
 605         } finally {
 606             FibUtil.unlockCluster(lockManager, nextHopLockStr);
 607         }
 608     }
 609
 610     public void setConfTransType(String service, String transportType) {
 611
 612         if (!service.equalsIgnoreCase("L3VPN")) {
 613             LOG.error("Incorrect service {} provided for setting the transport type.", service);
 614             return;
 615         }
 616
 617         L3VPNTransportTypes transType = L3VPNTransportTypes.validateTransportType(transportType
 618                 .toUpperCase(Locale.getDefault()));
 619
 620         if (transType != L3VPNTransportTypes.Invalid) {
 621             configuredTransportTypeL3VPN = transType;
 622         }
 623     }
 624
 625     public void writeConfTransTypeConfigDS() {
 626         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, getConfTransportTypeIdentifier(),
 627             createConfTransportType(configuredTransportTypeL3VPN.getTransportType()));
 628     }
 629
 630     public L3VPNTransportTypes getConfiguredTransportTypeL3VPN() {
 631         return this.configuredTransportTypeL3VPN;
 632     }
 633
 634     public String getReqTransType() {
 635         if (configuredTransportTypeL3VPN == L3VPNTransportTypes.Invalid) {
 636             /*
 637              * Restart scenario, Read from the ConfigDS.
 638              * if the value is Unset, cache value as VxLAN.
 639              */
 640             LOG.trace("configureTransportType is not yet set.");
 641             Optional<ConfTransportTypeL3vpn> configuredTransTypeFromConfig =
 642                 MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, getConfTransportTypeIdentifier());
 643
 644             if (configuredTransTypeFromConfig.isPresent()) {
 645                 if (configuredTransTypeFromConfig.get().getTransportType().equals(TunnelTypeGre.class)) {
 646                     configuredTransportTypeL3VPN = L3VPNTransportTypes.GRE;
 647                 } else {
 648                     configuredTransportTypeL3VPN = L3VPNTransportTypes.VxLAN;
 649                 }
 650                 LOG.trace("configuredTransportType set from config DS to {}",
 651                     getConfiguredTransportTypeL3VPN().getTransportType());
 652             } else {
 653                 setConfTransType("L3VPN", L3VPNTransportTypes.VxLAN.getTransportType());
 654                 LOG.trace("configuredTransportType is not set in the Config DS. VxLAN as default will be used.");
 655             }
 656         } else {
 657             LOG.trace("configuredTransportType is set as {}", getConfiguredTransportTypeL3VPN().getTransportType());
 658         }
 659         return getConfiguredTransportTypeL3VPN().getTransportType();
 660     }
 661
 662     public InstanceIdentifier<ConfTransportTypeL3vpn> getConfTransportTypeIdentifier() {
 663         return InstanceIdentifier.builder(ConfTransportTypeL3vpn.class).build();
 664     }
 665
 666     private ConfTransportTypeL3vpn createConfTransportType(String type) {
 667         ConfTransportTypeL3vpn confTransType;
 668         switch (type) {
 669             case ITMConstants.TUNNEL_TYPE_GRE:
 670                 confTransType = new ConfTransportTypeL3vpnBuilder().setTransportType(TunnelTypeGre.class).build();
 671                 LOG.trace("Setting the confTransportType to GRE.");
 672                 break;
 673             case ITMConstants.TUNNEL_TYPE_VXLAN:
 674                 confTransType = new ConfTransportTypeL3vpnBuilder().setTransportType(TunnelTypeVxlan.class).build();
 675                 LOG.trace("Setting the confTransportType to VxLAN.");
 676                 break;
 677             default:
 678                 LOG.trace("Invalid transport type {} passed to Config DS ", type);
 679                 confTransType = null;
 680                 break;
 681         }
 682         return confTransType;
 683     }
 684
 685     public Class<? extends TunnelTypeBase> getReqTunType(String transportType) {
 686         switch (transportType) {
 687             case "VXLAN":
 688                 return TunnelTypeVxlan.class;
 689             case "GRE":
 690                 return TunnelTypeGre.class;
 691             default:
 692                 return TunnelTypeMplsOverGre.class;
 693         }
 694     }
 695
 696     public String getTransportTypeStr(String tunType) {
 697         if (tunType.equals(TunnelTypeVxlan.class.toString())) {
 698             return ITMConstants.TUNNEL_TYPE_VXLAN;
 699         } else if (tunType.equals(TunnelTypeGre.class.toString())) {
 700             return ITMConstants.TUNNEL_TYPE_GRE;
 701         } else if (tunType.equals(TunnelTypeMplsOverGre.class.toString())) {
 702             return ITMConstants.TUNNEL_TYPE_MPLSoGRE;
 703         } else {
 704             return ITMConstants.TUNNEL_TYPE_INVALID;
 705         }
 706     }
 707
 708     @Override
 709     @PreDestroy
 710     public void close() {
 711         LOG.info("{} close", getClass().getSimpleName());
 712     }
 713
 714     // TODO Clean up the exception handling
 715     @SuppressWarnings("checkstyle:IllegalCatch")
 716     private String getTunnelRemoteNextHopPointer(BigInteger remoteDpnId, String nextHopIp) {
 717         if (nextHopIp != null && !nextHopIp.isEmpty()) {
 718             try {
 719                 // here use the config for tunnel type param
 720                 return getTunnelInterfaceName(remoteDpnId,
 721                     org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder
 722                         .getDefaultInstance(nextHopIp));
 723             } catch (Exception ex) {
 724                 LOG.error("Error while retrieving nexthop pointer for nexthop {} remoteDpn {}",
 725                         nextHopIp, remoteDpnId, ex);
 726             }
 727         }
 728
 729         return null;
 730     }
 731
 732     private String getExtPortRemoteNextHopPointer(BigInteger remoteDpnId, ElanInstance elanInstance) {
 733         return elanService.getExternalElanInterface(elanInstance.getElanInstanceName(), remoteDpnId);
 734     }
 735
 736     /**
 737      * Get the interface type associated with the type of ELAN used for routing
 738      * traffic to/from remote compute nodes.
 739      *
 740      * @param elanInstance The elan instance
 741      * @return L2vlan for flat/VLAN network type and Tunnel otherwise
 742      */
 743     private Class<? extends InterfaceType> getInterfaceType(ElanInstance elanInstance) {
 744         Class<? extends SegmentTypeBase> segmentType = elanInstance.getSegmentType();
 745         if (SegmentTypeFlat.class.equals(segmentType) || SegmentTypeVlan.class.equals(segmentType)) {
 746             return L2vlan.class;
 747         }
 748
 749         return Tunnel.class;
 750     }
 751
 752     private ElanInstance getElanInstanceForPrefix(long vpnId, String prefixIp) {
 753         ElanInstance elanInstance = null;
 754         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, prefixIp);
 755         if (prefix != null) {
 756             Uuid subnetId = prefix.getSubnetId();
 757             if (subnetId != null) {
 758                 Subnetmap subnetMap = fibUtil.getSubnetMap(subnetId);
 759                 if (subnetMap != null && subnetMap.getNetworkId() != null) {
 760                     elanInstance = elanService.getElanInstance(subnetMap.getNetworkId().getValue());
 761                 }
 762             }
 763         }
 764
 765         return elanInstance;
 766     }
 767
 768     static class AdjacencyResult {
 769         private final String interfaceName;
 770         private final Class<? extends InterfaceType> interfaceType;
 771         private final String nextHopIp;
 772         private final String prefix;
 773
 774         AdjacencyResult(String interfaceName, Class<? extends InterfaceType> interfaceType, String nextHopIp,
 775                         String prefix) {
 776             this.interfaceName = interfaceName;
 777             this.interfaceType = interfaceType;
 778             this.nextHopIp = nextHopIp;
 779             this.prefix = prefix;
 780         }
 781
 782         public String getInterfaceName() {
 783             return interfaceName;
 784         }
 785
 786         public Class<? extends InterfaceType> getInterfaceType() {
 787             return interfaceType;
 788         }
 789
 790         public String getNextHopIp() {
 791             return nextHopIp;
 792         }
 793
 794         public String getPrefix() {
 795             return prefix;
 796         }
 797
 798         @Override
 799         public int hashCode() {
 800             final int prime = 31;
 801             int result = 1;
 802             result = prime * result + (interfaceName == null ? 0 : interfaceName.hashCode());
 803             return result;
 804         }
 805
 806         @Override
 807         public boolean equals(Object obj) {
 808             if (obj == null) {
 809                 return false;
 810             }
 811
 812             boolean result = false;
 813             if (getClass() != obj.getClass()) {
 814                 return result;
 815             } else {
 816                 AdjacencyResult other = (AdjacencyResult) obj;
 817                 result = interfaceName.equals(other.interfaceName);
 818             }
 819             return result;
 820         }
 821     }
 822
 823     protected long setupLoadBalancingNextHop(Long parentVpnId, BigInteger dpnId,
 824             String destPrefix, List<BucketInfo> listBucketInfo, boolean addOrRemove) {
 825         long groupId = createNextHopPointer(getNextHopKey(parentVpnId, destPrefix));
 826         if (groupId == FibConstants.INVALID_GROUP_ID) {
 827             LOG.error("Unable to allocate/retrieve groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 828             return groupId;
 829         }
 830         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(
 831                 dpnId, groupId, destPrefix, GroupTypes.GroupSelect, listBucketInfo);
 832         if (addOrRemove) {
 833             mdsalApiManager.syncInstallGroup(groupEntity);
 834             try {
 835                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 836             } catch (InterruptedException e1) {
 837                 LOG.warn("Thread got interrupted while programming LB group {}", groupEntity);
 838                 Thread.currentThread().interrupt();
 839             }
 840         } else {
 841             mdsalApiManager.removeGroup(groupEntity);
 842         }
 843         return groupId;
 844     }
 845
 846     long createNextHopGroups(Long vpnId, String rd, BigInteger dpnId, VrfEntry vrfEntry,
 847             Routes routes, List<Routes> vpnExtraRoutes) {
 848         List<BucketInfo> listBucketInfo = new ArrayList<>();
 849         List<Routes> clonedVpnExtraRoutes  = new ArrayList<>(vpnExtraRoutes);
 850         if (clonedVpnExtraRoutes.contains(routes)) {
 851             listBucketInfo.addAll(getBucketsForLocalNexthop(vpnId, dpnId, vrfEntry, routes));
 852             clonedVpnExtraRoutes.remove(routes);
 853         }
 854         listBucketInfo.addAll(getBucketsForRemoteNexthop(vpnId, dpnId, vrfEntry, rd, clonedVpnExtraRoutes));
 855         return setupLoadBalancingNextHop(vpnId, dpnId, vrfEntry.getDestPrefix(), listBucketInfo, true);
 856     }
 857
 858     private List<BucketInfo> getBucketsForLocalNexthop(Long vpnId, BigInteger dpnId,
 859             VrfEntry vrfEntry, Routes routes) {
 860         List<BucketInfo> listBucketInfo = new CopyOnWriteArrayList<>();
 861         routes.getNexthopIpList().parallelStream().forEach(nextHopIp -> {
 862             String localNextHopIP;
 863             if (isIpv4Address(nextHopIp)) {
 864                 localNextHopIP = nextHopIp + NwConstants.IPV4PREFIX;
 865             } else {
 866                 localNextHopIP = nextHopIp + NwConstants.IPV6PREFIX;
 867             }
 868             Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 869             if (localNextHopInfo != null) {
 870                 long groupId = getLocalNextHopGroup(vpnId, localNextHopIP);
 871                 if (groupId == FibConstants.INVALID_GROUP_ID) {
 872                     LOG.error("Unable to allocate groupId for vpnId {} , prefix {} , interface {}", vpnId,
 873                             vrfEntry.getDestPrefix(), localNextHopInfo.getVpnInterfaceName());
 874                     return;
 875                 }
 876                 List<ActionInfo> actionsInfos =
 877                         Collections.singletonList(new ActionGroup(groupId));
 878                 BucketInfo bucket = new BucketInfo(actionsInfos);
 879                 bucket.setWeight(1);
 880                 listBucketInfo.add(bucket);
 881             }
 882         });
 883         LOG.trace("LOCAL: listbucket {}, vpnId {}, dpnId {}, routes {}", listBucketInfo, vpnId, dpnId, routes);
 884         return listBucketInfo;
 885     }
 886
 887     private List<BucketInfo> getBucketsForRemoteNexthop(Long vpnId, BigInteger dpnId, VrfEntry vrfEntry, String rd,
 888             List<Routes> vpnExtraRoutes) {
 889         List<BucketInfo> listBucketInfo = new ArrayList<>();
 890         Map<String, List<ActionInfo>> egressActionMap = new HashMap<>();
 891         vpnExtraRoutes.forEach(vpnExtraRoute -> vpnExtraRoute.getNexthopIpList().forEach(nextHopIp -> {
 892             String nextHopPrefixIp;
 893             if (isIpv4Address(nextHopIp)) {
 894                 nextHopPrefixIp = nextHopIp + NwConstants.IPV4PREFIX;
 895             } else {
 896                 nextHopPrefixIp = nextHopIp + NwConstants.IPV6PREFIX;
 897             }
 898             List<String> tepIpAddresses = fibUtil.getNextHopAddresses(rd, nextHopPrefixIp);
 899             if (tepIpAddresses.isEmpty()) {
 900                 return;
 901             }
 902             // There would be only one nexthop address for a VM ip which would be the tep Ip
 903             String tepIp = tepIpAddresses.get(0);
 904             AdjacencyResult adjacencyResult = getRemoteNextHopPointer(dpnId, vpnId,
 905                     vrfEntry.getDestPrefix(), tepIp);
 906             if (adjacencyResult == null) {
 907                 return;
 908             }
 909             String egressInterface = adjacencyResult.getInterfaceName();
 910             if (!FibUtil.isTunnelInterface(adjacencyResult)) {
 911                 return;
 912             }
 913             Class<? extends TunnelTypeBase> tunnelType = VpnExtraRouteHelper.getTunnelType(itmManager, egressInterface);
 914             Interface ifState = fibUtil.getInterfaceStateFromOperDS(egressInterface);
 915             if (ifState == null || ifState.getOperStatus() != OperStatus.Up) {
 916                 LOG.trace("Tunnel not up {}", egressInterface);
 917                 return;
 918             }
 919             if (!TunnelTypeVxlan.class.equals(tunnelType)) {
 920                 return;
 921             }
 922             Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).get();
 923             Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, nextHopPrefixIp);
 924             if (prefixInfo == null) {
 925                 LOG.error("No prefix info found for prefix {} in rd {} for VPN {}", nextHopPrefixIp, rd,
 926                         vpnId);
 927                 return;
 928             }
 929             BigInteger tunnelId;
 930             if (fibUtil.enforceVxlanDatapathSemanticsforInternalRouterVpn(prefixInfo.getSubnetId(), vpnId,
 931                     rd)) {
 932                 java.util.Optional<Long> optionalVni = fibUtil.getVniForVxlanNetwork(prefixInfo.getSubnetId());
 933                 if (!optionalVni.isPresent()) {
 934                     LOG.error("VNI not found for nexthop {} vrfEntry {} with subnetId {}", nextHopIp,
 935                             vrfEntry, prefixInfo.getSubnetId());
 936                     return;
 937                 }
 938                 tunnelId = BigInteger.valueOf(optionalVni.get());
 939             } else {
 940                 tunnelId = BigInteger.valueOf(label);
 941             }
 942             List<ActionInfo> actionInfos = new ArrayList<>();
 943             actionInfos.add(new ActionSetFieldTunnelId(tunnelId));
 944             String ifName = prefixInfo.getVpnInterfaceName();
 945             String vpnName = fibUtil.getVpnNameFromId(vpnId);
 946             if (vpnName == null) {
 947                 return;
 948             }
 949             String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, nextHopPrefixIp);
 950             actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
 951                     new MacAddress(macAddress)));
 952             List<ActionInfo> egressActions;
 953             if (egressActionMap.containsKey(egressInterface)) {
 954                 egressActions = egressActionMap.get(egressInterface);
 955             } else {
 956                 egressActions = getEgressActionsForInterface(egressInterface, actionInfos.size(), true);
 957                 egressActionMap.put(egressInterface, egressActions);
 958             }
 959             if (egressActions.isEmpty()) {
 960                 LOG.error("Failed to retrieve egress action for prefix {} route-paths {}"
 961                         + " interface {}." + " Aborting remote FIB entry creation.",
 962                         vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), egressInterface);
 963             }
 964             actionInfos.addAll(egressActions);
 965             BucketInfo bucket = new BucketInfo(actionInfos);
 966             bucket.setWeight(1);
 967             listBucketInfo.add(bucket);
 968         }));
 969         LOG.trace("LOCAL: listbucket {}, rd {}, dpnId {}, routes {}", listBucketInfo, rd, dpnId, vpnExtraRoutes);
 970         return listBucketInfo;
 971     }
 972
 973     public void createDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId, String destinationIp) {
 974         Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
 975         int noOfDcGws = availableDcGws.size();
 976         if (noOfDcGws == 1) {
 977             LOG.trace("There are no enough DC GateWays {} present to program LB group", availableDcGws);
 978             return;
 979         }
 980         // TODO : Place the logic to construct all possible DC-GW combination here.
 981         String groupIdKey = FibUtil.getGreLbGroupKey(availableDcGws);
 982         Long groupId = createNextHopPointer(groupIdKey);
 983         List<Bucket> listBucket = new ArrayList<>();
 984         for (int index = 0; index < noOfDcGws; index++) {
 985             if (isTunnelUp(availableDcGws.get(index), dpnId)) {
 986                 listBucket.add(buildBucketForDcGwLbGroup(availableDcGws.get(index), dpnId, index));
 987             }
 988         }
 989         Group group = MDSALUtil.buildGroup(groupId, groupIdKey, GroupTypes.GroupSelect,
 990                         MDSALUtil.buildBucketLists(listBucket));
 991         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
 992             mdsalApiManager.addGroupToTx(dpnId, group, confTx);
 993         }), LOG, "Error adding load-balancing group");
 994         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(operTx -> {
 995             FibUtil.updateLbGroupInfo(dpnId, destinationIp, groupIdKey, groupId.toString(), operTx);
 996         }), LOG, "Error updating load-balancing group info");
 997         LOG.trace("LB group {} towards DC-GW installed on dpn {}. Group - {}", groupIdKey, dpnId, group);
 998     }
 999
1000     private boolean isTunnelUp(String dcGwIp, BigInteger dpnId) {
1001         String tunnelName = getTunnelRemoteNextHopPointer(dpnId, dcGwIp);
1002         if (tunnelName != null) {
1003             InstanceIdentifier<StateTunnelList> tunnelStateId =
1004                     InstanceIdentifier.builder(TunnelsState.class).child(
1005                             StateTunnelList.class, new StateTunnelListKey(tunnelName)).build();
1006             return MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, tunnelStateId)
1007                     .toJavaUtil().map(StateTunnelList::getOperState)
1008                     .orElse(TunnelOperStatus.Down) == TunnelOperStatus.Up;
1009         }
1010         return false;
1011     }
1012
1013     private List<Action> getEgressActions(String interfaceName, int actionKey) {
1014         List<Action> actions = Collections.emptyList();
1015         try {
1016             GetEgressActionsForInterfaceInputBuilder egressAction =
1017                     new GetEgressActionsForInterfaceInputBuilder().setIntfName(interfaceName).setActionKey(actionKey);
1018             Future<RpcResult<GetEgressActionsForInterfaceOutput>> result =
1019                     odlInterfaceRpcService.getEgressActionsForInterface(egressAction.build());
1020             RpcResult<GetEgressActionsForInterfaceOutput> rpcResult = result.get();
1021             if (!rpcResult.isSuccessful()) {
1022                 LOG.warn("RPC Call to Get egress actions for interface {} returned with Errors {}",
1023                         interfaceName, rpcResult.getErrors());
1024             } else {
1025                 actions = rpcResult.getResult().getAction();
1026             }
1027         } catch (InterruptedException | ExecutionException e) {
1028             LOG.warn("Exception when egress actions for interface {}", interfaceName, e);
1029         }
1030         return actions;
1031     }
1032
1033     /**
1034      * This method is invoked when the tunnel state is removed from DS.
1035      * If the there is just one DC-GW left in configuration then the LB groups can be deleted.
1036      * Otherwise, the groups are just updated.
1037      */
1038     public void removeOrUpdateDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId,
1039             String destinationIp) {
1040         Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
1041         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1042             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(operTx -> {
1043                 int noOfDcGws = availableDcGws.size();
1044                 // If availableDcGws does not contain the destination Ip it means this is a configuration delete.
1045                 if (!availableDcGws.contains(destinationIp)) {
1046                     availableDcGws.add(destinationIp);
1047                     Collections.sort(availableDcGws);
1048                 }
1049                 // TODO : Place the logic to construct all possible DC-GW combination here.
1050                 int bucketId = availableDcGws.indexOf(destinationIp);
1051                 Optional<DpnLbNexthops> dpnLbNextHops = fibUtil.getDpnLbNexthops(dpnId, destinationIp);
1052                 if (!dpnLbNextHops.isPresent()) {
1053                     return;
1054                 }
1055                 List<String> nextHopKeys = dpnLbNextHops.get().getNexthopKey();
1056                 nextHopKeys.forEach(nextHopKey -> {
1057                     Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
1058                     if (!optionalNextHops.isPresent()) {
1059                         return;
1060                     }
1061                     Nexthops nexthops = optionalNextHops.get();
1062                     final String groupId = nexthops.getGroupId();
1063                     final long groupIdValue = Long.parseLong(groupId);
1064                     if (noOfDcGws > 1) {
1065                         mdsalApiManager.removeBucketToTx(dpnId, groupIdValue, bucketId, confTx);
1066                     } else {
1067                         Group group = MDSALUtil.buildGroup(groupIdValue, nextHopKey, GroupTypes.GroupSelect,
1068                                 MDSALUtil.buildBucketLists(Collections.emptyList()));
1069                         LOG.trace("Removed LB group {} on dpn {}", group, dpnId);
1070                         mdsalApiManager.removeGroupToTx(dpnId, group, confTx);
1071                         removeNextHopPointer(nextHopKey);
1072                     }
1073                     // When the DC-GW is removed from configuration.
1074                     if (noOfDcGws != availableDcGws.size()) {
1075                         FibUtil.removeOrUpdateNextHopInfo(dpnId, nextHopKey, groupId, nexthops, operTx);
1076                     }
1077                 });
1078                 FibUtil.removeDpnIdToNextHopInfo(destinationIp, dpnId, operTx);
1079             }), LOG, "Error removing or updating load-balancing group");
1080         }), LOG, "Error removing or updating load-balancing group");
1081     }
1082
1083     /**
1084      * This method is invoked when the tunnel status is updated.
1085      * The bucket is directly removed/added based on the operational status of the tunnel.
1086      */
1087     public void updateDcGwLoadBalancingGroup(List<String> availableDcGws,
1088             BigInteger dpnId, String destinationIp, boolean isTunnelUp) {
1089         Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
1090         ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1091             // TODO : Place the logic to construct all possible DC-GW combination here.
1092             int bucketId = availableDcGws.indexOf(destinationIp);
1093             Optional<DpnLbNexthops> dpnLbNextHops = fibUtil.getDpnLbNexthops(dpnId, destinationIp);
1094             if (!dpnLbNextHops.isPresent()) {
1095                 return;
1096             }
1097             List<String> nextHopKeys = dpnLbNextHops.get().getNexthopKey();
1098             nextHopKeys.forEach(nextHopKey -> {
1099                 Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
1100                 if (!optionalNextHops.isPresent()) {
1101                     return;
1102                 }
1103                 Nexthops nexthops = optionalNextHops.get();
1104                 final String groupId = nexthops.getGroupId();
1105                 final long groupIdValue = Long.parseLong(groupId);
1106                 if (isTunnelUp) {
1107                     Bucket bucket = buildBucketForDcGwLbGroup(destinationIp, dpnId, bucketId);
1108                     LOG.trace("Added bucket {} to group {} on dpn {}.", bucket, groupId, dpnId);
1109                     mdsalApiManager.addBucketToTx(dpnId, groupIdValue, bucket , confTx);
1110                 } else {
1111                     LOG.trace("Removed bucketId {} from group {} on dpn {}.", bucketId, groupId, dpnId);
1112                     mdsalApiManager.removeBucketToTx(dpnId, groupIdValue, bucketId, confTx);
1113                 }
1114             });
1115         }), LOG, "Error updating load-balancing group");
1116     }
1117
1118     private Bucket buildBucketForDcGwLbGroup(String ipAddress, BigInteger dpnId,
1119             int index) {
1120         List<Action> listAction = new ArrayList<>();
1121         // ActionKey 0 goes to mpls label.
1122         int actionKey = 1;
1123         listAction.add(new ActionPushMpls().buildAction());
1124         listAction.add(new ActionRegMove(actionKey++, FibConstants.NXM_REG_MAPPING
1125                 .get(index), 0, 19).buildAction());
1126         String tunnelInterfaceName = getTunnelInterfaceName(dpnId, new IpAddress(ipAddress.toCharArray()));
1127         List<Action> egressActions = getEgressActions(tunnelInterfaceName, actionKey++);
1128         if (!egressActions.isEmpty()) {
1129             listAction.addAll(getEgressActions(tunnelInterfaceName, actionKey++));
1130         } else {
1131             // clear off actions if there is no egress actions.
1132             listAction = Collections.emptyList();
1133         }
1134         return MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, index,
1135                 MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP);
1136     }
1137
1138     public void programDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId,
1139             String destinationIp, int addRemoveOrUpdate, boolean isTunnelUp) {
1140         if (NwConstants.ADD_FLOW == addRemoveOrUpdate) {
1141             createDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp);
1142         } else if (NwConstants.DEL_FLOW == addRemoveOrUpdate) {
1143             removeOrUpdateDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp);
1144         } else if (NwConstants.MOD_FLOW == addRemoveOrUpdate) {
1145             updateDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp, isTunnelUp);
1146         }
1147     }
1148 }