fibmanager/impl/src/main/java/org/opendaylight/netvirt/fibmanager/NexthopManager.java

   1 /*
   2  * Copyright © 2015, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.fibmanager;
   9
  10 import static org.opendaylight.genius.mdsalutil.NWUtil.isIpv4Address;
  11
  12 import com.google.common.base.Optional;
  13 import com.google.common.base.Preconditions;
  14 import java.math.BigInteger;
  15 import java.util.ArrayList;
  16 import java.util.Collections;
  17 import java.util.HashMap;
  18 import java.util.List;
  19 import java.util.Locale;
  20 import java.util.Map;
  21 import java.util.concurrent.CopyOnWriteArrayList;
  22 import java.util.concurrent.ExecutionException;
  23 import java.util.concurrent.Future;
  24 import javax.annotation.PreDestroy;
  25 import javax.inject.Inject;
  26 import javax.inject.Singleton;
  27 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  28 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  29 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  30 import org.opendaylight.genius.itm.globals.ITMConstants;
  31 import org.opendaylight.genius.mdsalutil.ActionInfo;
  32 import org.opendaylight.genius.mdsalutil.BucketInfo;
  33 import org.opendaylight.genius.mdsalutil.GroupEntity;
  34 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  35 import org.opendaylight.genius.mdsalutil.NwConstants;
  36 import org.opendaylight.genius.mdsalutil.actions.ActionGroup;
  37 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  38 import org.opendaylight.genius.mdsalutil.actions.ActionOutput;
  39 import org.opendaylight.genius.mdsalutil.actions.ActionPushMpls;
  40 import org.opendaylight.genius.mdsalutil.actions.ActionPushVlan;
  41 import org.opendaylight.genius.mdsalutil.actions.ActionRegLoad;
  42 import org.opendaylight.genius.mdsalutil.actions.ActionRegMove;
  43 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  44 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetSource;
  45 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldTunnelId;
  46 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldVlanVid;
  47 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  48 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  49 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  50 import org.opendaylight.netvirt.fibmanager.api.L3VPNTransportTypes;
  51 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  52 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev140508.L2vlan;
  53 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.iana._if.type.rev140508.Tunnel;
  54 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  55 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.InterfaceType;
  56 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  57 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface.OperStatus;
  58 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  59 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.OutputActionCase;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.PushVlanActionCase;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.action.SetFieldCase;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInput;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdInputBuilder;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.AllocateIdOutput;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInput;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInputBuilder;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInput;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.ReleaseIdInputBuilder;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeBase;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeGre;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeMplsOverGre;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rev160406.TunnelTypeVxlan;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceInputBuilder;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.GetEgressActionsForInterfaceOutput;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelOperStatus;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.TunnelsState;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelListKey;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameInputBuilder;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetInternalOrExternalInterfaceNameOutput;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameInputBuilder;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.GetTunnelInterfaceNameOutput;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupInput;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupInputBuilder;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.AddGroupOutput;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.service.rev130918.SalGroupService;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupId;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupRef;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.GroupTypes;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.Buckets;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.group.buckets.Bucket;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.group.types.rev131018.groups.Group;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.inventory.rev130819.NodeRef;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeBase;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeFlat;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.SegmentTypeVlan;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.instances.ElanInstance;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.L3nexthop;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthops;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.VpnNexthopsKey;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthop;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopBuilder;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3nexthop.rev150409.l3nexthop.vpnnexthops.VpnNexthopKey;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpn;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.ConfTransportTypeL3vpnBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.dpid.l3vpn.lb.nexthops.DpnLbNexthops;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.l3vpn.lb.nexthops.Nexthops;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowjava.nx.match.rev140421.NxmNxReg6;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.add.group.input.buckets.bucket.action.action.NxActionResubmitRpcAddGroupCase;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.nodes.node.table.flow.instructions.instruction.instruction.apply.actions._case.apply.actions.action.action.NxActionRegLoadNodesNodeTableFlowApplyActionsCase;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.openflowplugin.extension.nicira.action.rev140714.nx.action.reg.load.grouping.NxRegLoad;
 121 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 122 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 123 import org.opendaylight.yangtools.yang.common.RpcResult;
 124 import org.slf4j.Logger;
 125 import org.slf4j.LoggerFactory;
 126
 127 @Singleton
 128 public class NexthopManager implements AutoCloseable {
 129     private static final Logger LOG = LoggerFactory.getLogger(NexthopManager.class);
 130     private static final String NEXTHOP_ID_POOL_NAME = "nextHopPointerPool";
 131     private static final long WAIT_TIME_FOR_SYNC_INSTALL = Long.getLong("wait.time.sync.install", 300L);
 132
 133     private final DataBroker dataBroker;
 134     private final IMdsalApiManager mdsalApiManager;
 135     private final OdlInterfaceRpcService interfaceManager;
 136     private final ItmRpcService itmManager;
 137     private final IdManagerService idManager;
 138     private final IElanService elanService;
 139     private final SalGroupService salGroupService;
 140     private final JobCoordinator jobCoordinator;
 141     private final FibUtil fibUtil;
 142     private volatile L3VPNTransportTypes configuredTransportTypeL3VPN = L3VPNTransportTypes.Invalid;
 143
 144     /**
 145      * Provides nexthop functions.
 146      * Creates group ID pool
 147      *
 148      * @param dataBroker       - dataBroker reference
 149      * @param mdsalApiManager  - mdsalApiManager reference
 150      * @param idManager        - idManager reference
 151      * @param interfaceManager - interfaceManager reference
 152      * @param itmManager       - itmManager reference
 153      */
 154     @Inject
 155     public NexthopManager(final DataBroker dataBroker,
 156                           final IMdsalApiManager mdsalApiManager,
 157                           final IdManagerService idManager,
 158                           final OdlInterfaceRpcService interfaceManager,
 159                           final ItmRpcService itmManager,
 160                           final IElanService elanService,
 161                           final SalGroupService salGroupService,
 162                           final JobCoordinator jobCoordinator,
 163                           final FibUtil fibUtil) {
 164         this.dataBroker = dataBroker;
 165         this.mdsalApiManager = mdsalApiManager;
 166         this.idManager = idManager;
 167         this.interfaceManager = interfaceManager;
 168         this.itmManager = itmManager;
 169         this.elanService = elanService;
 170         this.salGroupService = salGroupService;
 171         this.jobCoordinator = jobCoordinator;
 172         this.fibUtil = fibUtil;
 173         createIdPool();
 174     }
 175
 176     private void createIdPool() {
 177         CreateIdPoolInput createPool = new CreateIdPoolInputBuilder()
 178             .setPoolName(NEXTHOP_ID_POOL_NAME)
 179             .setLow(150000L)
 180             .setHigh(175000L)
 181             .build();
 182         try {
 183             Future<RpcResult<Void>> result = idManager.createIdPool(createPool);
 184             if (result != null && result.get().isSuccessful()) {
 185                 LOG.info("Created IdPool for NextHopPointerPool");
 186             }
 187         } catch (InterruptedException | ExecutionException e) {
 188             LOG.error("Failed to create idPool for NextHopPointerPool", e);
 189         }
 190     }
 191
 192     private String getNextHopKey(long vpnId, String ipAddress) {
 193         return "nexthop." + vpnId + ipAddress;
 194     }
 195
 196     public OdlInterfaceRpcService getInterfaceManager() {
 197         return interfaceManager;
 198     }
 199
 200     protected long createNextHopPointer(String nexthopKey) {
 201         AllocateIdInput getIdInput = new AllocateIdInputBuilder()
 202             .setPoolName(NEXTHOP_ID_POOL_NAME).setIdKey(nexthopKey)
 203             .build();
 204         //TODO: Proper error handling once IdManager code is complete
 205         try {
 206             Future<RpcResult<AllocateIdOutput>> result = idManager.allocateId(getIdInput);
 207             RpcResult<AllocateIdOutput> rpcResult = result.get();
 208             return rpcResult.getResult().getIdValue();
 209         } catch (NullPointerException | InterruptedException | ExecutionException e) {
 210             LOG.trace("", e);
 211         }
 212         return 0;
 213     }
 214
 215     protected void removeNextHopPointer(String nexthopKey) {
 216         ReleaseIdInput idInput = new ReleaseIdInputBuilder()
 217             .setPoolName(NEXTHOP_ID_POOL_NAME)
 218             .setIdKey(nexthopKey).build();
 219         try {
 220             Future<RpcResult<Void>> result = idManager.releaseId(idInput);
 221             RpcResult<Void> rpcResult = result.get();
 222             if (!rpcResult.isSuccessful()) {
 223                 LOG.error("RPC Call to Get Unique Id for nexthopKey {} returned with Errors {}",
 224                         nexthopKey, rpcResult.getErrors());
 225             }
 226         } catch (InterruptedException | ExecutionException e) {
 227             LOG.warn("Exception when getting Unique Id for key {}", nexthopKey, e);
 228         }
 229     }
 230
 231     protected List<ActionInfo> getEgressActionsForInterface(final String ifName, int actionKey) {
 232         List<ActionInfo> listActionInfo = new ArrayList<>();
 233         try {
 234             Future<RpcResult<GetEgressActionsForInterfaceOutput>> result =
 235                 interfaceManager.getEgressActionsForInterface(
 236                     new GetEgressActionsForInterfaceInputBuilder().setIntfName(ifName).build());
 237             RpcResult<GetEgressActionsForInterfaceOutput> rpcResult = result.get();
 238             if (!rpcResult.isSuccessful()) {
 239                 LOG.error("RPC Call to Get egress actions for interface {} returned with Errors {}",
 240                     ifName, rpcResult.getErrors());
 241             } else {
 242                 List<org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.list.Action> actions =
 243                     rpcResult.getResult().getAction();
 244                 for (Action action : actions) {
 245                     actionKey = action.getKey().getOrder() + actionKey;
 246                     org.opendaylight.yang.gen.v1.urn.opendaylight.action.types.rev131112.action.Action
 247                         actionClass = action.getAction();
 248                     if (actionClass instanceof OutputActionCase) {
 249                         listActionInfo.add(new ActionOutput(actionKey,
 250                             ((OutputActionCase) actionClass).getOutputAction().getOutputNodeConnector()));
 251                     } else if (actionClass instanceof PushVlanActionCase) {
 252                         listActionInfo.add(new ActionPushVlan(actionKey));
 253                     } else if (actionClass instanceof SetFieldCase) {
 254                         if (((SetFieldCase) actionClass).getSetField().getVlanMatch() != null) {
 255                             int vlanVid = ((SetFieldCase) actionClass).getSetField().getVlanMatch()
 256                                 .getVlanId().getVlanId().getValue();
 257                             listActionInfo.add(new ActionSetFieldVlanVid(actionKey, vlanVid));
 258                         }
 259                     } else if (actionClass instanceof NxActionResubmitRpcAddGroupCase) {
 260                         Short tableId = ((NxActionResubmitRpcAddGroupCase) actionClass).getNxResubmit().getTable();
 261                         listActionInfo.add(new ActionNxResubmit(actionKey, tableId));
 262                     } else if (actionClass instanceof NxActionRegLoadNodesNodeTableFlowApplyActionsCase) {
 263                         NxRegLoad nxRegLoad =
 264                             ((NxActionRegLoadNodesNodeTableFlowApplyActionsCase) actionClass).getNxRegLoad();
 265                         listActionInfo.add(new ActionRegLoad(actionKey, NxmNxReg6.class,
 266                             nxRegLoad.getDst().getStart(), nxRegLoad.getDst().getEnd(),
 267                             nxRegLoad.getValue().longValue()));
 268                     }
 269                 }
 270             }
 271         } catch (InterruptedException | ExecutionException e) {
 272             LOG.warn("Exception when egress actions for interface {}", ifName, e);
 273         }
 274         return listActionInfo;
 275     }
 276
 277     protected String getTunnelInterfaceName(BigInteger srcDpId, BigInteger dstDpId) {
 278         Class<? extends TunnelTypeBase> tunType = getReqTunType(getReqTransType().toUpperCase(Locale.getDefault()));
 279         Future<RpcResult<GetTunnelInterfaceNameOutput>> result;
 280         try {
 281             result = itmManager.getTunnelInterfaceName(new GetTunnelInterfaceNameInputBuilder()
 282                 .setSourceDpid(srcDpId)
 283                 .setDestinationDpid(dstDpId)
 284                 .setTunnelType(tunType)
 285                 .build());
 286             RpcResult<GetTunnelInterfaceNameOutput> rpcResult = result.get();
 287             if (!rpcResult.isSuccessful()) {
 288                 LOG.warn("RPC Call to getTunnelInterfaceId returned with Errors {}", rpcResult.getErrors());
 289             } else {
 290                 return rpcResult.getResult().getInterfaceName();
 291             }
 292         } catch (InterruptedException | ExecutionException e) {
 293             LOG.warn("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstDpId, e);
 294         }
 295         return null;
 296     }
 297
 298     protected String getTunnelInterfaceName(BigInteger srcDpId, org.opendaylight.yang.gen.v1.urn.ietf.params
 299         .xml.ns.yang.ietf.inet.types.rev130715.IpAddress dstIp) {
 300         Class<? extends TunnelTypeBase> tunType = getReqTunType(getReqTransType().toUpperCase(Locale.getDefault()));
 301         Future<RpcResult<GetInternalOrExternalInterfaceNameOutput>> result;
 302         try {
 303             result = itmManager.getInternalOrExternalInterfaceName(new GetInternalOrExternalInterfaceNameInputBuilder()
 304                 .setSourceDpid(srcDpId)
 305                 .setDestinationIp(dstIp)
 306                 .setTunnelType(tunType)
 307                 .build());
 308             RpcResult<GetInternalOrExternalInterfaceNameOutput> rpcResult = result.get();
 309             if (!rpcResult.isSuccessful()) {
 310                 LOG.warn("RPC Call to getTunnelInterfaceName returned with Errors {}", rpcResult.getErrors());
 311             } else {
 312                 return rpcResult.getResult().getInterfaceName();
 313             }
 314         } catch (InterruptedException | ExecutionException e) {
 315             LOG.warn("Exception when getting tunnel interface Id for tunnel between {} and  {}", srcDpId, dstIp, e);
 316         }
 317         return null;
 318     }
 319
 320
 321     public long getLocalNextHopGroup(long vpnId,
 322             String ipNextHopAddress) {
 323         long groupId = createNextHopPointer(getNextHopKey(vpnId, ipNextHopAddress));
 324         if (groupId == FibConstants.INVALID_GROUP_ID) {
 325             LOG.error("Unable to allocate groupId for vpnId {} , prefix {}", vpnId, ipNextHopAddress);
 326         }
 327         return groupId;
 328     }
 329
 330     public long createLocalNextHop(long vpnId, BigInteger dpnId, String ifName,
 331             String ipNextHopAddress, String ipPrefixAddress, String gwMacAddress, String jobKey) {
 332         String vpnName = fibUtil.getVpnNameFromId(vpnId);
 333         if (vpnName == null) {
 334             return 0;
 335         }
 336         String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, ipPrefixAddress);
 337         String ipAddress = macAddress != null ? ipPrefixAddress : ipNextHopAddress;
 338         long groupId = createNextHopPointer(getNextHopKey(vpnId, ipAddress));
 339         if (groupId == 0) {
 340             LOG.error("Unable to allocate groupId for vpnId {} , prefix {}  IntfName {}, nextHopAddr {}",
 341                     vpnId, ipAddress, ifName, ipNextHopAddress);
 342             return groupId;
 343         }
 344         String nextHopLockStr = vpnId + ipAddress;
 345         jobCoordinator.enqueueJob(jobKey, () -> {
 346             synchronized (nextHopLockStr.intern()) {
 347                 VpnNexthop nexthop = getVpnNexthop(vpnId, ipAddress);
 348                 LOG.trace("nexthop: {} retrieved for vpnId {}, prefix {}, ifName {} on dpn {}", nexthop, vpnId,
 349                         ipAddress, ifName, dpnId);
 350                 if (nexthop == null) {
 351                     String encMacAddress = macAddress == null
 352                         ? fibUtil.getMacAddressFromPrefix(ifName, vpnName, ipAddress) : macAddress;
 353                     List<BucketInfo> listBucketInfo = new ArrayList<>();
 354                     List<ActionInfo> listActionInfo = new ArrayList<>();
 355                     int actionKey = 0;
 356                     // MAC re-write
 357                     if (encMacAddress != null) {
 358                         if (gwMacAddress != null) {
 359                             LOG.trace("The Local NextHop Group Source Mac {} for VpnInterface {} on VPN {}",
 360                                     gwMacAddress, ifName, vpnId);
 361                             listActionInfo
 362                                     .add(new ActionSetFieldEthernetSource(actionKey++, new MacAddress(gwMacAddress)));
 363                         }
 364                         listActionInfo
 365                                 .add(new ActionSetFieldEthernetDestination(actionKey++, new MacAddress(encMacAddress)));
 366                         // listActionInfo.add(0, new ActionPopMpls());
 367                     } else {
 368                         // FIXME: Log message here.
 369                         LOG.debug("mac address for new local nexthop is null");
 370                     }
 371                     listActionInfo.addAll(getEgressActionsForInterface(ifName, actionKey));
 372                     BucketInfo bucket = new BucketInfo(listActionInfo);
 373
 374                     listBucketInfo.add(bucket);
 375                     GroupEntity groupEntity = MDSALUtil.buildGroupEntity(dpnId, groupId, ipAddress, GroupTypes.GroupAll,
 376                             listBucketInfo);
 377                     LOG.trace("Install LNH Group: id {}, mac address {}, interface {} for prefix {}", groupId,
 378                             encMacAddress, ifName, ipAddress);
 379                     //Try to install group directly on the DPN bypassing the FRM, in order to avoid waiting for the
 380                     // group to get installed before programming the flows
 381                     installGroupOnDpn(groupId, dpnId, ipAddress, listBucketInfo, getNextHopKey(vpnId, ipAddress),
 382                             GroupTypes.GroupAll);
 383                     // install Group
 384                     mdsalApiManager.syncInstallGroup(groupEntity);
 385                     // update MD-SAL DS
 386                     addVpnNexthopToDS(dpnId, vpnId, ipAddress, groupId);
 387
 388                 } else {
 389                     // nexthop exists already; a new flow is going to point to
 390                     // it, increment the flowrefCount by 1
 391                     int flowrefCnt = nexthop.getFlowrefCount() + 1;
 392                     VpnNexthop nh = new VpnNexthopBuilder().setKey(new VpnNexthopKey(ipAddress))
 393                             .setFlowrefCount(flowrefCnt).build();
 394                     LOG.trace("Updating vpnnextHop {} for refCount {} to Operational DS", nh, flowrefCnt);
 395                     MDSALUtil.syncUpdate(dataBroker, LogicalDatastoreType.OPERATIONAL, getVpnNextHopIdentifier(vpnId,
 396                             ipAddress), nh);
 397                 }
 398             }
 399             return Collections.emptyList();
 400         });
 401         return groupId;
 402     }
 403
 404     private void installGroupOnDpn(long groupId, BigInteger dpnId, String groupName, List<BucketInfo> bucketsInfo,
 405                                      String nextHopKey, GroupTypes groupType) {
 406         NodeRef nodeRef = FibUtil.buildNodeRef(dpnId);
 407         Buckets buckets = FibUtil.buildBuckets(bucketsInfo);
 408         GroupRef groupRef = new GroupRef(FibUtil.buildGroupInstanceIdentifier(groupId, dpnId));
 409         AddGroupInput input = new AddGroupInputBuilder().setNode(nodeRef).setGroupId(new GroupId(groupId))
 410                 .setBuckets(buckets).setGroupRef(groupRef).setGroupType(groupType)
 411                 .setGroupName(groupName).build();
 412         Future<RpcResult<AddGroupOutput>> groupStats = salGroupService.addGroup(input);
 413         RpcResult<AddGroupOutput> rpcResult = null;
 414         try {
 415             rpcResult = groupStats.get();
 416             if (rpcResult != null && rpcResult.isSuccessful()) {
 417                 LOG.info("Group {} with key {} has been successfully installed directly on dpn {}.", groupId,
 418                         nextHopKey, dpnId);
 419             } else {
 420                 LOG.error("Unable to install group {} with key {} directly on dpn {} due to {}.", groupId, nextHopKey,
 421                         dpnId, rpcResult != null ? rpcResult.getErrors() : null);
 422             }
 423         } catch (InterruptedException | ExecutionException e) {
 424             LOG.error("Error while installing group {} directly on dpn {}", groupId, dpnId);
 425         }
 426     }
 427
 428     protected void addVpnNexthopToDS(BigInteger dpnId, long vpnId, String ipPrefix, long egressPointer) {
 429         InstanceIdentifierBuilder<VpnNexthops> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
 430             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId));
 431
 432         // Add nexthop to vpn node
 433         VpnNexthop nh = new VpnNexthopBuilder()
 434             .setKey(new VpnNexthopKey(ipPrefix))
 435             .setDpnId(dpnId)
 436             .setIpAddress(ipPrefix)
 437             .setFlowrefCount(1)
 438             .setEgressPointer(egressPointer).build();
 439
 440         InstanceIdentifier<VpnNexthop> id1 = idBuilder
 441             .child(VpnNexthop.class, new VpnNexthopKey(ipPrefix)).build();
 442         LOG.trace("Adding vpnnextHop {} to Operational DS", nh);
 443         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, id1, nh);
 444
 445     }
 446
 447     protected InstanceIdentifier<VpnNexthop> getVpnNextHopIdentifier(long vpnId, String ipAddress) {
 448         InstanceIdentifier<VpnNexthop> id = InstanceIdentifier.builder(L3nexthop.class)
 449             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId)).child(VpnNexthop.class,
 450                 new VpnNexthopKey(ipAddress)).build();
 451         return id;
 452     }
 453
 454     protected VpnNexthop getVpnNexthop(long vpnId, String ipAddress) {
 455
 456         // check if vpn node is there
 457         InstanceIdentifierBuilder<VpnNexthops> idBuilder =
 458             InstanceIdentifier.builder(L3nexthop.class).child(VpnNexthops.class,
 459                 new VpnNexthopsKey(vpnId));
 460         InstanceIdentifier<VpnNexthops> id = idBuilder.build();
 461         Optional<VpnNexthops> vpnNexthops = MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 462         if (vpnNexthops.isPresent()) {
 463             // get nexthops list for vpn
 464             List<VpnNexthop> nexthops = vpnNexthops.get().getVpnNexthop();
 465             for (VpnNexthop nexthop : nexthops) {
 466                 if (nexthop.getIpAddress().equals(ipAddress)) {
 467                     // return nexthop
 468                     LOG.trace("VpnNextHop : {}", nexthop);
 469                     return nexthop;
 470                 }
 471             }
 472             // return null if not found
 473         }
 474         return null;
 475     }
 476
 477     public AdjacencyResult getRemoteNextHopPointer(BigInteger remoteDpnId, long vpnId, String prefixIp,
 478             String nextHopIp) {
 479         String egressIfName = null;
 480         LOG.trace("getRemoteNextHopPointer: input [remoteDpnId {}, vpnId {}, prefixIp {}, nextHopIp {} ]", remoteDpnId,
 481             vpnId, prefixIp, nextHopIp);
 482
 483         Class<? extends InterfaceType> egressIfType;
 484         ElanInstance elanInstance = getElanInstanceForPrefix(vpnId, prefixIp);
 485         if (elanInstance != null) {
 486             egressIfType = getInterfaceType(elanInstance);
 487         } else {
 488             LOG.warn("Failed to determine network type for prefixIp {} using tunnel", prefixIp);
 489             egressIfType = Tunnel.class;
 490         }
 491
 492         if (Tunnel.class.equals(egressIfType)) {
 493             egressIfName = getTunnelRemoteNextHopPointer(remoteDpnId, nextHopIp);
 494         } else {
 495             egressIfName = getExtPortRemoteNextHopPointer(remoteDpnId, elanInstance);
 496         }
 497
 498         LOG.trace("NextHop pointer for prefixIp {} vpnId {} dpnId {} is {}", prefixIp, vpnId, remoteDpnId,
 499             egressIfName);
 500         return egressIfName != null ? new AdjacencyResult(egressIfName, egressIfType, nextHopIp,
 501                 prefixIp) : null;
 502     }
 503
 504     public BigInteger getDpnForPrefix(long vpnId, String prefixIp) {
 505         VpnNexthop vpnNexthop = getVpnNexthop(vpnId, prefixIp);
 506         BigInteger localDpnId = vpnNexthop == null ? null : vpnNexthop.getDpnId();
 507         return localDpnId;
 508     }
 509
 510     private void removeVpnNexthopFromDS(long vpnId, String ipPrefix) {
 511
 512         InstanceIdentifierBuilder<VpnNexthop> idBuilder = InstanceIdentifier.builder(L3nexthop.class)
 513             .child(VpnNexthops.class, new VpnNexthopsKey(vpnId))
 514             .child(VpnNexthop.class, new VpnNexthopKey(ipPrefix));
 515         InstanceIdentifier<VpnNexthop> id = idBuilder.build();
 516         // remove from DS
 517         LOG.trace("Removing vpn next hop from datastore : {}", id);
 518         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, id);
 519     }
 520
 521     public void removeLocalNextHop(BigInteger dpnId, Long vpnId, String ipNextHopAddress, String ipPrefixAddress) {
 522         String ipPrefixStr = vpnId + ipPrefixAddress;
 523         VpnNexthop prefixNh = null;
 524         synchronized (ipPrefixStr.intern()) {
 525             prefixNh = getVpnNexthop(vpnId, ipPrefixAddress);
 526         }
 527         String ipAddress = prefixNh != null ? ipPrefixAddress : ipNextHopAddress;
 528
 529         String nextHopLockStr = vpnId + ipAddress;
 530         synchronized (nextHopLockStr.intern()) {
 531             VpnNexthop nh = getVpnNexthop(vpnId, ipAddress);
 532             if (nh != null) {
 533                 int newFlowrefCnt = nh.getFlowrefCount() - 1;
 534                 if (newFlowrefCnt == 0) { //remove the group only if there are no more flows using this group
 535                     GroupEntity groupEntity = MDSALUtil.buildGroupEntity(
 536                         dpnId, nh.getEgressPointer(), ipAddress, GroupTypes.GroupAll,
 537                             Collections.EMPTY_LIST /*listBucketInfo*/);
 538                     // remove Group ...
 539                     mdsalApiManager.removeGroup(groupEntity);
 540                     //update MD-SAL DS
 541                     removeVpnNexthopFromDS(vpnId, ipAddress);
 542                     //release groupId
 543                     removeNextHopPointer(getNextHopKey(vpnId, ipAddress));
 544                     LOG.debug("Local Next hop {} for {} {} on dpn {} successfully deleted",
 545                         nh.getEgressPointer(), vpnId, ipAddress, dpnId);
 546                 } else {
 547                     //just update the flowrefCount of the vpnNexthop
 548                     VpnNexthop currNh = new VpnNexthopBuilder().setKey(new VpnNexthopKey(ipAddress))
 549                         .setFlowrefCount(newFlowrefCnt).build();
 550                     LOG.trace("Updating vpnnextHop {} for refCount {} to Operational DS", currNh, newFlowrefCnt);
 551                     MDSALUtil.syncUpdate(dataBroker, LogicalDatastoreType.OPERATIONAL, getVpnNextHopIdentifier(vpnId,
 552                             ipAddress), currNh);
 553                 }
 554             } else {
 555                 //throw error
 556                 LOG.error("Local Next hop for Prefix {} VpnId {} on dpn {} not deleted", ipAddress, vpnId, dpnId);
 557             }
 558         }
 559     }
 560
 561     public void setConfTransType(String service, String transportType) {
 562
 563         if (!service.equalsIgnoreCase("L3VPN")) {
 564             LOG.error("Incorrect service {} provided for setting the transport type.", service);
 565             return;
 566         }
 567
 568         L3VPNTransportTypes transType = L3VPNTransportTypes.validateTransportType(transportType
 569                 .toUpperCase(Locale.getDefault()));
 570
 571         if (transType != L3VPNTransportTypes.Invalid) {
 572             configuredTransportTypeL3VPN = transType;
 573         }
 574     }
 575
 576     public void writeConfTransTypeConfigDS() {
 577         MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, getConfTransportTypeIdentifier(),
 578             createConfTransportType(configuredTransportTypeL3VPN.getTransportType()));
 579     }
 580
 581     public L3VPNTransportTypes getConfiguredTransportTypeL3VPN() {
 582         return this.configuredTransportTypeL3VPN;
 583     }
 584
 585     public String getReqTransType() {
 586         if (configuredTransportTypeL3VPN == L3VPNTransportTypes.Invalid) {
 587             /*
 588              * Restart scenario, Read from the ConfigDS.
 589              * if the value is Unset, cache value as VxLAN.
 590              */
 591             LOG.trace("configureTransportType is not yet set.");
 592             Optional<ConfTransportTypeL3vpn> configuredTransTypeFromConfig =
 593                 MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, getConfTransportTypeIdentifier());
 594
 595             if (configuredTransTypeFromConfig.isPresent()) {
 596                 if (configuredTransTypeFromConfig.get().getTransportType().equals(TunnelTypeGre.class)) {
 597                     configuredTransportTypeL3VPN = L3VPNTransportTypes.GRE;
 598                 } else {
 599                     configuredTransportTypeL3VPN = L3VPNTransportTypes.VxLAN;
 600                 }
 601                 LOG.trace("configuredTransportType set from config DS to {}",
 602                     getConfiguredTransportTypeL3VPN().getTransportType());
 603             } else {
 604                 setConfTransType("L3VPN", L3VPNTransportTypes.VxLAN.getTransportType());
 605                 LOG.trace("configuredTransportType is not set in the Config DS. VxLAN as default will be used.");
 606             }
 607         } else {
 608             LOG.trace("configuredTransportType is set as {}", getConfiguredTransportTypeL3VPN().getTransportType());
 609         }
 610         return getConfiguredTransportTypeL3VPN().getTransportType();
 611     }
 612
 613     public InstanceIdentifier<ConfTransportTypeL3vpn> getConfTransportTypeIdentifier() {
 614         return InstanceIdentifier.builder(ConfTransportTypeL3vpn.class).build();
 615     }
 616
 617     private ConfTransportTypeL3vpn createConfTransportType(String type) {
 618         ConfTransportTypeL3vpn confTransType;
 619         switch (type) {
 620             case ITMConstants.TUNNEL_TYPE_GRE:
 621                 confTransType = new ConfTransportTypeL3vpnBuilder().setTransportType(TunnelTypeGre.class).build();
 622                 LOG.trace("Setting the confTransportType to GRE.");
 623                 break;
 624             case ITMConstants.TUNNEL_TYPE_VXLAN:
 625                 confTransType = new ConfTransportTypeL3vpnBuilder().setTransportType(TunnelTypeVxlan.class).build();
 626                 LOG.trace("Setting the confTransportType to VxLAN.");
 627                 break;
 628             default:
 629                 LOG.trace("Invalid transport type {} passed to Config DS ", type);
 630                 confTransType = null;
 631                 break;
 632         }
 633         return confTransType;
 634     }
 635
 636     public Class<? extends TunnelTypeBase> getReqTunType(String transportType) {
 637         switch (transportType) {
 638             case "VXLAN":
 639                 return TunnelTypeVxlan.class;
 640             case "GRE":
 641                 return TunnelTypeGre.class;
 642             default:
 643                 return TunnelTypeMplsOverGre.class;
 644         }
 645     }
 646
 647     public String getTransportTypeStr(String tunType) {
 648         if (tunType.equals(TunnelTypeVxlan.class.toString())) {
 649             return ITMConstants.TUNNEL_TYPE_VXLAN;
 650         } else if (tunType.equals(TunnelTypeGre.class.toString())) {
 651             return ITMConstants.TUNNEL_TYPE_GRE;
 652         } else if (tunType.equals(TunnelTypeMplsOverGre.class.toString())) {
 653             return ITMConstants.TUNNEL_TYPE_MPLSoGRE;
 654         } else {
 655             return ITMConstants.TUNNEL_TYPE_INVALID;
 656         }
 657     }
 658
 659     @Override
 660     @PreDestroy
 661     public void close() throws Exception {
 662         LOG.info("{} close", getClass().getSimpleName());
 663     }
 664
 665     // TODO Clean up the exception handling
 666     @SuppressWarnings("checkstyle:IllegalCatch")
 667     private String getTunnelRemoteNextHopPointer(BigInteger remoteDpnId, String nextHopIp) {
 668         if (nextHopIp != null && !nextHopIp.isEmpty()) {
 669             try {
 670                 // here use the config for tunnel type param
 671                 return getTunnelInterfaceName(remoteDpnId,
 672                     org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder
 673                         .getDefaultInstance(nextHopIp));
 674             } catch (Exception ex) {
 675                 LOG.error("Error while retrieving nexthop pointer for nexthop {} remoteDpn {}",
 676                         nextHopIp, remoteDpnId, ex);
 677             }
 678         }
 679
 680         return null;
 681     }
 682
 683     private String getExtPortRemoteNextHopPointer(BigInteger remoteDpnId, ElanInstance elanInstance) {
 684         return elanService.getExternalElanInterface(elanInstance.getElanInstanceName(), remoteDpnId);
 685     }
 686
 687     /**
 688      * Get the interface type associated with the type of ELAN used for routing
 689      * traffic to/from remote compute nodes.
 690      *
 691      * @param elanInstance The elan instance
 692      * @return L2vlan for flat/VLAN network type and Tunnel otherwise
 693      */
 694     private Class<? extends InterfaceType> getInterfaceType(ElanInstance elanInstance) {
 695         Class<? extends SegmentTypeBase> segmentType = elanInstance.getSegmentType();
 696         if (SegmentTypeFlat.class.equals(segmentType) || SegmentTypeVlan.class.equals(segmentType)) {
 697             return L2vlan.class;
 698         }
 699
 700         return Tunnel.class;
 701     }
 702
 703     private ElanInstance getElanInstanceForPrefix(long vpnId, String prefixIp) {
 704         ElanInstance elanInstance = null;
 705         Prefixes prefix = fibUtil.getPrefixToInterface(vpnId, prefixIp);
 706         if (prefix != null) {
 707             Uuid subnetId = prefix.getSubnetId();
 708             if (subnetId != null) {
 709                 Subnetmap subnetMap = fibUtil.getSubnetMap(subnetId);
 710                 if (subnetMap != null && subnetMap.getNetworkId() != null) {
 711                     elanInstance = elanService.getElanInstance(subnetMap.getNetworkId().getValue());
 712                 }
 713             }
 714         }
 715
 716         return elanInstance;
 717     }
 718
 719     static class AdjacencyResult {
 720         private final String interfaceName;
 721         private final Class<? extends InterfaceType> interfaceType;
 722         private final String nextHopIp;
 723         private final String prefix;
 724
 725         AdjacencyResult(String interfaceName, Class<? extends InterfaceType> interfaceType, String nextHopIp,
 726                         String prefix) {
 727             this.interfaceName = interfaceName;
 728             this.interfaceType = interfaceType;
 729             this.nextHopIp = nextHopIp;
 730             this.prefix = prefix;
 731         }
 732
 733         public String getInterfaceName() {
 734             return interfaceName;
 735         }
 736
 737         public Class<? extends InterfaceType> getInterfaceType() {
 738             return interfaceType;
 739         }
 740
 741         public String getNextHopIp() {
 742             return nextHopIp;
 743         }
 744
 745         public String getPrefix() {
 746             return prefix;
 747         }
 748
 749         @Override
 750         public int hashCode() {
 751             final int prime = 31;
 752             int result = 1;
 753             result = prime * result + (interfaceName == null ? 0 : interfaceName.hashCode());
 754             return result;
 755         }
 756
 757         @Override
 758         public boolean equals(Object obj) {
 759             if (obj == null) {
 760                 return false;
 761             }
 762
 763             boolean result = false;
 764             if (getClass() != obj.getClass()) {
 765                 return result;
 766             } else {
 767                 AdjacencyResult other = (AdjacencyResult) obj;
 768                 result = interfaceName.equals(other.interfaceName);
 769             }
 770             return result;
 771         }
 772     }
 773
 774     protected long setupLoadBalancingNextHop(Long parentVpnId, BigInteger dpnId,
 775             String destPrefix, List<BucketInfo> listBucketInfo, boolean addOrRemove) {
 776         long groupId = createNextHopPointer(getNextHopKey(parentVpnId, destPrefix));
 777         if (groupId == FibConstants.INVALID_GROUP_ID) {
 778             LOG.error("Unable to allocate/retrieve groupId for vpnId {} , prefix {}", parentVpnId, destPrefix);
 779             return groupId;
 780         }
 781         GroupEntity groupEntity = MDSALUtil.buildGroupEntity(
 782                 dpnId, groupId, destPrefix, GroupTypes.GroupSelect, listBucketInfo);
 783         if (addOrRemove) {
 784             mdsalApiManager.syncInstallGroup(groupEntity);
 785             try {
 786                 Thread.sleep(WAIT_TIME_FOR_SYNC_INSTALL);
 787             } catch (InterruptedException e1) {
 788                 LOG.warn("Thread got interrupted while programming LB group {}", groupEntity);
 789                 Thread.currentThread().interrupt();
 790             }
 791         } else {
 792             mdsalApiManager.removeGroup(groupEntity);
 793         }
 794         return groupId;
 795     }
 796
 797     long createNextHopGroups(Long vpnId, String rd, BigInteger dpnId, VrfEntry vrfEntry,
 798             Routes routes, List<Routes> vpnExtraRoutes) {
 799         List<BucketInfo> listBucketInfo = new ArrayList<>();
 800         List<Routes> clonedVpnExtraRoutes  = new ArrayList<>(vpnExtraRoutes);
 801         if (clonedVpnExtraRoutes.contains(routes)) {
 802             listBucketInfo.addAll(getBucketsForLocalNexthop(vpnId, dpnId, vrfEntry, routes));
 803             clonedVpnExtraRoutes.remove(routes);
 804         }
 805         listBucketInfo.addAll(getBucketsForRemoteNexthop(vpnId, dpnId, vrfEntry, rd, clonedVpnExtraRoutes));
 806         return setupLoadBalancingNextHop(vpnId, dpnId, vrfEntry.getDestPrefix(), listBucketInfo, true);
 807     }
 808
 809     private List<BucketInfo> getBucketsForLocalNexthop(Long vpnId, BigInteger dpnId,
 810             VrfEntry vrfEntry, Routes routes) {
 811         List<BucketInfo> listBucketInfo = new CopyOnWriteArrayList<>();
 812         routes.getNexthopIpList().parallelStream().forEach(nextHopIp -> {
 813             String localNextHopIP;
 814             if (isIpv4Address(nextHopIp)) {
 815                 localNextHopIP = nextHopIp + NwConstants.IPV4PREFIX;
 816             } else {
 817                 localNextHopIP = nextHopIp + NwConstants.IPV6PREFIX;
 818             }
 819             Prefixes localNextHopInfo = fibUtil.getPrefixToInterface(vpnId, localNextHopIP);
 820             if (localNextHopInfo != null) {
 821                 long groupId = getLocalNextHopGroup(vpnId, localNextHopIP);
 822                 if (groupId == FibConstants.INVALID_GROUP_ID) {
 823                     LOG.error("Unable to allocate groupId for vpnId {} , prefix {} , interface {}", vpnId,
 824                             vrfEntry.getDestPrefix(), localNextHopInfo.getVpnInterfaceName());
 825                     return;
 826                 }
 827                 List<ActionInfo> actionsInfos =
 828                         Collections.singletonList(new ActionGroup(groupId));
 829                 BucketInfo bucket = new BucketInfo(actionsInfos);
 830                 bucket.setWeight(1);
 831                 listBucketInfo.add(bucket);
 832             }
 833         });
 834         LOG.trace("LOCAL: listbucket {}, vpnId {}, dpnId {}, routes {}", listBucketInfo, vpnId, dpnId, routes);
 835         return listBucketInfo;
 836     }
 837
 838     private List<BucketInfo> getBucketsForRemoteNexthop(Long vpnId, BigInteger dpnId, VrfEntry vrfEntry, String rd,
 839             List<Routes> vpnExtraRoutes) {
 840         List<BucketInfo> listBucketInfo = new ArrayList<>();
 841         Map<String, List<ActionInfo>> egressActionMap = new HashMap<>();
 842         vpnExtraRoutes.forEach(vpnExtraRoute -> vpnExtraRoute.getNexthopIpList().forEach(nextHopIp -> {
 843             String nextHopPrefixIp;
 844             if (isIpv4Address(nextHopIp)) {
 845                 nextHopPrefixIp = nextHopIp + NwConstants.IPV4PREFIX;
 846             } else {
 847                 nextHopPrefixIp = nextHopIp + NwConstants.IPV6PREFIX;
 848             }
 849             List<String> tepIpAddresses = fibUtil.getNextHopAddresses(rd, nextHopPrefixIp);
 850             if (tepIpAddresses.isEmpty()) {
 851                 return;
 852             }
 853             // There would be only one nexthop address for a VM ip which would be the tep Ip
 854             String tepIp = tepIpAddresses.get(0);
 855             AdjacencyResult adjacencyResult = getRemoteNextHopPointer(dpnId, vpnId,
 856                     vrfEntry.getDestPrefix(), tepIp);
 857             if (adjacencyResult == null) {
 858                 return;
 859             }
 860             String egressInterface = adjacencyResult.getInterfaceName();
 861             if (!FibUtil.isTunnelInterface(adjacencyResult)) {
 862                 return;
 863             }
 864             Class<? extends TunnelTypeBase> tunnelType = VpnExtraRouteHelper
 865                     .getTunnelType(interfaceManager,
 866                             egressInterface);
 867             Interface ifState = fibUtil.getInterfaceStateFromOperDS(egressInterface);
 868             if (ifState == null || ifState.getOperStatus() != OperStatus.Up) {
 869                 LOG.trace("Tunnel not up {}", egressInterface);
 870                 return;
 871             }
 872             if (!TunnelTypeVxlan.class.equals(tunnelType)) {
 873                 return;
 874             }
 875             Long label = FibUtil.getLabelFromRoutePaths(vrfEntry).get();
 876             Prefixes prefixInfo = fibUtil.getPrefixToInterface(vpnId, nextHopPrefixIp);
 877             BigInteger tunnelId;
 878             if (fibUtil.enforceVxlanDatapathSemanticsforInternalRouterVpn(prefixInfo.getSubnetId(), vpnId,
 879                     rd)) {
 880                 java.util.Optional<Long> optionalVni = fibUtil.getVniForVxlanNetwork(prefixInfo.getSubnetId());
 881                 if (!optionalVni.isPresent()) {
 882                     LOG.error("VNI not found for nexthop {} vrfEntry {} with subnetId {}", nextHopIp,
 883                             vrfEntry, prefixInfo.getSubnetId());
 884                     return;
 885                 }
 886                 tunnelId = BigInteger.valueOf(optionalVni.get());
 887             } else {
 888                 tunnelId = BigInteger.valueOf(label);
 889             }
 890             List<ActionInfo> actionInfos = new ArrayList<>();
 891             actionInfos.add(new ActionSetFieldTunnelId(tunnelId));
 892             String ifName = prefixInfo.getVpnInterfaceName();
 893             String vpnName = fibUtil.getVpnNameFromId(vpnId);
 894             if (vpnName == null) {
 895                 return;
 896             }
 897             String macAddress = fibUtil.getMacAddressFromPrefix(ifName, vpnName, nextHopPrefixIp);
 898             actionInfos.add(new ActionSetFieldEthernetDestination(actionInfos.size(),
 899                     new MacAddress(macAddress)));
 900             List<ActionInfo> egressActions;
 901             if (egressActionMap.containsKey(egressInterface)) {
 902                 egressActions = egressActionMap.get(egressInterface);
 903             } else {
 904                 egressActions = getEgressActionsForInterface(egressInterface, actionInfos.size());
 905                 egressActionMap.put(egressInterface, egressActions);
 906             }
 907             if (egressActions.isEmpty()) {
 908                 LOG.error("Failed to retrieve egress action for prefix {} route-paths {}"
 909                         + " interface {}." + " Aborting remote FIB entry creation.",
 910                         vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), egressInterface);
 911             }
 912             actionInfos.addAll(egressActions);
 913             BucketInfo bucket = new BucketInfo(actionInfos);
 914             bucket.setWeight(1);
 915             listBucketInfo.add(bucket);
 916         }));
 917         LOG.trace("LOCAL: listbucket {}, rd {}, dpnId {}, routes {}", listBucketInfo, rd, dpnId, vpnExtraRoutes);
 918         return listBucketInfo;
 919     }
 920
 921     public void createDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId, String destinationIp) {
 922         Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
 923         int noOfDcGws = availableDcGws.size();
 924         if (noOfDcGws == 1) {
 925             LOG.trace("There are no enough DC GateWays {} present to program LB group", availableDcGws);
 926             return;
 927         }
 928         // TODO : Place the logic to construct all possible DC-GW combination here.
 929         String groupIdKey = FibUtil.getGreLbGroupKey(availableDcGws);
 930         Long groupId = createNextHopPointer(groupIdKey);
 931         List<Bucket> listBucket = new ArrayList<>();
 932         for (int index = 0; index < noOfDcGws; index++) {
 933             if (isTunnelUp(availableDcGws.get(index), dpnId)) {
 934                 listBucket.add(buildBucketForDcGwLbGroup(availableDcGws.get(index), dpnId, index));
 935             }
 936         }
 937         Group group = MDSALUtil.buildGroup(groupId, groupIdKey, GroupTypes.GroupSelect,
 938                         MDSALUtil.buildBucketLists(listBucket));
 939         WriteTransaction configTx = dataBroker.newWriteOnlyTransaction();
 940         WriteTransaction operationalTx = dataBroker.newWriteOnlyTransaction();
 941         mdsalApiManager.addGroupToTx(dpnId, group, configTx);
 942         FibUtil.updateLbGroupInfo(dpnId, destinationIp, groupIdKey, groupId.toString(), operationalTx);
 943         configTx.submit();
 944         operationalTx.submit();
 945         LOG.trace("LB group {} towards DC-GW installed on dpn {}. Group - {}", groupIdKey, dpnId, group);
 946     }
 947
 948     private boolean isTunnelUp(String dcGwIp, BigInteger dpnId) {
 949         String tunnelName = getTunnelRemoteNextHopPointer(dpnId, dcGwIp);
 950         if (tunnelName != null) {
 951             InstanceIdentifier<StateTunnelList> tunnelStateId =
 952                     InstanceIdentifier.builder(TunnelsState.class).child(
 953                             StateTunnelList.class, new StateTunnelListKey(tunnelName)).build();
 954             return MDSALUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, tunnelStateId)
 955                     .toJavaUtil().map(StateTunnelList::getOperState)
 956                     .orElse(TunnelOperStatus.Down) == TunnelOperStatus.Up;
 957         }
 958         return false;
 959     }
 960
 961     private List<Action> getEgressActions(String interfaceName, int actionKey) {
 962         List<Action> actions = Collections.emptyList();
 963         try {
 964             GetEgressActionsForInterfaceInputBuilder egressAction =
 965                     new GetEgressActionsForInterfaceInputBuilder().setIntfName(interfaceName).setActionKey(actionKey);
 966             Future<RpcResult<GetEgressActionsForInterfaceOutput>> result =
 967                     interfaceManager.getEgressActionsForInterface(egressAction.build());
 968             RpcResult<GetEgressActionsForInterfaceOutput> rpcResult = result.get();
 969             if (!rpcResult.isSuccessful()) {
 970                 LOG.warn("RPC Call to Get egress actions for interface {} returned with Errors {}",
 971                         interfaceName, rpcResult.getErrors());
 972             } else {
 973                 actions = rpcResult.getResult().getAction();
 974             }
 975         } catch (InterruptedException | ExecutionException e) {
 976             LOG.warn("Exception when egress actions for interface {}", interfaceName, e);
 977         }
 978         return actions;
 979     }
 980
 981     /**
 982      * This method is invoked when the tunnel state is removed from DS.
 983      * If the there is just one DC-GW left in configuration then the LB groups can be deleted.
 984      * Otherwise, the groups are just updated.
 985      */
 986     public void removeOrUpdateDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId,
 987             String destinationIp) {
 988         Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
 989         WriteTransaction configTx = dataBroker.newWriteOnlyTransaction();
 990         WriteTransaction operationalTx = dataBroker.newWriteOnlyTransaction();
 991         int noOfDcGws = availableDcGws.size();
 992         // If availableDcGws does not contain the destination Ip it means this is a configuration delete.
 993         if (!availableDcGws.contains(destinationIp)) {
 994             availableDcGws.add(destinationIp);
 995             Collections.sort(availableDcGws);
 996         }
 997         // TODO : Place the logic to construct all possible DC-GW combination here.
 998         int bucketId = availableDcGws.indexOf(destinationIp);
 999         Optional<DpnLbNexthops> dpnLbNextHops = fibUtil.getDpnLbNexthops(dpnId, destinationIp);
1000         if (!dpnLbNextHops.isPresent()) {
1001             return;
1002         }
1003         List<String> nextHopKeys = dpnLbNextHops.get().getNexthopKey();
1004         nextHopKeys.forEach(nextHopKey -> {
1005             Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
1006             if (!optionalNextHops.isPresent()) {
1007                 return;
1008             }
1009             Nexthops nexthops = optionalNextHops.get();
1010             final String groupId = nexthops.getGroupId();
1011             final long groupIdValue = Long.parseLong(groupId);
1012             if (noOfDcGws > 1) {
1013                 mdsalApiManager.removeBucketToTx(dpnId, groupIdValue, bucketId, configTx);
1014             } else {
1015                 Group group = MDSALUtil.buildGroup(groupIdValue, nextHopKey, GroupTypes.GroupSelect,
1016                         MDSALUtil.buildBucketLists(Collections.emptyList()));
1017                 LOG.trace("Removed LB group {} on dpn {}", group, dpnId);
1018                 mdsalApiManager.removeGroupToTx(dpnId, group, configTx);
1019                 removeNextHopPointer(nextHopKey);
1020             }
1021             // When the DC-GW is removed from configuration.
1022             if (noOfDcGws != availableDcGws.size()) {
1023                 FibUtil.removeOrUpdateNextHopInfo(dpnId, nextHopKey, groupId, nexthops, operationalTx);
1024             }
1025         });
1026         FibUtil.removeDpnIdToNextHopInfo(destinationIp, dpnId, operationalTx);
1027         configTx.submit();
1028         operationalTx.submit();
1029         return;
1030     }
1031
1032     /**
1033      * This method is invoked when the tunnel status is updated.
1034      * The bucket is directly removed/added based on the operational status of the tunnel.
1035      */
1036     public void updateDcGwLoadBalancingGroup(List<String> availableDcGws,
1037             BigInteger dpnId, String destinationIp, boolean isTunnelUp) {
1038         Preconditions.checkNotNull(availableDcGws, "There are no dc-gws present");
1039         WriteTransaction configTx = dataBroker.newWriteOnlyTransaction();
1040         // TODO : Place the logic to construct all possible DC-GW combination here.
1041         int bucketId = availableDcGws.indexOf(destinationIp);
1042         Optional<DpnLbNexthops> dpnLbNextHops = fibUtil.getDpnLbNexthops(dpnId, destinationIp);
1043         if (!dpnLbNextHops.isPresent()) {
1044             return;
1045         }
1046         List<String> nextHopKeys = dpnLbNextHops.get().getNexthopKey();
1047         nextHopKeys.forEach(nextHopKey -> {
1048             Optional<Nexthops> optionalNextHops = fibUtil.getNexthops(nextHopKey);
1049             if (!optionalNextHops.isPresent()) {
1050                 return;
1051             }
1052             Nexthops nexthops = optionalNextHops.get();
1053             final String groupId = nexthops.getGroupId();
1054             final long groupIdValue = Long.parseLong(groupId);
1055             if (isTunnelUp) {
1056                 Bucket bucket = buildBucketForDcGwLbGroup(destinationIp, dpnId, bucketId);
1057                 LOG.trace("Added bucket {} to group {} on dpn {}.", bucket, groupId, dpnId);
1058                 mdsalApiManager.addBucketToTx(dpnId, groupIdValue, bucket , configTx);
1059             } else {
1060                 LOG.trace("Removed bucketId {} from group {} on dpn {}.", bucketId, groupId, dpnId);
1061                 mdsalApiManager.removeBucketToTx(dpnId, groupIdValue, bucketId, configTx);
1062             }
1063         });
1064         configTx.submit();
1065         return;
1066     }
1067
1068     private Bucket buildBucketForDcGwLbGroup(String ipAddress, BigInteger dpnId,
1069             int index) {
1070         List<Action> listAction = new ArrayList<>();
1071         // ActionKey 0 goes to mpls label.
1072         int actionKey = 1;
1073         listAction.add(new ActionPushMpls().buildAction());
1074         listAction.add(new ActionRegMove(actionKey++, FibConstants.NXM_REG_MAPPING
1075                 .get(index), 0, 19).buildAction());
1076         String tunnelInterfaceName = getTunnelInterfaceName(dpnId, new IpAddress(ipAddress.toCharArray()));
1077         List<Action> egressActions = getEgressActions(tunnelInterfaceName, actionKey++);
1078         if (!egressActions.isEmpty()) {
1079             listAction.addAll(getEgressActions(tunnelInterfaceName, actionKey++));
1080         } else {
1081             // clear off actions if there is no egress actions.
1082             listAction = Collections.emptyList();
1083         }
1084         return MDSALUtil.buildBucket(listAction, MDSALUtil.GROUP_WEIGHT, index,
1085                 MDSALUtil.WATCH_PORT, MDSALUtil.WATCH_GROUP);
1086     }
1087
1088     public void programDcGwLoadBalancingGroup(List<String> availableDcGws, BigInteger dpnId,
1089             String destinationIp, int addRemoveOrUpdate, boolean isTunnelUp) {
1090         if (NwConstants.ADD_FLOW == addRemoveOrUpdate) {
1091             createDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp);
1092         } else if (NwConstants.DEL_FLOW == addRemoveOrUpdate) {
1093             removeOrUpdateDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp);
1094         } else if (NwConstants.MOD_FLOW == addRemoveOrUpdate) {
1095             updateDcGwLoadBalancingGroup(availableDcGws, dpnId, destinationIp, isTunnelUp);
1096         }
1097     }
1098 }