natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/VpnFloatingIpHandler.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  11 import static org.opendaylight.netvirt.natservice.internal.NatUtil.buildfloatingIpIdToPortMappingIdentifier;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.JdkFutureAdapters;
  17 import com.google.common.util.concurrent.ListenableFuture;
  18 import com.google.common.util.concurrent.MoreExecutors;
  19
  20 import java.math.BigInteger;
  21 import java.util.ArrayList;
  22 import java.util.Collections;
  23 import java.util.List;
  24 import java.util.concurrent.ExecutionException;
  25 import java.util.concurrent.Future;
  26 import javax.inject.Inject;
  27 import javax.inject.Singleton;
  28 import org.eclipse.jdt.annotation.NonNull;
  29 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  30 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  31 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  32 import org.opendaylight.genius.infra.Datastore.Configuration;
  33 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  34 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  35 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  36 import org.opendaylight.genius.infra.TypedWriteTransaction;
  37 import org.opendaylight.genius.mdsalutil.ActionInfo;
  38 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  39 import org.opendaylight.genius.mdsalutil.MatchInfo;
  40 import org.opendaylight.genius.mdsalutil.NwConstants;
  41 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  42 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  43 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  44 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  45 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  46 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  47 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  48 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  49 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  50 import org.opendaylight.infrautils.utils.concurrent.JdkFutures;
  51 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  52 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  53 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  54 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  55 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  56 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  57 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  58 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
  59 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  60 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  61 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.OdlArputilService;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpRequestInput;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpRequestInputBuilder;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.interfaces.InterfaceAddress;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.interfaces.InterfaceAddressBuilder;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.info.router.ports.ports.InternalToExternalPortMap;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.port.info.FloatingIpIdToPortMapping;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelOutput;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
  89 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  90 import org.opendaylight.yangtools.yang.common.RpcResult;
  91 import org.opendaylight.yangtools.yang.common.Uint32;
  92 import org.opendaylight.yangtools.yang.common.Uint64;
  93 import org.slf4j.Logger;
  94 import org.slf4j.LoggerFactory;
  95
  96 @Singleton
  97 public class VpnFloatingIpHandler implements FloatingIPHandler {
  98     private static final Logger LOG = LoggerFactory.getLogger(VpnFloatingIpHandler.class);
  99     private static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 100     private static final String FLOWID_PREFIX = "NAT.";
 101
 102     private final DataBroker dataBroker;
 103     private final ManagedNewTransactionRunner txRunner;
 104     private final IMdsalApiManager mdsalManager;
 105     private final VpnRpcService vpnService;
 106     private final IBgpManager bgpManager;
 107     private final FibRpcService fibService;
 108     private final IVpnManager vpnManager;
 109     private final IFibManager fibManager;
 110     private final OdlArputilService arpUtilService;
 111     private final IElanService elanService;
 112     private final EvpnDnatFlowProgrammer evpnDnatFlowProgrammer;
 113     private final NatServiceCounters natServiceCounters;
 114     private final NatOverVxlanUtil natOverVxlanUtil;
 115
 116     @Inject
 117     public VpnFloatingIpHandler(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 118                                 final VpnRpcService vpnService,
 119                                 final IBgpManager bgpManager,
 120                                 final FibRpcService fibService,
 121                                 final IFibManager fibManager,
 122                                 final OdlArputilService arputilService,
 123                                 final IVpnManager vpnManager,
 124                                 final IElanService elanService,
 125                                 final EvpnDnatFlowProgrammer evpnDnatFlowProgrammer,
 126                                 final NatOverVxlanUtil natOverVxlanUtil,
 127                                 NatServiceCounters natServiceCounters) {
 128         this.dataBroker = dataBroker;
 129         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 130         this.mdsalManager = mdsalManager;
 131         this.vpnService = vpnService;
 132         this.bgpManager = bgpManager;
 133         this.fibService = fibService;
 134         this.fibManager = fibManager;
 135         this.arpUtilService = arputilService;
 136         this.vpnManager = vpnManager;
 137         this.elanService = elanService;
 138         this.evpnDnatFlowProgrammer = evpnDnatFlowProgrammer;
 139         this.natServiceCounters = natServiceCounters;
 140         this.natOverVxlanUtil = natOverVxlanUtil;
 141     }
 142
 143     @Override
 144     public void onAddFloatingIp(final Uint64 dpnId, final String routerUuid, final Uint32 routerId,
 145                                 final Uuid networkId, final String interfaceName,
 146                                 final InternalToExternalPortMap mapping, final String rd,
 147                                 TypedReadWriteTransaction<Configuration> confTx) {
 148         String externalIp = mapping.getExternalIp();
 149         String internalIp = mapping.getInternalIp();
 150         Uuid floatingIpId = mapping.getExternalId();
 151         Uuid subnetId = NatUtil.getFloatingIpPortSubnetIdFromFloatingIpId(dataBroker, floatingIpId);
 152         String floatingIpPortMacAddress = NatUtil.getFloatingIpPortMacFromFloatingIpId(dataBroker, floatingIpId);
 153         if (floatingIpPortMacAddress == null) {
 154             LOG.error("onAddFloatingIp: Unable to retrieve floatingIp port MAC address from floatingIpId {} for "
 155                     + "router {} to handle floatingIp {}", floatingIpId, routerUuid, externalIp);
 156             return;
 157         }
 158         Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, subnetId);
 159         final String vpnName = externalSubnet.isPresent() ? subnetId.getValue() :
 160             NatUtil.getAssociatedVPN(dataBroker, networkId);
 161         final String subnetVpnName = externalSubnet.isPresent() ? subnetId.getValue() : null;
 162         if (vpnName == null) {
 163             LOG.error("onAddFloatingIp: No VPN is associated with ext nw {} to handle add floating ip {} configuration "
 164                     + "for router {}", networkId, externalIp, routerId);
 165             return;
 166         }
 167         if (rd == null) {
 168             LOG.error("onAddFloatingIp: Unable to retrieve external (internet) VPN RD from external VPN {} for "
 169                     + "router {} to handle floatingIp {}", vpnName, routerId, externalIp);
 170             return;
 171         }
 172         ProviderTypes provType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerUuid, networkId);
 173         if (provType == null) {
 174             return;
 175         }
 176         /*
 177          *  For external network of type GRE, it is required to use "Internet VPN VNI" for intra-DC
 178          *  communication, but we still require "MPLS labels" to reach SNAT/DNAT VMs from external
 179          *  entities via MPLSOverGRE.
 180          *
 181          *  MPLSOverGRE based external networks, the ``opendaylight-vni-ranges`` pool will be
 182          *  used to carve out a unique VNI per Internet VPN (GRE-provider-type) to be used in the
 183          *  datapath for traffic forwarding for ``SNAT-to-DNAT`` and ``DNAT-to-DNAT`` cases within the
 184          *  DataCenter.
 185         */
 186         String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
 187         LOG.debug("onAddFloatingIp: Nexthop ip for prefix {} is {}", externalIp, nextHopIp);
 188         if (provType == ProviderTypes.VXLAN) {
 189             Uuid floatingIpInterface = NatEvpnUtil.getFloatingIpInterfaceIdFromFloatingIpId(dataBroker, floatingIpId);
 190             evpnDnatFlowProgrammer.onAddFloatingIp(dpnId, routerUuid, routerId, vpnName, internalIp,
 191                     externalIp, networkId, interfaceName, floatingIpInterface.getValue(), floatingIpPortMacAddress,
 192                     rd, nextHopIp, confTx);
 193             return;
 194         }
 195         /*
 196          *  MPLS label will be used to advertise prefixes and in "L3_LFIB_TABLE" (table 20) taking the packet
 197          *  to "INBOUND_NAPT_TABLE" (table 44) and "PDNAT_TABLE" (table 25).
 198          */
 199         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
 200             .setIpPrefix(externalIp).build();
 201         ListenableFuture<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
 202
 203         ListenableFuture<RpcResult<CreateFibEntryOutput>> future = Futures.transformAsync(labelFuture, result -> {
 204             if (result.isSuccessful()) {
 205                 GenerateVpnLabelOutput output = result.getResult();
 206                 Uint32 label = output.getLabel();
 207                 LOG.debug("onAddFloatingIp : Generated label {} for prefix {}", label, externalIp);
 208                 FloatingIPListener.updateOperationalDS(dataBroker, routerUuid, interfaceName, label,
 209                         internalIp, externalIp);
 210                 /*
 211                  * For external network of type VXLAN all packets going from VMs within the DC, towards the
 212                  * external gateway device via the External VXLAN Tunnel,we are setting the VXLAN Tunnel ID to
 213                  * the L3VNI value of VPNInstance to which the VM belongs to.
 214                  */
 215                 Uint32 l3vni = Uint32.ZERO;
 216                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanService, provType)) {
 217                     l3vni = natOverVxlanUtil.getInternetVpnVni(vpnName, l3vni);
 218                 }
 219                 String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
 220                 //Inform BGP
 221                 NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd,
 222                     fibExternalIp, nextHopIp, networkId.getValue(), floatingIpPortMacAddress,
 223                         label, l3vni, RouteOrigin.STATIC, dpnId);
 224
 225                 List<Instruction> instructions = new ArrayList<>();
 226                 List<ActionInfo> actionsInfos = new ArrayList<>();
 227                 actionsInfos.add(new ActionNxResubmit(NwConstants.PDNAT_TABLE));
 228                 instructions.add(new InstructionApplyActions(actionsInfos).buildInstruction(0));
 229
 230                 List<ActionInfo> actionInfoFib = new ArrayList<>();
 231                 List<Instruction> customInstructions = new ArrayList<>();
 232                 actionInfoFib.add(new ActionSetFieldEthernetDestination(new MacAddress(floatingIpPortMacAddress)));
 233                 customInstructions.add(new InstructionApplyActions(actionInfoFib).buildInstruction(0));
 234                 customInstructions.add(new InstructionGotoTable(NwConstants.PDNAT_TABLE).buildInstruction(1));
 235
 236                 ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 237                     innerConfTx -> {
 238                         makeTunnelTableEntry(vpnName, dpnId, label, instructions, innerConfTx, provType);
 239                         makeLFibTableEntry(dpnId, label, floatingIpPortMacAddress, NwConstants.PDNAT_TABLE,
 240                             innerConfTx);
 241                     }), LOG, "Error adding tunnel or FIB table entries");
 242
 243                 CreateFibEntryInput input = new CreateFibEntryInputBuilder().setVpnName(vpnName)
 244                         .setSourceDpid(dpnId).setInstruction(customInstructions)
 245                         .setIpAddress(fibExternalIp).setServiceId(label)
 246                         .setIpAddressSource(CreateFibEntryInput.IpAddressSource.FloatingIP)
 247                         .setInstruction(customInstructions).build();
 248                 //Future<RpcResult<java.lang.Void>> createFibEntry(CreateFibEntryInput input);
 249                 ListenableFuture<RpcResult<CreateFibEntryOutput>> future1 = fibService.createFibEntry(input);
 250                 LOG.debug("onAddFloatingIp : Add Floating Ip {} , found associated to fixed port {}",
 251                         externalIp, interfaceName);
 252                 String networkVpnName =  NatUtil.getAssociatedVPN(dataBroker, networkId);
 253                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 254                     vpnManager.addSubnetMacIntoVpnInstance(networkVpnName, subnetVpnName,
 255                             floatingIpPortMacAddress, dpnId, tx);
 256                     vpnManager.addArpResponderFlowsToExternalNetworkIps(routerUuid,
 257                             Collections.singleton(externalIp),
 258                             floatingIpPortMacAddress, dpnId, networkId);
 259                 });
 260                 return future1;
 261             } else {
 262                 String errMsg = String.format("onAddFloatingIp : Could not retrieve the label for prefix %s "
 263                         + "in VPN %s, %s", externalIp, vpnName, result.getErrors());
 264                 LOG.error(errMsg);
 265                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
 266             }
 267         }, MoreExecutors.directExecutor());
 268
 269         Futures.addCallback(future, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
 270
 271             @Override
 272             public void onFailure(@NonNull Throwable error) {
 273                 LOG.error("onAddFloatingIp : Error in generate label or fib install process", error);
 274             }
 275
 276             @Override
 277             public void onSuccess(@NonNull RpcResult<CreateFibEntryOutput> result) {
 278                 if (result.isSuccessful()) {
 279                     LOG.info("onAddFloatingIp : Successfully installed custom FIB routes for prefix {}", externalIp);
 280                 } else {
 281                     LOG.error("onAddFloatingIp : Error in rpc call to create custom Fib entries for prefix {} "
 282                             + "in DPN {}, {}", externalIp, dpnId, result.getErrors());
 283                 }
 284             }
 285         }, MoreExecutors.directExecutor());
 286
 287         // Handle GARP transmission
 288         final IpAddress extrenalAddress = IpAddressBuilder.getDefaultInstance(externalIp);
 289         sendGarpOnInterface(dpnId, networkId, extrenalAddress, floatingIpPortMacAddress);
 290
 291     }
 292
 293     @Override
 294     public void onRemoveFloatingIp(final Uint64 dpnId, String routerUuid, Uint32 routerId, final Uuid networkId,
 295                                    InternalToExternalPortMap mapping, final Uint32 label, final String vrfId,
 296                                    TypedReadWriteTransaction<Configuration> confTx) {
 297         String externalIp = mapping.getExternalIp();
 298         Uuid floatingIpId = mapping.getExternalId();
 299         Uuid subnetId = NatUtil.getFloatingIpPortSubnetIdFromFloatingIpId(confTx, floatingIpId);
 300         Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(confTx, subnetId);
 301         final String vpnName = externalSubnet.isPresent() ? subnetId.getValue() :
 302             NatUtil.getAssociatedVPN(dataBroker, networkId);
 303         if (vpnName == null) {
 304             LOG.error("onRemoveFloatingIp: No VPN associated with ext nw {} to remove floating ip {} configuration "
 305                     + "for router {}", networkId, externalIp, routerUuid);
 306             return;
 307         }
 308
 309         //Remove floating mac from mymac table
 310         LOG.debug("onRemoveFloatingIp: Removing FloatingIp {}", externalIp);
 311         String floatingIpPortMacAddress = NatUtil.getFloatingIpPortMacFromFloatingIpId(confTx, floatingIpId);
 312         if (floatingIpPortMacAddress == null) {
 313             LOG.error("onRemoveFloatingIp: Unable to retrieve floatingIp port MAC address from floatingIpId {} for "
 314                     + "router {} to remove floatingIp {}", floatingIpId, routerUuid, externalIp);
 315             return;
 316         }
 317
 318         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 319             String networkVpnName =  NatUtil.getAssociatedVPN(tx, networkId);
 320             vpnManager.removeSubnetMacFromVpnInstance(networkVpnName, subnetId.getValue(), floatingIpPortMacAddress,
 321                     dpnId, tx);
 322             vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerUuid, Collections.singletonList(externalIp),
 323                     floatingIpPortMacAddress, dpnId, networkId);
 324         }), LOG, "onRemoveFloatingIp");
 325
 326         removeFromFloatingIpPortInfo(floatingIpId);
 327         ProviderTypes provType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerUuid, networkId);
 328         if (provType == null) {
 329             return;
 330         }
 331         if (provType == ProviderTypes.VXLAN) {
 332             Uuid floatingIpInterface = NatEvpnUtil.getFloatingIpInterfaceIdFromFloatingIpId(dataBroker, floatingIpId);
 333             evpnDnatFlowProgrammer.onRemoveFloatingIp(dpnId, vpnName, externalIp, floatingIpInterface.getValue(),
 334                     floatingIpPortMacAddress, routerId);
 335             return;
 336         }
 337         cleanupFibEntries(dpnId, vpnName, externalIp, label, vrfId, confTx, provType);
 338     }
 339
 340     @Override
 341     public void cleanupFibEntries(Uint64 dpnId, String vpnName, String externalIp,
 342                                   Uint32 label, final String rd, TypedReadWriteTransaction<Configuration> confTx,
 343                                   ProviderTypes provType) {
 344         //Remove Prefix from BGP
 345         String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
 346         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, fibExternalIp, vpnName);
 347         NatUtil.deletePrefixToInterface(dataBroker, NatUtil.getVpnId(dataBroker, vpnName), fibExternalIp);
 348         //Remove custom FIB routes
 349
 350         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
 351         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(vpnName)
 352             .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
 353             .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.FloatingIP).build();
 354         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
 355
 356         ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture = Futures.transformAsync(future, result -> {
 357             //Release label
 358             if (result.isSuccessful()) {
 359                 /*  check if any floating IP information is available in vpn-to-dpn-list for given dpn id. If exist any
 360                  *  floating IP then do not remove INTERNAL_TUNNEL_TABLE (table=36) -> PDNAT_TABLE (table=25) flow entry
 361                  */
 362                 Boolean removeTunnelFlow = Boolean.TRUE;
 363                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanService, provType)) {
 364                     if (NatUtil.isFloatingIpPresentForDpn(dataBroker, dpnId, rd, vpnName, externalIp,
 365                             false)) {
 366                         removeTunnelFlow = Boolean.FALSE;
 367                     }
 368                 }
 369                 if (removeTunnelFlow) {
 370                     removeTunnelTableEntry(dpnId, label, confTx);
 371                 }
 372                 removeLFibTableEntry(dpnId, label, confTx);
 373                 RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
 374                         .setVpnName(vpnName).setIpPrefix(externalIp).build();
 375                 Future<RpcResult<RemoveVpnLabelOutput>> labelFuture1 = vpnService.removeVpnLabel(labelInput);
 376                 if (labelFuture1.get() == null || !labelFuture1.get().isSuccessful()) {
 377                     String errMsg = String.format(
 378                             "VpnFloatingIpHandler: RPC call to remove VPN label on dpn %s "
 379                                     + "for prefix %s failed for vpn %s - %s",
 380                             dpnId, externalIp, vpnName, result.getErrors());
 381                     LOG.error(errMsg);
 382                     return Futures.immediateFailedFuture(new RuntimeException(errMsg));
 383                 }
 384                 return JdkFutureAdapters.listenInPoolThread(labelFuture1);
 385             } else {
 386                 String errMsg = String.format("onRemoveFloatingIp :RPC call to remove custom FIB entries "
 387                         + "on dpn %s for prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
 388                 LOG.error(errMsg);
 389                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
 390             }
 391         }, MoreExecutors.directExecutor());
 392
 393         Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
 394
 395             @Override
 396             public void onFailure(@NonNull Throwable error) {
 397                 LOG.error("onRemoveFloatingIp : Error in removing the label or custom fib entries", error);
 398             }
 399
 400             @Override
 401             public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
 402                 if (result.isSuccessful()) {
 403                     LOG.debug("onRemoveFloatingIp : Successfully removed the label for the prefix {} from VPN {}",
 404                             externalIp, vpnName);
 405                 } else {
 406                     LOG.error("onRemoveFloatingIp : Error in removing the label for prefix {} from VPN {}, {}",
 407                         externalIp, vpnName, result.getErrors());
 408                 }
 409             }
 410         }, MoreExecutors.directExecutor());
 411     }
 412
 413     private String getFlowRef(Uint64 dpnId, short tableId, Uint32 id, String ipAddress) {
 414         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR + id
 415                 + NwConstants.FLOWID_SEPARATOR + ipAddress;
 416     }
 417
 418     private void removeTunnelTableEntry(Uint64 dpnId, Uint32 serviceId,
 419             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 420
 421         LOG.debug("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
 422         mdsalManager.removeFlow(confTx, dpnId,
 423             new FlowKey(new FlowId(getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""))),
 424             NwConstants.INTERNAL_TUNNEL_TABLE);
 425         LOG.debug("removeTunnelTableEntry : Terminating service Entry for dpID {} : label : {} removed successfully",
 426                 dpnId, serviceId);
 427     }
 428
 429     private void makeTunnelTableEntry(String vpnName, Uint64 dpnId, Uint32 serviceId,
 430             List<Instruction> customInstructions, TypedWriteTransaction<Configuration> confTx, ProviderTypes provType) {
 431         List<MatchInfo> mkMatches = new ArrayList<>();
 432
 433         LOG.info("makeTunnelTableEntry on DpnId = {} and serviceId = {}", dpnId, serviceId);
 434         int flowPriority = NatConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY;
 435         // Increased the 36->25 flow priority. If SNAT is also configured on the same
 436         // DPN, then the traffic will be hijacked to DNAT and if there are no DNAT match,
 437         // then handled back to using using flow 25->44(which will be installed as part of SNAT)
 438         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanService, provType)) {
 439             mkMatches.add(new MatchTunnelId(Uint64.valueOf(
 440                     natOverVxlanUtil.getInternetVpnVni(vpnName, serviceId).longValue())));
 441             flowPriority = NatConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY + 1;
 442         } else {
 443             mkMatches.add(new MatchTunnelId(Uint64.valueOf(serviceId)));
 444         }
 445
 446         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
 447             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), flowPriority,
 448             String.format("%s:%s", "TST Flow Entry ", serviceId),
 449             0, 0, Uint64.valueOf(COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId.longValue()))),
 450                 mkMatches, customInstructions);
 451
 452         mdsalManager.addFlow(confTx, dpnId, terminatingServiceTableFlowEntity);
 453     }
 454
 455     private void makeLFibTableEntry(Uint64 dpId, Uint32 serviceId, String floatingIpPortMacAddress, short tableId,
 456                                     TypedWriteTransaction<Configuration> confTx) {
 457         List<MatchInfo> matches = new ArrayList<>();
 458         matches.add(MatchEthernetType.MPLS_UNICAST);
 459         matches.add(new MatchMplsLabel(serviceId.longValue()));
 460
 461         List<Instruction> instructions = new ArrayList<>();
 462         List<ActionInfo> actionsInfos = new ArrayList<>();
 463         //NAT is required for IPv4 only. Hence always etherType will be IPv4
 464         actionsInfos.add(new ActionPopMpls(NwConstants.ETHTYPE_IPV4));
 465         actionsInfos.add(new ActionSetFieldEthernetDestination(new MacAddress(floatingIpPortMacAddress)));
 466         Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
 467         instructions.add(writeInstruction);
 468         instructions.add(new InstructionGotoTable(tableId).buildInstruction(1));
 469
 470         // Install the flow entry in L3_LFIB_TABLE
 471         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
 472
 473         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
 474             10, flowRef, 0, 0,
 475             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
 476
 477         mdsalManager.addFlow(confTx, dpId, flowEntity);
 478
 479         LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
 480     }
 481
 482     private void removeLFibTableEntry(Uint64 dpnId, Uint32 serviceId,
 483             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 484
 485         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
 486
 487         LOG.debug("removeLFibTableEntry : removing LFib entry with flow ref {}", flowRef);
 488
 489         mdsalManager.removeFlow(confTx, dpnId, new FlowKey(new FlowId(flowRef)), NwConstants.L3_LFIB_TABLE);
 490
 491         LOG.debug("removeLFibTableEntry : LFIB Entry for dpID : {} label : {} removed successfully",
 492                 dpnId, serviceId);
 493     }
 494
 495     // TODO Clean up the exception handling
 496     @SuppressWarnings("checkstyle:IllegalCatch")
 497     private void sendGarpOnInterface(final Uint64 dpnId, Uuid networkId, final IpAddress floatingIpAddress,
 498                                      String floatingIpPortMacAddress) {
 499         if (floatingIpAddress.getIpv4Address() == null) {
 500             LOG.error("sendGarpOnInterface : Failed to send GARP for IP. recieved IPv6.");
 501             natServiceCounters.garpFailedIpv6();
 502             return;
 503         }
 504
 505         String interfaceName = elanService.getExternalElanInterface(networkId.getValue(), dpnId);
 506         if (interfaceName == null) {
 507             LOG.warn("sendGarpOnInterface : Failed to send GARP for IP. Failed to retrieve interface name "
 508                     + "from network {} and dpn id {}.", networkId.getValue(), dpnId);
 509             natServiceCounters.garpFailedMissingInterface();
 510             return;
 511         }
 512
 513         try {
 514             // find the external network interface name for dpn
 515             List<InterfaceAddress> interfaceAddresses = new ArrayList<>();
 516             interfaceAddresses.add(new InterfaceAddressBuilder()
 517                 .setInterface(interfaceName)
 518                 .setIpAddress(floatingIpAddress)
 519                 .setMacaddress(new PhysAddress(floatingIpPortMacAddress)).build());
 520
 521             SendArpRequestInput sendArpRequestInput = new SendArpRequestInputBuilder().setIpaddress(floatingIpAddress)
 522                 .setInterfaceAddress(interfaceAddresses).build();
 523
 524             JdkFutures.addErrorLogging(arpUtilService.sendArpRequest(sendArpRequestInput), LOG, "sendArpRequest");
 525             natServiceCounters.garpSent();
 526         } catch (Exception e) {
 527             LOG.error("sendGarpOnInterface : Failed to send GARP request for floating ip {} from interface {}",
 528                 floatingIpAddress.getIpv4Address().getValue(), interfaceName, e);
 529             natServiceCounters.garpFailedSend();
 530         }
 531     }
 532
 533     // TODO Clean up the exception handling
 534     @SuppressWarnings("checkstyle:IllegalCatch")
 535     private void removeFromFloatingIpPortInfo(Uuid floatingIpId) {
 536         InstanceIdentifier id = buildfloatingIpIdToPortMappingIdentifier(floatingIpId);
 537         try {
 538             Optional<FloatingIpIdToPortMapping> optFloatingIpIdToPortMapping =
 539                     SingleTransactionDataBroker.syncReadOptionalAndTreatReadFailedExceptionAsAbsentOptional(dataBroker,
 540                             LogicalDatastoreType.CONFIGURATION, id);
 541             if (optFloatingIpIdToPortMapping.isPresent() && optFloatingIpIdToPortMapping.get().isFloatingIpDeleted()) {
 542                 LOG.debug("Deleting floating IP UUID {} to Floating IP neutron port mapping from Floating "
 543                     + "IP Port Info Config DS", floatingIpId.getValue());
 544                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
 545             }
 546         } catch (Exception e) {
 547             LOG.error("removeFromFloatingIpPortInfo : Deleting floating IP UUID {} to Floating IP neutron port "
 548                     + "mapping from Floating IP Port Info Config DS failed", floatingIpId.getValue(), e);
 549         }
 550     }
 551 }