natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/VpnFloatingIpHandler.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  11 import static org.opendaylight.netvirt.natservice.internal.NatUtil.buildfloatingIpIdToPortMappingIdentifier;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import java.math.BigInteger;
  19 import java.util.ArrayList;
  20 import java.util.Collections;
  21 import java.util.List;
  22 import java.util.concurrent.ExecutionException;
  23 import javax.inject.Inject;
  24 import javax.inject.Singleton;
  25 import org.eclipse.jdt.annotation.NonNull;
  26 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  27 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  28 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  29 import org.opendaylight.genius.infra.Datastore.Configuration;
  30 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  31 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  32 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  33 import org.opendaylight.genius.infra.TypedWriteTransaction;
  34 import org.opendaylight.genius.mdsalutil.ActionInfo;
  35 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  36 import org.opendaylight.genius.mdsalutil.MatchInfo;
  37 import org.opendaylight.genius.mdsalutil.NwConstants;
  38 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  39 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  40 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  41 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  42 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  43 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  44 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  45 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  46 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  47 import org.opendaylight.infrautils.utils.concurrent.JdkFutures;
  48 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  49 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  50 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  51 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  52 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  53 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  54 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  55 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
  56 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  57 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  58 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.OdlArputilService;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpRequestInput;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpRequestInputBuilder;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.interfaces.InterfaceAddress;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.interfaces.InterfaceAddressBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.info.router.ports.ports.InternalToExternalPortMap;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.port.info.FloatingIpIdToPortMapping;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelOutput;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
  86 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  87 import org.opendaylight.yangtools.yang.common.RpcResult;
  88 import org.slf4j.Logger;
  89 import org.slf4j.LoggerFactory;
  90
  91 @Singleton
  92 public class VpnFloatingIpHandler implements FloatingIPHandler {
  93     private static final Logger LOG = LoggerFactory.getLogger(VpnFloatingIpHandler.class);
  94     private static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
  95     private static final String FLOWID_PREFIX = "NAT.";
  96
  97     private final DataBroker dataBroker;
  98     private final ManagedNewTransactionRunner txRunner;
  99     private final IMdsalApiManager mdsalManager;
 100     private final VpnRpcService vpnService;
 101     private final IBgpManager bgpManager;
 102     private final FibRpcService fibService;
 103     private final IVpnManager vpnManager;
 104     private final IFibManager fibManager;
 105     private final OdlArputilService arpUtilService;
 106     private final IElanService elanService;
 107     private final EvpnDnatFlowProgrammer evpnDnatFlowProgrammer;
 108     private final NatServiceCounters natServiceCounters;
 109     private final NatOverVxlanUtil natOverVxlanUtil;
 110
 111     @Inject
 112     public VpnFloatingIpHandler(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 113                                 final VpnRpcService vpnService,
 114                                 final IBgpManager bgpManager,
 115                                 final FibRpcService fibService,
 116                                 final IFibManager fibManager,
 117                                 final OdlArputilService arputilService,
 118                                 final IVpnManager vpnManager,
 119                                 final IElanService elanService,
 120                                 final EvpnDnatFlowProgrammer evpnDnatFlowProgrammer,
 121                                 final NatOverVxlanUtil natOverVxlanUtil,
 122                                 NatServiceCounters natServiceCounters) {
 123         this.dataBroker = dataBroker;
 124         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 125         this.mdsalManager = mdsalManager;
 126         this.vpnService = vpnService;
 127         this.bgpManager = bgpManager;
 128         this.fibService = fibService;
 129         this.fibManager = fibManager;
 130         this.arpUtilService = arputilService;
 131         this.vpnManager = vpnManager;
 132         this.elanService = elanService;
 133         this.evpnDnatFlowProgrammer = evpnDnatFlowProgrammer;
 134         this.natServiceCounters = natServiceCounters;
 135         this.natOverVxlanUtil = natOverVxlanUtil;
 136     }
 137
 138     @Override
 139     public void onAddFloatingIp(final BigInteger dpnId, final String routerUuid, final long routerId,
 140                                 final Uuid networkId, final String interfaceName,
 141                                 final InternalToExternalPortMap mapping,
 142                                 TypedReadWriteTransaction<Configuration> confTx) {
 143         String externalIp = mapping.getExternalIp();
 144         String internalIp = mapping.getInternalIp();
 145         Uuid floatingIpId = mapping.getExternalId();
 146         Uuid subnetId = NatUtil.getFloatingIpPortSubnetIdFromFloatingIpId(dataBroker, floatingIpId);
 147         String floatingIpPortMacAddress = NatUtil.getFloatingIpPortMacFromFloatingIpId(dataBroker, floatingIpId);
 148         if (floatingIpPortMacAddress == null) {
 149             LOG.error("onAddFloatingIp: Unable to retrieve floatingIp port MAC address from floatingIpId {} for "
 150                     + "router {} to handle floatingIp {}", floatingIpId, routerUuid, externalIp);
 151             return;
 152         }
 153         Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, subnetId);
 154         final String vpnName = externalSubnet.isPresent() ? subnetId.getValue() :
 155             NatUtil.getAssociatedVPN(dataBroker, networkId);
 156         final String subnetVpnName = externalSubnet.isPresent() ? subnetId.getValue() : null;
 157         if (vpnName == null) {
 158             LOG.error("onAddFloatingIp: No VPN is associated with ext nw {} to handle add floating ip {} configuration "
 159                     + "for router {}", networkId, externalIp, routerId);
 160             return;
 161         }
 162         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
 163         if (rd == null) {
 164             LOG.error("onAddFloatingIp: Unable to retrieve external (internet) VPN RD from external VPN {} for "
 165                     + "router {} to handle floatingIp {}", vpnName, routerId, externalIp);
 166             return;
 167         }
 168         ProviderTypes provType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerUuid, networkId);
 169         if (provType == null) {
 170             return;
 171         }
 172         /*
 173          *  For external network of type GRE, it is required to use "Internet VPN VNI" for intra-DC
 174          *  communication, but we still require "MPLS labels" to reach SNAT/DNAT VMs from external
 175          *  entities via MPLSOverGRE.
 176          *
 177          *  MPLSOverGRE based external networks, the ``opendaylight-vni-ranges`` pool will be
 178          *  used to carve out a unique VNI per Internet VPN (GRE-provider-type) to be used in the
 179          *  datapath for traffic forwarding for ``SNAT-to-DNAT`` and ``DNAT-to-DNAT`` cases within the
 180          *  DataCenter.
 181         */
 182         String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
 183         LOG.debug("onAddFloatingIp: Nexthop ip for prefix {} is {}", externalIp, nextHopIp);
 184         if (provType == ProviderTypes.VXLAN) {
 185             Uuid floatingIpInterface = NatEvpnUtil.getFloatingIpInterfaceIdFromFloatingIpId(dataBroker, floatingIpId);
 186             evpnDnatFlowProgrammer.onAddFloatingIp(dpnId, routerUuid, routerId, vpnName, internalIp,
 187                     externalIp, networkId, interfaceName, floatingIpInterface.getValue(), floatingIpPortMacAddress,
 188                     rd, nextHopIp, confTx);
 189             return;
 190         }
 191         /*
 192          *  MPLS label will be used to advertise prefixes and in "L3_LFIB_TABLE" (table 20) taking the packet
 193          *  to "INBOUND_NAPT_TABLE" (table 44) and "PDNAT_TABLE" (table 25).
 194          */
 195         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
 196             .setIpPrefix(externalIp).build();
 197         ListenableFuture<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
 198
 199         ListenableFuture<RpcResult<CreateFibEntryOutput>> future = Futures.transformAsync(labelFuture, result -> {
 200             if (result.isSuccessful()) {
 201                 GenerateVpnLabelOutput output = result.getResult();
 202                 long label = output.getLabel();
 203                 LOG.debug("onAddFloatingIp : Generated label {} for prefix {}", label, externalIp);
 204                 FloatingIPListener.updateOperationalDS(dataBroker, routerUuid, interfaceName, label,
 205                         internalIp, externalIp);
 206                 /*
 207                  * For external network of type VXLAN all packets going from VMs within the DC, towards the
 208                  * external gateway device via the External VXLAN Tunnel,we are setting the VXLAN Tunnel ID to
 209                  * the L3VNI value of VPNInstance to which the VM belongs to.
 210                  */
 211                 long l3vni = 0;
 212                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanService, provType)) {
 213                     l3vni = natOverVxlanUtil.getInternetVpnVni(vpnName, l3vni).longValue();
 214                 }
 215                 String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
 216                 //Inform BGP
 217                 NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd,
 218                     fibExternalIp, nextHopIp, networkId.getValue(), floatingIpPortMacAddress,
 219                         label, l3vni, RouteOrigin.STATIC, dpnId);
 220
 221                 List<Instruction> instructions = new ArrayList<>();
 222                 List<ActionInfo> actionsInfos = new ArrayList<>();
 223                 actionsInfos.add(new ActionNxResubmit(NwConstants.PDNAT_TABLE));
 224                 instructions.add(new InstructionApplyActions(actionsInfos).buildInstruction(0));
 225
 226                 List<ActionInfo> actionInfoFib = new ArrayList<>();
 227                 List<Instruction> customInstructions = new ArrayList<>();
 228                 actionInfoFib.add(new ActionSetFieldEthernetDestination(new MacAddress(floatingIpPortMacAddress)));
 229                 customInstructions.add(new InstructionApplyActions(actionInfoFib).buildInstruction(0));
 230                 customInstructions.add(new InstructionGotoTable(NwConstants.PDNAT_TABLE).buildInstruction(1));
 231
 232                 ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 233                     innerConfTx -> {
 234                         makeTunnelTableEntry(vpnName, dpnId, label, instructions, innerConfTx, provType);
 235                         makeLFibTableEntry(dpnId, label, floatingIpPortMacAddress, NwConstants.PDNAT_TABLE,
 236                             innerConfTx);
 237                     }), LOG, "Error adding tunnel or FIB table entries");
 238
 239                 CreateFibEntryInput input = new CreateFibEntryInputBuilder().setVpnName(vpnName)
 240                         .setSourceDpid(dpnId).setInstruction(customInstructions)
 241                         .setIpAddress(fibExternalIp).setServiceId(label)
 242                         .setIpAddressSource(CreateFibEntryInput.IpAddressSource.FloatingIP)
 243                         .setInstruction(customInstructions).build();
 244                 //Future<RpcResult<java.lang.Void>> createFibEntry(CreateFibEntryInput input);
 245                 ListenableFuture<RpcResult<CreateFibEntryOutput>> future1 = fibService.createFibEntry(input);
 246                 LOG.debug("onAddFloatingIp : Add Floating Ip {} , found associated to fixed port {}",
 247                         externalIp, interfaceName);
 248                 String networkVpnName =  NatUtil.getAssociatedVPN(dataBroker, networkId);
 249                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 250                     vpnManager.addSubnetMacIntoVpnInstance(networkVpnName, subnetVpnName,
 251                             floatingIpPortMacAddress, dpnId, tx);
 252                     vpnManager.addArpResponderFlowsToExternalNetworkIps(routerUuid,
 253                             Collections.singleton(externalIp),
 254                             floatingIpPortMacAddress, dpnId, networkId);
 255                 });
 256                 return future1;
 257             } else {
 258                 String errMsg = String.format("onAddFloatingIp : Could not retrieve the label for prefix %s "
 259                         + "in VPN %s, %s", externalIp, vpnName, result.getErrors());
 260                 LOG.error(errMsg);
 261                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
 262             }
 263         }, MoreExecutors.directExecutor());
 264
 265         Futures.addCallback(future, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
 266
 267             @Override
 268             public void onFailure(@NonNull Throwable error) {
 269                 LOG.error("onAddFloatingIp : Error in generate label or fib install process", error);
 270             }
 271
 272             @Override
 273             public void onSuccess(@NonNull RpcResult<CreateFibEntryOutput> result) {
 274                 if (result.isSuccessful()) {
 275                     LOG.info("onAddFloatingIp : Successfully installed custom FIB routes for prefix {}", externalIp);
 276                 } else {
 277                     LOG.error("onAddFloatingIp : Error in rpc call to create custom Fib entries for prefix {} "
 278                             + "in DPN {}, {}", externalIp, dpnId, result.getErrors());
 279                 }
 280             }
 281         }, MoreExecutors.directExecutor());
 282
 283         // Handle GARP transmission
 284         final IpAddress extrenalAddress = IpAddressBuilder.getDefaultInstance(externalIp);
 285         sendGarpOnInterface(dpnId, networkId, extrenalAddress, floatingIpPortMacAddress);
 286
 287     }
 288
 289     @Override
 290     public void onRemoveFloatingIp(final BigInteger dpnId, String routerUuid, long routerId, final Uuid networkId,
 291                                    InternalToExternalPortMap mapping, final long label,
 292                                    TypedReadWriteTransaction<Configuration> confTx) {
 293         String externalIp = mapping.getExternalIp();
 294         Uuid floatingIpId = mapping.getExternalId();
 295         Uuid subnetId = NatUtil.getFloatingIpPortSubnetIdFromFloatingIpId(confTx, floatingIpId);
 296         Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(confTx, subnetId);
 297         final String vpnName = externalSubnet.isPresent() ? subnetId.getValue() :
 298             NatUtil.getAssociatedVPN(dataBroker, networkId);
 299         if (vpnName == null) {
 300             LOG.error("onRemoveFloatingIp: No VPN associated with ext nw {} to remove floating ip {} configuration "
 301                     + "for router {}", networkId, externalIp, routerUuid);
 302             return;
 303         }
 304
 305         //Remove floating mac from mymac table
 306         LOG.debug("onRemoveFloatingIp: Removing FloatingIp {}", externalIp);
 307         String floatingIpPortMacAddress = NatUtil.getFloatingIpPortMacFromFloatingIpId(confTx, floatingIpId);
 308         if (floatingIpPortMacAddress == null) {
 309             LOG.error("onRemoveFloatingIp: Unable to retrieve floatingIp port MAC address from floatingIpId {} for "
 310                     + "router {} to remove floatingIp {}", floatingIpId, routerUuid, externalIp);
 311             return;
 312         }
 313
 314         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 315             String networkVpnName =  NatUtil.getAssociatedVPN(tx, networkId);
 316             vpnManager.removeSubnetMacFromVpnInstance(networkVpnName, subnetId.getValue(), floatingIpPortMacAddress,
 317                     dpnId, tx);
 318             vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerUuid, Collections.singletonList(externalIp),
 319                     floatingIpPortMacAddress, dpnId, networkId);
 320         }), LOG, "onRemoveFloatingIp");
 321
 322         removeFromFloatingIpPortInfo(floatingIpId);
 323         ProviderTypes provType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerUuid, networkId);
 324         if (provType == null) {
 325             return;
 326         }
 327         if (provType == ProviderTypes.VXLAN) {
 328             Uuid floatingIpInterface = NatEvpnUtil.getFloatingIpInterfaceIdFromFloatingIpId(dataBroker, floatingIpId);
 329             evpnDnatFlowProgrammer.onRemoveFloatingIp(dpnId, vpnName, externalIp, floatingIpInterface.getValue(),
 330                     floatingIpPortMacAddress, routerId);
 331             return;
 332         }
 333         cleanupFibEntries(dpnId, vpnName, externalIp, label, confTx, provType);
 334     }
 335
 336     @Override
 337     public void cleanupFibEntries(BigInteger dpnId, String vpnName, String externalIp,
 338                                   long label, TypedReadWriteTransaction<Configuration> confTx, ProviderTypes provType) {
 339         //Remove Prefix from BGP
 340         String rd = NatUtil.getVpnRd(confTx, vpnName);
 341         String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
 342         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, fibExternalIp, vpnName, LOG);
 343
 344         //Remove custom FIB routes
 345
 346         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
 347         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(vpnName)
 348             .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
 349             .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.FloatingIP).build();
 350         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
 351
 352         ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture = Futures.transformAsync(future, result -> {
 353             //Release label
 354             if (result.isSuccessful()) {
 355                 /*  check if any floating IP information is available in vpn-to-dpn-list for given dpn id. If exist any
 356                  *  floating IP then do not remove INTERNAL_TUNNEL_TABLE (table=36) -> PDNAT_TABLE (table=25) flow entry
 357                  */
 358                 Boolean removeTunnelFlow = Boolean.TRUE;
 359                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanService, provType)) {
 360                     if (NatUtil.isFloatingIpPresentForDpn(dataBroker, dpnId, rd, vpnName, externalIp, false)) {
 361                         removeTunnelFlow = Boolean.FALSE;
 362                     }
 363                 }
 364                 if (removeTunnelFlow) {
 365                     removeTunnelTableEntry(dpnId, label, confTx);
 366                 }
 367                 removeLFibTableEntry(dpnId, label, confTx);
 368                 RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
 369                         .setVpnName(vpnName).setIpPrefix(externalIp).build();
 370                 return vpnService.removeVpnLabel(labelInput);
 371             } else {
 372                 String errMsg = String.format("onRemoveFloatingIp :RPC call to remove custom FIB entries "
 373                         + "on dpn %s for prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
 374                 LOG.error(errMsg);
 375                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
 376             }
 377         }, MoreExecutors.directExecutor());
 378
 379         Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
 380
 381             @Override
 382             public void onFailure(@NonNull Throwable error) {
 383                 LOG.error("onRemoveFloatingIp : Error in removing the label or custom fib entries", error);
 384             }
 385
 386             @Override
 387             public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
 388                 if (result.isSuccessful()) {
 389                     LOG.debug("onRemoveFloatingIp : Successfully removed the label for the prefix {} from VPN {}",
 390                             externalIp, vpnName);
 391                 } else {
 392                     LOG.error("onRemoveFloatingIp : Error in removing the label for prefix {} from VPN {}, {}",
 393                         externalIp, vpnName, result.getErrors());
 394                 }
 395             }
 396         }, MoreExecutors.directExecutor());
 397     }
 398
 399     private String getFlowRef(BigInteger dpnId, short tableId, long id, String ipAddress) {
 400         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR + id
 401                 + NwConstants.FLOWID_SEPARATOR + ipAddress;
 402     }
 403
 404     private void removeTunnelTableEntry(BigInteger dpnId, long serviceId,
 405             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 406
 407         LOG.debug("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
 408         mdsalManager.removeFlow(confTx, dpnId,
 409             new FlowKey(new FlowId(getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""))),
 410             NwConstants.INTERNAL_TUNNEL_TABLE);
 411         LOG.debug("removeTunnelTableEntry : Terminating service Entry for dpID {} : label : {} removed successfully",
 412                 dpnId, serviceId);
 413     }
 414
 415     private void makeTunnelTableEntry(String vpnName, BigInteger dpnId, long serviceId,
 416             List<Instruction> customInstructions, TypedWriteTransaction<Configuration> confTx, ProviderTypes provType) {
 417         List<MatchInfo> mkMatches = new ArrayList<>();
 418
 419         LOG.info("makeTunnelTableEntry on DpnId = {} and serviceId = {}", dpnId, serviceId);
 420         int flowPriority = 5;
 421         // Increased the 36->25 flow priority. If SNAT is also configured on the same
 422         // DPN, then the traffic will be hijacked to DNAT and if there are no DNAT match,
 423         // then handled back to using using flow 25->44(which will be installed as part of SNAT)
 424         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanService, provType)) {
 425             mkMatches.add(new MatchTunnelId(natOverVxlanUtil.getInternetVpnVni(vpnName, serviceId)));
 426             flowPriority = 6;
 427         } else {
 428             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(serviceId)));
 429         }
 430
 431         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
 432             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), flowPriority,
 433             String.format("%s:%d", "TST Flow Entry ", serviceId),
 434             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId)), mkMatches, customInstructions);
 435
 436         mdsalManager.addFlow(confTx, dpnId, terminatingServiceTableFlowEntity);
 437     }
 438
 439     private void makeLFibTableEntry(BigInteger dpId, long serviceId, String floatingIpPortMacAddress, short tableId,
 440                                     TypedWriteTransaction<Configuration> confTx) {
 441         List<MatchInfo> matches = new ArrayList<>();
 442         matches.add(MatchEthernetType.MPLS_UNICAST);
 443         matches.add(new MatchMplsLabel(serviceId));
 444
 445         List<Instruction> instructions = new ArrayList<>();
 446         List<ActionInfo> actionsInfos = new ArrayList<>();
 447         //NAT is required for IPv4 only. Hence always etherType will be IPv4
 448         actionsInfos.add(new ActionPopMpls(NwConstants.ETHTYPE_IPV4));
 449         actionsInfos.add(new ActionSetFieldEthernetDestination(new MacAddress(floatingIpPortMacAddress)));
 450         Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
 451         instructions.add(writeInstruction);
 452         instructions.add(new InstructionGotoTable(tableId).buildInstruction(1));
 453
 454         // Install the flow entry in L3_LFIB_TABLE
 455         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
 456
 457         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
 458             10, flowRef, 0, 0,
 459             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
 460
 461         mdsalManager.addFlow(confTx, dpId, flowEntity);
 462
 463         LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
 464     }
 465
 466     private void removeLFibTableEntry(BigInteger dpnId, long serviceId,
 467             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 468
 469         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
 470
 471         LOG.debug("removeLFibTableEntry : removing LFib entry with flow ref {}", flowRef);
 472
 473         mdsalManager.removeFlow(confTx, dpnId, new FlowKey(new FlowId(flowRef)), NwConstants.L3_LFIB_TABLE);
 474
 475         LOG.debug("removeLFibTableEntry : LFIB Entry for dpID : {} label : {} removed successfully",
 476                 dpnId, serviceId);
 477     }
 478
 479     // TODO Clean up the exception handling
 480     @SuppressWarnings("checkstyle:IllegalCatch")
 481     private void sendGarpOnInterface(final BigInteger dpnId, Uuid networkId, final IpAddress floatingIpAddress,
 482                                      String floatingIpPortMacAddress) {
 483         if (floatingIpAddress.getIpv4Address() == null) {
 484             LOG.error("sendGarpOnInterface : Failed to send GARP for IP. recieved IPv6.");
 485             natServiceCounters.garpFailedIpv6();
 486             return;
 487         }
 488
 489         String interfaceName = elanService.getExternalElanInterface(networkId.getValue(), dpnId);
 490         if (interfaceName == null) {
 491             LOG.warn("sendGarpOnInterface : Failed to send GARP for IP. Failed to retrieve interface name "
 492                     + "from network {} and dpn id {}.", networkId.getValue(), dpnId);
 493             natServiceCounters.garpFailedMissingInterface();
 494             return;
 495         }
 496
 497         try {
 498             // find the external network interface name for dpn
 499             List<InterfaceAddress> interfaceAddresses = new ArrayList<>();
 500             interfaceAddresses.add(new InterfaceAddressBuilder()
 501                 .setInterface(interfaceName)
 502                 .setIpAddress(floatingIpAddress)
 503                 .setMacaddress(new PhysAddress(floatingIpPortMacAddress)).build());
 504
 505             SendArpRequestInput sendArpRequestInput = new SendArpRequestInputBuilder().setIpaddress(floatingIpAddress)
 506                 .setInterfaceAddress(interfaceAddresses).build();
 507
 508             JdkFutures.addErrorLogging(arpUtilService.sendArpRequest(sendArpRequestInput), LOG, "sendArpRequest");
 509             natServiceCounters.garpSent();
 510         } catch (Exception e) {
 511             LOG.error("sendGarpOnInterface : Failed to send GARP request for floating ip {} from interface {}",
 512                 floatingIpAddress.getIpv4Address().getValue(), interfaceName, e);
 513             natServiceCounters.garpFailedSend();
 514         }
 515     }
 516
 517     // TODO Clean up the exception handling
 518     @SuppressWarnings("checkstyle:IllegalCatch")
 519     private void removeFromFloatingIpPortInfo(Uuid floatingIpId) {
 520         InstanceIdentifier id = buildfloatingIpIdToPortMappingIdentifier(floatingIpId);
 521         try {
 522             Optional<FloatingIpIdToPortMapping> optFloatingIpIdToPortMapping =
 523                     SingleTransactionDataBroker.syncReadOptionalAndTreatReadFailedExceptionAsAbsentOptional(dataBroker,
 524                             LogicalDatastoreType.CONFIGURATION, id);
 525             if (optFloatingIpIdToPortMapping.isPresent() && optFloatingIpIdToPortMapping.get().isFloatingIpDeleted()) {
 526                 LOG.debug("Deleting floating IP UUID {} to Floating IP neutron port mapping from Floating "
 527                     + "IP Port Info Config DS", floatingIpId.getValue());
 528                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
 529             }
 530         } catch (Exception e) {
 531             LOG.error("removeFromFloatingIpPortInfo : Deleting floating IP UUID {} to Floating IP neutron port "
 532                     + "mapping from Floating IP Port Info Config DS failed", floatingIpId.getValue(), e);
 533         }
 534     }
 535
 536 }