natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/VpnFloatingIpHandler.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  11 import static org.opendaylight.netvirt.natservice.internal.NatUtil.buildfloatingIpIdToPortMappingIdentifier;
  12
  13 import com.google.common.util.concurrent.FutureCallback;
  14 import com.google.common.util.concurrent.Futures;
  15 import com.google.common.util.concurrent.JdkFutureAdapters;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import java.math.BigInteger;
  19 import java.util.ArrayList;
  20 import java.util.Collections;
  21 import java.util.List;
  22 import java.util.Optional;
  23 import java.util.concurrent.ExecutionException;
  24 import java.util.concurrent.Future;
  25 import javax.inject.Inject;
  26 import javax.inject.Singleton;
  27 import org.eclipse.jdt.annotation.NonNull;
  28 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  29 import org.opendaylight.genius.infra.Datastore.Configuration;
  30 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  31 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  32 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  33 import org.opendaylight.genius.infra.TypedWriteTransaction;
  34 import org.opendaylight.genius.mdsalutil.ActionInfo;
  35 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  36 import org.opendaylight.genius.mdsalutil.MatchInfo;
  37 import org.opendaylight.genius.mdsalutil.NwConstants;
  38 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  39 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  40 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  41 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  42 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  43 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  44 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  45 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  46 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  47 import org.opendaylight.infrautils.utils.concurrent.JdkFutures;
  48 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  49 import org.opendaylight.mdsal.binding.api.DataBroker;
  50 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  51 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  52 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  53 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  54 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  55 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  56 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  57 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
  58 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  59 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  60 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.OdlArputilService;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpRequestInput;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpRequestInputBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.interfaces.InterfaceAddress;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.interfaces.InterfaceAddressBuilder;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.info.router.ports.ports.InternalToExternalPortMap;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.port.info.FloatingIpIdToPortMapping;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelOutput;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
  88 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  89 import org.opendaylight.yangtools.yang.common.RpcResult;
  90 import org.opendaylight.yangtools.yang.common.Uint32;
  91 import org.opendaylight.yangtools.yang.common.Uint64;
  92 import org.slf4j.Logger;
  93 import org.slf4j.LoggerFactory;
  94
  95 @Singleton
  96 public class VpnFloatingIpHandler implements FloatingIPHandler {
  97     private static final Logger LOG = LoggerFactory.getLogger(VpnFloatingIpHandler.class);
  98     private static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
  99     private static final String FLOWID_PREFIX = "NAT.";
 100
 101     private final DataBroker dataBroker;
 102     private final ManagedNewTransactionRunner txRunner;
 103     private final IMdsalApiManager mdsalManager;
 104     private final VpnRpcService vpnService;
 105     private final IBgpManager bgpManager;
 106     private final FibRpcService fibService;
 107     private final IVpnManager vpnManager;
 108     private final IFibManager fibManager;
 109     private final OdlArputilService arpUtilService;
 110     private final IElanService elanService;
 111     private final EvpnDnatFlowProgrammer evpnDnatFlowProgrammer;
 112     private final NatServiceCounters natServiceCounters;
 113     private final NatOverVxlanUtil natOverVxlanUtil;
 114
 115     @Inject
 116     public VpnFloatingIpHandler(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 117                                 final VpnRpcService vpnService,
 118                                 final IBgpManager bgpManager,
 119                                 final FibRpcService fibService,
 120                                 final IFibManager fibManager,
 121                                 final OdlArputilService arputilService,
 122                                 final IVpnManager vpnManager,
 123                                 final IElanService elanService,
 124                                 final EvpnDnatFlowProgrammer evpnDnatFlowProgrammer,
 125                                 final NatOverVxlanUtil natOverVxlanUtil,
 126                                 NatServiceCounters natServiceCounters) {
 127         this.dataBroker = dataBroker;
 128         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 129         this.mdsalManager = mdsalManager;
 130         this.vpnService = vpnService;
 131         this.bgpManager = bgpManager;
 132         this.fibService = fibService;
 133         this.fibManager = fibManager;
 134         this.arpUtilService = arputilService;
 135         this.vpnManager = vpnManager;
 136         this.elanService = elanService;
 137         this.evpnDnatFlowProgrammer = evpnDnatFlowProgrammer;
 138         this.natServiceCounters = natServiceCounters;
 139         this.natOverVxlanUtil = natOverVxlanUtil;
 140     }
 141
 142     @Override
 143     public void onAddFloatingIp(final Uint64 dpnId, final String routerUuid, final Uint32 routerId,
 144                                 final Uuid networkId, final String interfaceName,
 145                                 final InternalToExternalPortMap mapping, final String rd,
 146                                 TypedReadWriteTransaction<Configuration> confTx) {
 147         String externalIp = mapping.getExternalIp();
 148         String internalIp = mapping.getInternalIp();
 149         Uuid floatingIpId = mapping.getExternalId();
 150         Uuid subnetId = NatUtil.getFloatingIpPortSubnetIdFromFloatingIpId(dataBroker, floatingIpId);
 151         String floatingIpPortMacAddress = NatUtil.getFloatingIpPortMacFromFloatingIpId(dataBroker, floatingIpId);
 152         if (floatingIpPortMacAddress == null) {
 153             LOG.error("onAddFloatingIp: Unable to retrieve floatingIp port MAC address from floatingIpId {} for "
 154                     + "router {} to handle floatingIp {}", floatingIpId, routerUuid, externalIp);
 155             return;
 156         }
 157         Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, subnetId);
 158         final String vpnName = externalSubnet.isPresent() ? subnetId.getValue() :
 159             NatUtil.getAssociatedVPN(dataBroker, networkId);
 160         final String subnetVpnName = externalSubnet.isPresent() ? subnetId.getValue() : null;
 161         if (vpnName == null) {
 162             LOG.error("onAddFloatingIp: No VPN is associated with ext nw {} to handle add floating ip {} configuration "
 163                     + "for router {}", networkId, externalIp, routerId);
 164             return;
 165         }
 166         if (rd == null) {
 167             LOG.error("onAddFloatingIp: Unable to retrieve external (internet) VPN RD from external VPN {} for "
 168                     + "router {} to handle floatingIp {}", vpnName, routerId, externalIp);
 169             return;
 170         }
 171         ProviderTypes provType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerUuid, networkId);
 172         if (provType == null) {
 173             return;
 174         }
 175         /*
 176          *  For external network of type GRE, it is required to use "Internet VPN VNI" for intra-DC
 177          *  communication, but we still require "MPLS labels" to reach SNAT/DNAT VMs from external
 178          *  entities via MPLSOverGRE.
 179          *
 180          *  MPLSOverGRE based external networks, the ``opendaylight-vni-ranges`` pool will be
 181          *  used to carve out a unique VNI per Internet VPN (GRE-provider-type) to be used in the
 182          *  datapath for traffic forwarding for ``SNAT-to-DNAT`` and ``DNAT-to-DNAT`` cases within the
 183          *  DataCenter.
 184         */
 185         String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
 186         LOG.debug("onAddFloatingIp: Nexthop ip for prefix {} is {}", externalIp, nextHopIp);
 187         if (provType == ProviderTypes.VXLAN) {
 188             Uuid floatingIpInterface = NatEvpnUtil.getFloatingIpInterfaceIdFromFloatingIpId(dataBroker, floatingIpId);
 189             evpnDnatFlowProgrammer.onAddFloatingIp(dpnId, routerUuid, routerId, vpnName, internalIp,
 190                     externalIp, networkId, interfaceName, floatingIpInterface.getValue(), floatingIpPortMacAddress,
 191                     rd, nextHopIp, confTx);
 192             return;
 193         }
 194         /*
 195          *  MPLS label will be used to advertise prefixes and in "L3_LFIB_TABLE" (table 20) taking the packet
 196          *  to "INBOUND_NAPT_TABLE" (table 44) and "PDNAT_TABLE" (table 25).
 197          */
 198         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
 199             .setIpPrefix(externalIp).build();
 200         ListenableFuture<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
 201
 202         ListenableFuture<RpcResult<CreateFibEntryOutput>> future = Futures.transformAsync(labelFuture, result -> {
 203             if (result.isSuccessful()) {
 204                 GenerateVpnLabelOutput output = result.getResult();
 205                 Uint32 label = output.getLabel();
 206                 LOG.debug("onAddFloatingIp : Generated label {} for prefix {}", label, externalIp);
 207                 FloatingIPListener.updateOperationalDS(dataBroker, routerUuid, interfaceName, label,
 208                         internalIp, externalIp);
 209                 /*
 210                  * For external network of type VXLAN all packets going from VMs within the DC, towards the
 211                  * external gateway device via the External VXLAN Tunnel,we are setting the VXLAN Tunnel ID to
 212                  * the L3VNI value of VPNInstance to which the VM belongs to.
 213                  */
 214                 Uint32 l3vni = Uint32.ZERO;
 215                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanService, provType)) {
 216                     l3vni = natOverVxlanUtil.getInternetVpnVni(vpnName, l3vni);
 217                 }
 218                 String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
 219                 //Inform BGP
 220                 NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd,
 221                     fibExternalIp, nextHopIp, networkId.getValue(), floatingIpPortMacAddress,
 222                         label, l3vni, RouteOrigin.STATIC, dpnId);
 223
 224                 List<Instruction> instructions = new ArrayList<>();
 225                 List<ActionInfo> actionsInfos = new ArrayList<>();
 226                 actionsInfos.add(new ActionNxResubmit(NwConstants.PDNAT_TABLE));
 227                 instructions.add(new InstructionApplyActions(actionsInfos).buildInstruction(0));
 228
 229                 List<ActionInfo> actionInfoFib = new ArrayList<>();
 230                 List<Instruction> customInstructions = new ArrayList<>();
 231                 actionInfoFib.add(new ActionSetFieldEthernetDestination(new MacAddress(floatingIpPortMacAddress)));
 232                 customInstructions.add(new InstructionApplyActions(actionInfoFib).buildInstruction(0));
 233                 customInstructions.add(new InstructionGotoTable(NwConstants.PDNAT_TABLE).buildInstruction(1));
 234
 235                 ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 236                     innerConfTx -> {
 237                         makeTunnelTableEntry(vpnName, dpnId, label, instructions, innerConfTx, provType);
 238                         makeLFibTableEntry(dpnId, label, floatingIpPortMacAddress, NwConstants.PDNAT_TABLE,
 239                             innerConfTx);
 240                     }), LOG, "Error adding tunnel or FIB table entries");
 241
 242                 CreateFibEntryInput input = new CreateFibEntryInputBuilder().setVpnName(vpnName)
 243                         .setSourceDpid(dpnId).setInstruction(customInstructions)
 244                         .setIpAddress(fibExternalIp).setServiceId(label)
 245                         .setIpAddressSource(CreateFibEntryInput.IpAddressSource.FloatingIP)
 246                         .setInstruction(customInstructions).build();
 247                 //Future<RpcResult<java.lang.Void>> createFibEntry(CreateFibEntryInput input);
 248                 ListenableFuture<RpcResult<CreateFibEntryOutput>> future1 = fibService.createFibEntry(input);
 249                 LOG.debug("onAddFloatingIp : Add Floating Ip {} , found associated to fixed port {}",
 250                         externalIp, interfaceName);
 251                 String networkVpnName =  NatUtil.getAssociatedVPN(dataBroker, networkId);
 252                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 253                     vpnManager.addSubnetMacIntoVpnInstance(networkVpnName, subnetVpnName,
 254                             floatingIpPortMacAddress, dpnId, tx);
 255                     vpnManager.addArpResponderFlowsToExternalNetworkIps(routerUuid,
 256                             Collections.singleton(externalIp),
 257                             floatingIpPortMacAddress, dpnId, networkId);
 258                 });
 259                 return future1;
 260             } else {
 261                 String errMsg = String.format("onAddFloatingIp : Could not retrieve the label for prefix %s "
 262                         + "in VPN %s, %s", externalIp, vpnName, result.getErrors());
 263                 LOG.error(errMsg);
 264                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
 265             }
 266         }, MoreExecutors.directExecutor());
 267
 268         Futures.addCallback(future, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
 269
 270             @Override
 271             public void onFailure(@NonNull Throwable error) {
 272                 LOG.error("onAddFloatingIp : Error in generate label or fib install process", error);
 273             }
 274
 275             @Override
 276             public void onSuccess(@NonNull RpcResult<CreateFibEntryOutput> result) {
 277                 if (result.isSuccessful()) {
 278                     LOG.info("onAddFloatingIp : Successfully installed custom FIB routes for prefix {}", externalIp);
 279                 } else {
 280                     LOG.error("onAddFloatingIp : Error in rpc call to create custom Fib entries for prefix {} "
 281                             + "in DPN {}, {}", externalIp, dpnId, result.getErrors());
 282                 }
 283             }
 284         }, MoreExecutors.directExecutor());
 285
 286         // Handle GARP transmission
 287         final IpAddress extrenalAddress = IpAddressBuilder.getDefaultInstance(externalIp);
 288         sendGarpOnInterface(dpnId, networkId, extrenalAddress, floatingIpPortMacAddress);
 289
 290     }
 291
 292     @Override
 293     public void onRemoveFloatingIp(final Uint64 dpnId, String routerUuid, Uint32 routerId, final Uuid networkId,
 294                                    InternalToExternalPortMap mapping, final Uint32 label, final String vrfId,
 295                                    TypedReadWriteTransaction<Configuration> confTx) {
 296         String externalIp = mapping.getExternalIp();
 297         Uuid floatingIpId = mapping.getExternalId();
 298         Uuid subnetId = NatUtil.getFloatingIpPortSubnetIdFromFloatingIpId(confTx, floatingIpId);
 299         Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(confTx, subnetId);
 300         final String vpnName = externalSubnet.isPresent() ? subnetId.getValue() :
 301             NatUtil.getAssociatedVPN(dataBroker, networkId);
 302         if (vpnName == null) {
 303             LOG.error("onRemoveFloatingIp: No VPN associated with ext nw {} to remove floating ip {} configuration "
 304                     + "for router {}", networkId, externalIp, routerUuid);
 305             return;
 306         }
 307
 308         //Remove floating mac from mymac table
 309         LOG.debug("onRemoveFloatingIp: Removing FloatingIp {}", externalIp);
 310         String floatingIpPortMacAddress = NatUtil.getFloatingIpPortMacFromFloatingIpId(confTx, floatingIpId);
 311         if (floatingIpPortMacAddress == null) {
 312             LOG.error("onRemoveFloatingIp: Unable to retrieve floatingIp port MAC address from floatingIpId {} for "
 313                     + "router {} to remove floatingIp {}", floatingIpId, routerUuid, externalIp);
 314             return;
 315         }
 316
 317         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 318             String networkVpnName =  NatUtil.getAssociatedVPN(tx, networkId);
 319             vpnManager.removeSubnetMacFromVpnInstance(networkVpnName, subnetId.getValue(), floatingIpPortMacAddress,
 320                     dpnId, tx);
 321             vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerUuid, Collections.singletonList(externalIp),
 322                     floatingIpPortMacAddress, dpnId, networkId);
 323         }), LOG, "onRemoveFloatingIp");
 324
 325         removeFromFloatingIpPortInfo(floatingIpId);
 326         ProviderTypes provType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerUuid, networkId);
 327         if (provType == null) {
 328             return;
 329         }
 330         if (provType == ProviderTypes.VXLAN) {
 331             Uuid floatingIpInterface = NatEvpnUtil.getFloatingIpInterfaceIdFromFloatingIpId(dataBroker, floatingIpId);
 332             evpnDnatFlowProgrammer.onRemoveFloatingIp(dpnId, vpnName, externalIp, floatingIpInterface.getValue(),
 333                     floatingIpPortMacAddress, routerId);
 334             return;
 335         }
 336         cleanupFibEntries(dpnId, vpnName, externalIp, label, vrfId, confTx, provType);
 337     }
 338
 339     @Override
 340     public void cleanupFibEntries(Uint64 dpnId, String vpnName, String externalIp,
 341                                   Uint32 label, final String rd, TypedReadWriteTransaction<Configuration> confTx,
 342                                   ProviderTypes provType) {
 343         //Remove Prefix from BGP
 344         String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
 345         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, fibExternalIp, vpnName);
 346         NatUtil.deletePrefixToInterface(dataBroker, NatUtil.getVpnId(dataBroker, vpnName), fibExternalIp);
 347         //Remove custom FIB routes
 348
 349         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
 350         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(vpnName)
 351             .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
 352             .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.FloatingIP).build();
 353         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
 354
 355         ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture = Futures.transformAsync(future, result -> {
 356             //Release label
 357             if (result.isSuccessful()) {
 358                 /*  check if any floating IP information is available in vpn-to-dpn-list for given dpn id. If exist any
 359                  *  floating IP then do not remove INTERNAL_TUNNEL_TABLE (table=36) -> PDNAT_TABLE (table=25) flow entry
 360                  */
 361                 Boolean removeTunnelFlow = Boolean.TRUE;
 362                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanService, provType)) {
 363                     if (NatUtil.isFloatingIpPresentForDpn(dataBroker, dpnId, rd, vpnName, externalIp,
 364                             false)) {
 365                         removeTunnelFlow = Boolean.FALSE;
 366                     }
 367                 }
 368                 if (removeTunnelFlow) {
 369                     removeTunnelTableEntry(dpnId, label, confTx);
 370                 }
 371                 removeLFibTableEntry(dpnId, label, confTx);
 372                 RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
 373                         .setVpnName(vpnName).setIpPrefix(externalIp).build();
 374                 Future<RpcResult<RemoveVpnLabelOutput>> labelFuture1 = vpnService.removeVpnLabel(labelInput);
 375                 if (labelFuture1.get() == null || !labelFuture1.get().isSuccessful()) {
 376                     String errMsg = String.format(
 377                             "VpnFloatingIpHandler: RPC call to remove VPN label on dpn %s "
 378                                     + "for prefix %s failed for vpn %s - %s",
 379                             dpnId, externalIp, vpnName, result.getErrors());
 380                     LOG.error(errMsg);
 381                     return Futures.immediateFailedFuture(new RuntimeException(errMsg));
 382                 }
 383                 return JdkFutureAdapters.listenInPoolThread(labelFuture1);
 384             } else {
 385                 String errMsg = String.format("onRemoveFloatingIp :RPC call to remove custom FIB entries "
 386                         + "on dpn %s for prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
 387                 LOG.error(errMsg);
 388                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
 389             }
 390         }, MoreExecutors.directExecutor());
 391
 392         Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
 393
 394             @Override
 395             public void onFailure(@NonNull Throwable error) {
 396                 LOG.error("onRemoveFloatingIp : Error in removing the label or custom fib entries", error);
 397             }
 398
 399             @Override
 400             public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
 401                 if (result.isSuccessful()) {
 402                     LOG.debug("onRemoveFloatingIp : Successfully removed the label for the prefix {} from VPN {}",
 403                             externalIp, vpnName);
 404                 } else {
 405                     LOG.error("onRemoveFloatingIp : Error in removing the label for prefix {} from VPN {}, {}",
 406                         externalIp, vpnName, result.getErrors());
 407                 }
 408             }
 409         }, MoreExecutors.directExecutor());
 410     }
 411
 412     private String getFlowRef(Uint64 dpnId, short tableId, Uint32 id, String ipAddress) {
 413         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR + id
 414                 + NwConstants.FLOWID_SEPARATOR + ipAddress;
 415     }
 416
 417     private void removeTunnelTableEntry(Uint64 dpnId, Uint32 serviceId,
 418             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 419
 420         LOG.debug("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
 421         mdsalManager.removeFlow(confTx, dpnId,
 422             new FlowKey(new FlowId(getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""))),
 423             NwConstants.INTERNAL_TUNNEL_TABLE);
 424         LOG.debug("removeTunnelTableEntry : Terminating service Entry for dpID {} : label : {} removed successfully",
 425                 dpnId, serviceId);
 426     }
 427
 428     private void makeTunnelTableEntry(String vpnName, Uint64 dpnId, Uint32 serviceId,
 429             List<Instruction> customInstructions, TypedWriteTransaction<Configuration> confTx, ProviderTypes provType) {
 430         List<MatchInfo> mkMatches = new ArrayList<>();
 431
 432         LOG.info("makeTunnelTableEntry on DpnId = {} and serviceId = {}", dpnId, serviceId);
 433         int flowPriority = NatConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY;
 434         // Increased the 36->25 flow priority. If SNAT is also configured on the same
 435         // DPN, then the traffic will be hijacked to DNAT and if there are no DNAT match,
 436         // then handled back to using using flow 25->44(which will be installed as part of SNAT)
 437         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanService, provType)) {
 438             mkMatches.add(new MatchTunnelId(Uint64.valueOf(
 439                     natOverVxlanUtil.getInternetVpnVni(vpnName, serviceId).longValue())));
 440             flowPriority = NatConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY + 1;
 441         } else {
 442             mkMatches.add(new MatchTunnelId(Uint64.valueOf(serviceId)));
 443         }
 444
 445         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
 446             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), flowPriority,
 447             String.format("%s:%s", "TST Flow Entry ", serviceId),
 448             0, 0, Uint64.valueOf(COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId.longValue()))),
 449                 mkMatches, customInstructions);
 450
 451         mdsalManager.addFlow(confTx, dpnId, terminatingServiceTableFlowEntity);
 452     }
 453
 454     private void makeLFibTableEntry(Uint64 dpId, Uint32 serviceId, String floatingIpPortMacAddress, short tableId,
 455                                     TypedWriteTransaction<Configuration> confTx) {
 456         List<MatchInfo> matches = new ArrayList<>();
 457         matches.add(MatchEthernetType.MPLS_UNICAST);
 458         matches.add(new MatchMplsLabel(serviceId.longValue()));
 459
 460         List<Instruction> instructions = new ArrayList<>();
 461         List<ActionInfo> actionsInfos = new ArrayList<>();
 462         //NAT is required for IPv4 only. Hence always etherType will be IPv4
 463         actionsInfos.add(new ActionPopMpls(NwConstants.ETHTYPE_IPV4));
 464         actionsInfos.add(new ActionSetFieldEthernetDestination(new MacAddress(floatingIpPortMacAddress)));
 465         Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
 466         instructions.add(writeInstruction);
 467         instructions.add(new InstructionGotoTable(tableId).buildInstruction(1));
 468
 469         // Install the flow entry in L3_LFIB_TABLE
 470         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
 471
 472         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
 473             10, flowRef, 0, 0,
 474             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
 475
 476         mdsalManager.addFlow(confTx, dpId, flowEntity);
 477
 478         LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
 479     }
 480
 481     private void removeLFibTableEntry(Uint64 dpnId, Uint32 serviceId,
 482             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 483
 484         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
 485
 486         LOG.debug("removeLFibTableEntry : removing LFib entry with flow ref {}", flowRef);
 487
 488         mdsalManager.removeFlow(confTx, dpnId, new FlowKey(new FlowId(flowRef)), NwConstants.L3_LFIB_TABLE);
 489
 490         LOG.debug("removeLFibTableEntry : LFIB Entry for dpID : {} label : {} removed successfully",
 491                 dpnId, serviceId);
 492     }
 493
 494     // TODO Clean up the exception handling
 495     @SuppressWarnings("checkstyle:IllegalCatch")
 496     private void sendGarpOnInterface(final Uint64 dpnId, Uuid networkId, final IpAddress floatingIpAddress,
 497                                      String floatingIpPortMacAddress) {
 498         if (floatingIpAddress.getIpv4Address() == null) {
 499             LOG.error("sendGarpOnInterface : Failed to send GARP for IP. recieved IPv6.");
 500             natServiceCounters.garpFailedIpv6();
 501             return;
 502         }
 503
 504         String interfaceName = elanService.getExternalElanInterface(networkId.getValue(), dpnId);
 505         if (interfaceName == null) {
 506             LOG.warn("sendGarpOnInterface : Failed to send GARP for IP. Failed to retrieve interface name "
 507                     + "from network {} and dpn id {}.", networkId.getValue(), dpnId);
 508             natServiceCounters.garpFailedMissingInterface();
 509             return;
 510         }
 511
 512         try {
 513             // find the external network interface name for dpn
 514             List<InterfaceAddress> interfaceAddresses = new ArrayList<>();
 515             interfaceAddresses.add(new InterfaceAddressBuilder()
 516                 .setInterface(interfaceName)
 517                 .setIpAddress(floatingIpAddress)
 518                 .setMacaddress(new PhysAddress(floatingIpPortMacAddress)).build());
 519
 520             SendArpRequestInput sendArpRequestInput = new SendArpRequestInputBuilder().setIpaddress(floatingIpAddress)
 521                 .setInterfaceAddress(interfaceAddresses).build();
 522
 523             JdkFutures.addErrorLogging(arpUtilService.sendArpRequest(sendArpRequestInput), LOG, "sendArpRequest");
 524             natServiceCounters.garpSent();
 525         } catch (Exception e) {
 526             LOG.error("sendGarpOnInterface : Failed to send GARP request for floating ip {} from interface {}",
 527                 floatingIpAddress.getIpv4Address().getValue(), interfaceName, e);
 528             natServiceCounters.garpFailedSend();
 529         }
 530     }
 531
 532     // TODO Clean up the exception handling
 533     @SuppressWarnings("checkstyle:IllegalCatch")
 534     private void removeFromFloatingIpPortInfo(Uuid floatingIpId) {
 535         InstanceIdentifier id = buildfloatingIpIdToPortMappingIdentifier(floatingIpId);
 536         try {
 537             Optional<FloatingIpIdToPortMapping> optFloatingIpIdToPortMapping =
 538                     SingleTransactionDataBroker.syncReadOptionalAndTreatReadFailedExceptionAsAbsentOptional(dataBroker,
 539                             LogicalDatastoreType.CONFIGURATION, id);
 540             if (optFloatingIpIdToPortMapping.isPresent() && optFloatingIpIdToPortMapping.get().isFloatingIpDeleted()) {
 541                 LOG.debug("Deleting floating IP UUID {} to Floating IP neutron port mapping from Floating "
 542                     + "IP Port Info Config DS", floatingIpId.getValue());
 543                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
 544             }
 545         } catch (Exception e) {
 546             LOG.error("removeFromFloatingIpPortInfo : Deleting floating IP UUID {} to Floating IP neutron port "
 547                     + "mapping from Floating IP Port Info Config DS failed", floatingIpId.getValue(), e);
 548         }
 549     }
 550 }