natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/VpnFloatingIpHandler.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  11 import static org.opendaylight.netvirt.natservice.internal.NatUtil.buildfloatingIpIdToPortMappingIdentifier;
  12
  13 import com.google.common.util.concurrent.FutureCallback;
  14 import com.google.common.util.concurrent.Futures;
  15 import com.google.common.util.concurrent.JdkFutureAdapters;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import java.math.BigInteger;
  19 import java.util.ArrayList;
  20 import java.util.Collections;
  21 import java.util.HashMap;
  22 import java.util.List;
  23 import java.util.Map;
  24 import java.util.Optional;
  25 import java.util.concurrent.ExecutionException;
  26 import java.util.concurrent.Future;
  27 import javax.inject.Inject;
  28 import javax.inject.Singleton;
  29 import org.eclipse.jdt.annotation.NonNull;
  30 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  31 import org.opendaylight.genius.infra.Datastore.Configuration;
  32 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  33 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  34 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  35 import org.opendaylight.genius.infra.TypedWriteTransaction;
  36 import org.opendaylight.genius.mdsalutil.ActionInfo;
  37 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  38 import org.opendaylight.genius.mdsalutil.MatchInfo;
  39 import org.opendaylight.genius.mdsalutil.NwConstants;
  40 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  41 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  42 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  43 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  44 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  45 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  46 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  47 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  48 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  49 import org.opendaylight.infrautils.utils.concurrent.JdkFutures;
  50 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  51 import org.opendaylight.mdsal.binding.api.DataBroker;
  52 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  53 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  54 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  55 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  56 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  57 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  58 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  59 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
  60 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  61 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  62 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionKey;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.OdlArputilService;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpRequestInput;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpRequestInputBuilder;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.interfaces.InterfaceAddress;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.interfaces.InterfaceAddressBuilder;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.info.router.ports.ports.InternalToExternalPortMap;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.port.info.FloatingIpIdToPortMapping;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelOutput;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
  91 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  92 import org.opendaylight.yangtools.yang.common.RpcResult;
  93 import org.opendaylight.yangtools.yang.common.Uint32;
  94 import org.opendaylight.yangtools.yang.common.Uint64;
  95 import org.slf4j.Logger;
  96 import org.slf4j.LoggerFactory;
  97
  98 @Singleton
  99 public class VpnFloatingIpHandler implements FloatingIPHandler {
 100     private static final Logger LOG = LoggerFactory.getLogger(VpnFloatingIpHandler.class);
 101     private static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 102     private static final String FLOWID_PREFIX = "NAT.";
 103
 104     private final DataBroker dataBroker;
 105     private final ManagedNewTransactionRunner txRunner;
 106     private final IMdsalApiManager mdsalManager;
 107     private final VpnRpcService vpnService;
 108     private final IBgpManager bgpManager;
 109     private final FibRpcService fibService;
 110     private final IVpnManager vpnManager;
 111     private final IFibManager fibManager;
 112     private final OdlArputilService arpUtilService;
 113     private final IElanService elanService;
 114     private final EvpnDnatFlowProgrammer evpnDnatFlowProgrammer;
 115     private final NatServiceCounters natServiceCounters;
 116     private final NatOverVxlanUtil natOverVxlanUtil;
 117
 118     @Inject
 119     public VpnFloatingIpHandler(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 120                                 final VpnRpcService vpnService,
 121                                 final IBgpManager bgpManager,
 122                                 final FibRpcService fibService,
 123                                 final IFibManager fibManager,
 124                                 final OdlArputilService arputilService,
 125                                 final IVpnManager vpnManager,
 126                                 final IElanService elanService,
 127                                 final EvpnDnatFlowProgrammer evpnDnatFlowProgrammer,
 128                                 final NatOverVxlanUtil natOverVxlanUtil,
 129                                 NatServiceCounters natServiceCounters) {
 130         this.dataBroker = dataBroker;
 131         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 132         this.mdsalManager = mdsalManager;
 133         this.vpnService = vpnService;
 134         this.bgpManager = bgpManager;
 135         this.fibService = fibService;
 136         this.fibManager = fibManager;
 137         this.arpUtilService = arputilService;
 138         this.vpnManager = vpnManager;
 139         this.elanService = elanService;
 140         this.evpnDnatFlowProgrammer = evpnDnatFlowProgrammer;
 141         this.natServiceCounters = natServiceCounters;
 142         this.natOverVxlanUtil = natOverVxlanUtil;
 143     }
 144
 145     @Override
 146     public void onAddFloatingIp(final Uint64 dpnId, final String routerUuid, final Uint32 routerId,
 147                                 final Uuid networkId, final String interfaceName,
 148                                 final InternalToExternalPortMap mapping, final String rd,
 149                                 TypedReadWriteTransaction<Configuration> confTx) {
 150         String externalIp = mapping.getExternalIp();
 151         String internalIp = mapping.getInternalIp();
 152         Uuid floatingIpId = mapping.getExternalId();
 153         Uuid subnetId = NatUtil.getFloatingIpPortSubnetIdFromFloatingIpId(dataBroker, floatingIpId);
 154         String floatingIpPortMacAddress = NatUtil.getFloatingIpPortMacFromFloatingIpId(dataBroker, floatingIpId);
 155         if (floatingIpPortMacAddress == null) {
 156             LOG.error("onAddFloatingIp: Unable to retrieve floatingIp port MAC address from floatingIpId {} for "
 157                     + "router {} to handle floatingIp {}", floatingIpId, routerUuid, externalIp);
 158             return;
 159         }
 160         Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, subnetId);
 161         final String vpnName = externalSubnet.isPresent() ? subnetId.getValue() :
 162             NatUtil.getAssociatedVPN(dataBroker, networkId);
 163         final String subnetVpnName = externalSubnet.isPresent() ? subnetId.getValue() : null;
 164         if (vpnName == null) {
 165             LOG.error("onAddFloatingIp: No VPN is associated with ext nw {} to handle add floating ip {} configuration "
 166                     + "for router {}", networkId, externalIp, routerId);
 167             return;
 168         }
 169         if (rd == null) {
 170             LOG.error("onAddFloatingIp: Unable to retrieve external (internet) VPN RD from external VPN {} for "
 171                     + "router {} to handle floatingIp {}", vpnName, routerId, externalIp);
 172             return;
 173         }
 174         ProviderTypes provType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerUuid, networkId);
 175         if (provType == null) {
 176             return;
 177         }
 178         /*
 179          *  For external network of type GRE, it is required to use "Internet VPN VNI" for intra-DC
 180          *  communication, but we still require "MPLS labels" to reach SNAT/DNAT VMs from external
 181          *  entities via MPLSOverGRE.
 182          *
 183          *  MPLSOverGRE based external networks, the ``opendaylight-vni-ranges`` pool will be
 184          *  used to carve out a unique VNI per Internet VPN (GRE-provider-type) to be used in the
 185          *  datapath for traffic forwarding for ``SNAT-to-DNAT`` and ``DNAT-to-DNAT`` cases within the
 186          *  DataCenter.
 187         */
 188         String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
 189         LOG.debug("onAddFloatingIp: Nexthop ip for prefix {} is {}", externalIp, nextHopIp);
 190         if (provType == ProviderTypes.VXLAN) {
 191             Uuid floatingIpInterface = NatEvpnUtil.getFloatingIpInterfaceIdFromFloatingIpId(dataBroker, floatingIpId);
 192             evpnDnatFlowProgrammer.onAddFloatingIp(dpnId, routerUuid, routerId, vpnName, internalIp,
 193                     externalIp, networkId, interfaceName, floatingIpInterface.getValue(), floatingIpPortMacAddress,
 194                     rd, nextHopIp, confTx);
 195             return;
 196         }
 197         /*
 198          *  MPLS label will be used to advertise prefixes and in "L3_LFIB_TABLE" (table 20) taking the packet
 199          *  to "INBOUND_NAPT_TABLE" (table 44) and "PDNAT_TABLE" (table 25).
 200          */
 201         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
 202             .setIpPrefix(externalIp).build();
 203         ListenableFuture<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
 204
 205         ListenableFuture<RpcResult<CreateFibEntryOutput>> future = Futures.transformAsync(labelFuture, result -> {
 206             if (result.isSuccessful()) {
 207                 GenerateVpnLabelOutput output = result.getResult();
 208                 Uint32 label = output.getLabel();
 209                 LOG.debug("onAddFloatingIp : Generated label {} for prefix {}", label, externalIp);
 210                 FloatingIPListener.updateOperationalDS(dataBroker, routerUuid, interfaceName, label,
 211                         internalIp, externalIp);
 212                 /*
 213                  * For external network of type VXLAN all packets going from VMs within the DC, towards the
 214                  * external gateway device via the External VXLAN Tunnel,we are setting the VXLAN Tunnel ID to
 215                  * the L3VNI value of VPNInstance to which the VM belongs to.
 216                  */
 217                 Uint32 l3vni = Uint32.ZERO;
 218                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanService, provType)) {
 219                     l3vni = natOverVxlanUtil.getInternetVpnVni(vpnName, l3vni);
 220                 }
 221                 String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
 222                 //Inform BGP
 223                 NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd,
 224                     fibExternalIp, nextHopIp, networkId.getValue(), floatingIpPortMacAddress,
 225                         label, l3vni, RouteOrigin.STATIC, dpnId);
 226
 227                 List<Instruction> instructions = new ArrayList<>();
 228                 List<ActionInfo> actionsInfos = new ArrayList<>();
 229                 actionsInfos.add(new ActionNxResubmit(NwConstants.PDNAT_TABLE));
 230                 instructions.add(new InstructionApplyActions(actionsInfos).buildInstruction(0));
 231
 232                 List<ActionInfo> actionInfoFib = new ArrayList<>();
 233                 List<Instruction> customInstructions = new ArrayList<>();
 234                 actionInfoFib.add(new ActionSetFieldEthernetDestination(new MacAddress(floatingIpPortMacAddress)));
 235                 customInstructions.add(new InstructionApplyActions(actionInfoFib).buildInstruction(0));
 236                 customInstructions.add(new InstructionGotoTable(NwConstants.PDNAT_TABLE).buildInstruction(1));
 237
 238                 ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 239                     innerConfTx -> {
 240                         makeTunnelTableEntry(vpnName, dpnId, label, instructions, innerConfTx, provType);
 241                         makeLFibTableEntry(dpnId, label, floatingIpPortMacAddress, NwConstants.PDNAT_TABLE,
 242                             innerConfTx);
 243                     }), LOG, "Error adding tunnel or FIB table entries");
 244
 245                 CreateFibEntryInput input = new CreateFibEntryInputBuilder().setVpnName(vpnName)
 246                         .setSourceDpid(dpnId).setInstruction(customInstructions)
 247                         .setIpAddress(fibExternalIp).setServiceId(label)
 248                         .setIpAddressSource(CreateFibEntryInput.IpAddressSource.FloatingIP)
 249                         .setInstruction(customInstructions).build();
 250                 //Future<RpcResult<java.lang.Void>> createFibEntry(CreateFibEntryInput input);
 251                 ListenableFuture<RpcResult<CreateFibEntryOutput>> future1 = fibService.createFibEntry(input);
 252                 LOG.debug("onAddFloatingIp : Add Floating Ip {} , found associated to fixed port {}",
 253                         externalIp, interfaceName);
 254                 String networkVpnName =  NatUtil.getAssociatedVPN(dataBroker, networkId);
 255                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 256                     vpnManager.addSubnetMacIntoVpnInstance(networkVpnName, subnetVpnName,
 257                             floatingIpPortMacAddress, dpnId, tx);
 258                     vpnManager.addArpResponderFlowsToExternalNetworkIps(routerUuid,
 259                             Collections.singleton(externalIp),
 260                             floatingIpPortMacAddress, dpnId, networkId);
 261                 });
 262                 return future1;
 263             } else {
 264                 String errMsg = String.format("onAddFloatingIp : Could not retrieve the label for prefix %s "
 265                         + "in VPN %s, %s", externalIp, vpnName, result.getErrors());
 266                 LOG.error(errMsg);
 267                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
 268             }
 269         }, MoreExecutors.directExecutor());
 270
 271         Futures.addCallback(future, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
 272
 273             @Override
 274             public void onFailure(@NonNull Throwable error) {
 275                 LOG.error("onAddFloatingIp : Error in generate label or fib install process", error);
 276             }
 277
 278             @Override
 279             public void onSuccess(@NonNull RpcResult<CreateFibEntryOutput> result) {
 280                 if (result.isSuccessful()) {
 281                     LOG.info("onAddFloatingIp : Successfully installed custom FIB routes for prefix {}", externalIp);
 282                 } else {
 283                     LOG.error("onAddFloatingIp : Error in rpc call to create custom Fib entries for prefix {} "
 284                             + "in DPN {}, {}", externalIp, dpnId, result.getErrors());
 285                 }
 286             }
 287         }, MoreExecutors.directExecutor());
 288
 289         // Handle GARP transmission
 290         final IpAddress extrenalAddress = IpAddressBuilder.getDefaultInstance(externalIp);
 291         sendGarpOnInterface(dpnId, networkId, extrenalAddress, floatingIpPortMacAddress);
 292
 293     }
 294
 295     @Override
 296     public void onRemoveFloatingIp(final Uint64 dpnId, String routerUuid, Uint32 routerId, final Uuid networkId,
 297                                    InternalToExternalPortMap mapping, final Uint32 label, final String vrfId,
 298                                    TypedReadWriteTransaction<Configuration> confTx) {
 299         String externalIp = mapping.getExternalIp();
 300         Uuid floatingIpId = mapping.getExternalId();
 301         Uuid subnetId = NatUtil.getFloatingIpPortSubnetIdFromFloatingIpId(confTx, floatingIpId);
 302         Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(confTx, subnetId);
 303         final String vpnName = externalSubnet.isPresent() ? subnetId.getValue() :
 304             NatUtil.getAssociatedVPN(dataBroker, networkId);
 305         if (vpnName == null) {
 306             LOG.error("onRemoveFloatingIp: No VPN associated with ext nw {} to remove floating ip {} configuration "
 307                     + "for router {}", networkId, externalIp, routerUuid);
 308             return;
 309         }
 310
 311         //Remove floating mac from mymac table
 312         LOG.debug("onRemoveFloatingIp: Removing FloatingIp {}", externalIp);
 313         String floatingIpPortMacAddress = NatUtil.getFloatingIpPortMacFromFloatingIpId(confTx, floatingIpId);
 314         if (floatingIpPortMacAddress == null) {
 315             LOG.error("onRemoveFloatingIp: Unable to retrieve floatingIp port MAC address from floatingIpId {} for "
 316                     + "router {} to remove floatingIp {}", floatingIpId, routerUuid, externalIp);
 317             return;
 318         }
 319
 320         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 321             String networkVpnName =  NatUtil.getAssociatedVPN(tx, networkId);
 322             vpnManager.removeSubnetMacFromVpnInstance(networkVpnName, subnetId.getValue(), floatingIpPortMacAddress,
 323                     dpnId, tx);
 324             vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerUuid, Collections.singletonList(externalIp),
 325                     floatingIpPortMacAddress, dpnId, networkId);
 326         }), LOG, "onRemoveFloatingIp");
 327
 328         removeFromFloatingIpPortInfo(floatingIpId);
 329         ProviderTypes provType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerUuid, networkId);
 330         if (provType == null) {
 331             return;
 332         }
 333         if (provType == ProviderTypes.VXLAN) {
 334             Uuid floatingIpInterface = NatEvpnUtil.getFloatingIpInterfaceIdFromFloatingIpId(dataBroker, floatingIpId);
 335             evpnDnatFlowProgrammer.onRemoveFloatingIp(dpnId, vpnName, externalIp, floatingIpInterface.getValue(),
 336                     floatingIpPortMacAddress, routerId);
 337             return;
 338         }
 339         cleanupFibEntries(dpnId, vpnName, externalIp, label, vrfId, confTx, provType);
 340     }
 341
 342     @Override
 343     public void cleanupFibEntries(Uint64 dpnId, String vpnName, String externalIp,
 344                                   Uint32 label, final String rd, TypedReadWriteTransaction<Configuration> confTx,
 345                                   ProviderTypes provType) {
 346         //Remove Prefix from BGP
 347         String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
 348         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, fibExternalIp, vpnName);
 349         NatUtil.deletePrefixToInterface(dataBroker, NatUtil.getVpnId(dataBroker, vpnName), fibExternalIp);
 350         //Remove custom FIB routes
 351
 352         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
 353         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(vpnName)
 354             .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
 355             .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.FloatingIP).build();
 356         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
 357
 358         ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture = Futures.transformAsync(future, result -> {
 359             //Release label
 360             if (result.isSuccessful()) {
 361                 /*  check if any floating IP information is available in vpn-to-dpn-list for given dpn id. If exist any
 362                  *  floating IP then do not remove INTERNAL_TUNNEL_TABLE (table=36) -> PDNAT_TABLE (table=25) flow entry
 363                  */
 364                 Boolean removeTunnelFlow = Boolean.TRUE;
 365                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanService, provType)) {
 366                     if (NatUtil.isFloatingIpPresentForDpn(dataBroker, dpnId, rd, vpnName, externalIp,
 367                             false)) {
 368                         removeTunnelFlow = Boolean.FALSE;
 369                     }
 370                 }
 371                 if (removeTunnelFlow) {
 372                     removeTunnelTableEntry(dpnId, label, confTx);
 373                 }
 374                 removeLFibTableEntry(dpnId, label, confTx);
 375                 RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
 376                         .setVpnName(vpnName).setIpPrefix(externalIp).build();
 377                 Future<RpcResult<RemoveVpnLabelOutput>> labelFuture1 = vpnService.removeVpnLabel(labelInput);
 378                 if (labelFuture1.get() == null || !labelFuture1.get().isSuccessful()) {
 379                     String errMsg = String.format(
 380                             "VpnFloatingIpHandler: RPC call to remove VPN label on dpn %s "
 381                                     + "for prefix %s failed for vpn %s - %s",
 382                             dpnId, externalIp, vpnName, result.getErrors());
 383                     LOG.error(errMsg);
 384                     return Futures.immediateFailedFuture(new RuntimeException(errMsg));
 385                 }
 386                 return JdkFutureAdapters.listenInPoolThread(labelFuture1);
 387             } else {
 388                 String errMsg = String.format("onRemoveFloatingIp :RPC call to remove custom FIB entries "
 389                         + "on dpn %s for prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
 390                 LOG.error(errMsg);
 391                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
 392             }
 393         }, MoreExecutors.directExecutor());
 394
 395         Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
 396
 397             @Override
 398             public void onFailure(@NonNull Throwable error) {
 399                 LOG.error("onRemoveFloatingIp : Error in removing the label or custom fib entries", error);
 400             }
 401
 402             @Override
 403             public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
 404                 if (result.isSuccessful()) {
 405                     LOG.debug("onRemoveFloatingIp : Successfully removed the label for the prefix {} from VPN {}",
 406                             externalIp, vpnName);
 407                 } else {
 408                     LOG.error("onRemoveFloatingIp : Error in removing the label for prefix {} from VPN {}, {}",
 409                         externalIp, vpnName, result.getErrors());
 410                 }
 411             }
 412         }, MoreExecutors.directExecutor());
 413     }
 414
 415     private String getFlowRef(Uint64 dpnId, short tableId, Uint32 id, String ipAddress) {
 416         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR + id
 417                 + NwConstants.FLOWID_SEPARATOR + ipAddress;
 418     }
 419
 420     private void removeTunnelTableEntry(Uint64 dpnId, Uint32 serviceId,
 421             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 422
 423         LOG.debug("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
 424         mdsalManager.removeFlow(confTx, dpnId,
 425             new FlowKey(new FlowId(getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""))),
 426             NwConstants.INTERNAL_TUNNEL_TABLE);
 427         LOG.debug("removeTunnelTableEntry : Terminating service Entry for dpID {} : label : {} removed successfully",
 428                 dpnId, serviceId);
 429     }
 430
 431     private void makeTunnelTableEntry(String vpnName, Uint64 dpnId, Uint32 serviceId,
 432             List<Instruction> customInstructions, TypedWriteTransaction<Configuration> confTx, ProviderTypes provType) {
 433         List<MatchInfo> mkMatches = new ArrayList<>();
 434
 435         LOG.info("makeTunnelTableEntry on DpnId = {} and serviceId = {}", dpnId, serviceId);
 436         int flowPriority = NatConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY;
 437         // Increased the 36->25 flow priority. If SNAT is also configured on the same
 438         // DPN, then the traffic will be hijacked to DNAT and if there are no DNAT match,
 439         // then handled back to using using flow 25->44(which will be installed as part of SNAT)
 440         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanService, provType)) {
 441             mkMatches.add(new MatchTunnelId(Uint64.valueOf(
 442                     natOverVxlanUtil.getInternetVpnVni(vpnName, serviceId).longValue())));
 443             flowPriority = NatConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY + 1;
 444         } else {
 445             mkMatches.add(new MatchTunnelId(Uint64.valueOf(serviceId)));
 446         }
 447         Map<InstructionKey, Instruction> customInstructionsMap = new HashMap<InstructionKey, Instruction>();
 448         int instructionKey = 0;
 449         for (Instruction instructionObj : customInstructions) {
 450             customInstructionsMap.put(new InstructionKey(++instructionKey), instructionObj);
 451         }
 452         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
 453             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), flowPriority,
 454             String.format("%s:%s", "TST Flow Entry ", serviceId),
 455             0, 0, Uint64.valueOf(COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId.longValue()))),
 456                 mkMatches, customInstructionsMap);
 457
 458         mdsalManager.addFlow(confTx, dpnId, terminatingServiceTableFlowEntity);
 459     }
 460
 461     private void makeLFibTableEntry(Uint64 dpId, Uint32 serviceId, String floatingIpPortMacAddress, short tableId,
 462                                     TypedWriteTransaction<Configuration> confTx) {
 463         List<MatchInfo> matches = new ArrayList<>();
 464         matches.add(MatchEthernetType.MPLS_UNICAST);
 465         matches.add(new MatchMplsLabel(serviceId.longValue()));
 466
 467         Map<InstructionKey, Instruction> instructionMap = new HashMap<InstructionKey, Instruction>();
 468         int instructionKey = 0;
 469         List<ActionInfo> actionsInfos = new ArrayList<>();
 470         //NAT is required for IPv4 only. Hence always etherType will be IPv4
 471         actionsInfos.add(new ActionPopMpls(NwConstants.ETHTYPE_IPV4));
 472         actionsInfos.add(new ActionSetFieldEthernetDestination(new MacAddress(floatingIpPortMacAddress)));
 473         Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
 474         instructionMap.put(new InstructionKey(++instructionKey), writeInstruction);
 475         instructionMap.put(new InstructionKey(++instructionKey),
 476                 new InstructionGotoTable(tableId).buildInstruction(1));
 477
 478         // Install the flow entry in L3_LFIB_TABLE
 479         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
 480
 481         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
 482             10, flowRef, 0, 0,
 483             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructionMap);
 484
 485         mdsalManager.addFlow(confTx, dpId, flowEntity);
 486
 487         LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
 488     }
 489
 490     private void removeLFibTableEntry(Uint64 dpnId, Uint32 serviceId,
 491             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 492
 493         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
 494
 495         LOG.debug("removeLFibTableEntry : removing LFib entry with flow ref {}", flowRef);
 496
 497         mdsalManager.removeFlow(confTx, dpnId, new FlowKey(new FlowId(flowRef)), NwConstants.L3_LFIB_TABLE);
 498
 499         LOG.debug("removeLFibTableEntry : LFIB Entry for dpID : {} label : {} removed successfully",
 500                 dpnId, serviceId);
 501     }
 502
 503     // TODO Clean up the exception handling
 504     @SuppressWarnings("checkstyle:IllegalCatch")
 505     private void sendGarpOnInterface(final Uint64 dpnId, Uuid networkId, final IpAddress floatingIpAddress,
 506                                      String floatingIpPortMacAddress) {
 507         if (floatingIpAddress.getIpv4Address() == null) {
 508             LOG.error("sendGarpOnInterface : Failed to send GARP for IP. recieved IPv6.");
 509             natServiceCounters.garpFailedIpv6();
 510             return;
 511         }
 512
 513         String interfaceName = elanService.getExternalElanInterface(networkId.getValue(), dpnId);
 514         if (interfaceName == null) {
 515             LOG.warn("sendGarpOnInterface : Failed to send GARP for IP. Failed to retrieve interface name "
 516                     + "from network {} and dpn id {}.", networkId.getValue(), dpnId);
 517             natServiceCounters.garpFailedMissingInterface();
 518             return;
 519         }
 520
 521         try {
 522             // find the external network interface name for dpn
 523             List<InterfaceAddress> interfaceAddresses = new ArrayList<>();
 524             interfaceAddresses.add(new InterfaceAddressBuilder()
 525                 .setInterface(interfaceName)
 526                 .setIpAddress(floatingIpAddress)
 527                 .setMacaddress(new PhysAddress(floatingIpPortMacAddress)).build());
 528
 529             SendArpRequestInput sendArpRequestInput = new SendArpRequestInputBuilder().setIpaddress(floatingIpAddress)
 530                 .setInterfaceAddress(interfaceAddresses).build();
 531
 532             JdkFutures.addErrorLogging(arpUtilService.sendArpRequest(sendArpRequestInput), LOG, "sendArpRequest");
 533             natServiceCounters.garpSent();
 534         } catch (Exception e) {
 535             LOG.error("sendGarpOnInterface : Failed to send GARP request for floating ip {} from interface {}",
 536                 floatingIpAddress.getIpv4Address().getValue(), interfaceName, e);
 537             natServiceCounters.garpFailedSend();
 538         }
 539     }
 540
 541     // TODO Clean up the exception handling
 542     @SuppressWarnings("checkstyle:IllegalCatch")
 543     private void removeFromFloatingIpPortInfo(Uuid floatingIpId) {
 544         InstanceIdentifier id = buildfloatingIpIdToPortMappingIdentifier(floatingIpId);
 545         try {
 546             Optional<FloatingIpIdToPortMapping> optFloatingIpIdToPortMapping =
 547                     SingleTransactionDataBroker.syncReadOptionalAndTreatReadFailedExceptionAsAbsentOptional(dataBroker,
 548                             LogicalDatastoreType.CONFIGURATION, id);
 549             if (optFloatingIpIdToPortMapping.isPresent() && optFloatingIpIdToPortMapping.get().isFloatingIpDeleted()) {
 550                 LOG.debug("Deleting floating IP UUID {} to Floating IP neutron port mapping from Floating "
 551                     + "IP Port Info Config DS", floatingIpId.getValue());
 552                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
 553             }
 554         } catch (Exception e) {
 555             LOG.error("removeFromFloatingIpPortInfo : Deleting floating IP UUID {} to Floating IP neutron port "
 556                     + "mapping from Floating IP Port Info Config DS failed", floatingIpId.getValue(), e);
 557         }
 558     }
 559 }