natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/VpnFloatingIpHandler.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  11 import static org.opendaylight.netvirt.natservice.internal.NatUtil.buildfloatingIpIdToPortMappingIdentifier;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import java.math.BigInteger;
  19 import java.util.ArrayList;
  20 import java.util.Collections;
  21 import java.util.List;
  22 import java.util.concurrent.ExecutionException;
  23 import javax.annotation.Nonnull;
  24 import javax.inject.Inject;
  25 import javax.inject.Singleton;
  26 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  27 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  28 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  29 import org.opendaylight.genius.infra.Datastore.Configuration;
  30 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  31 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  32 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  33 import org.opendaylight.genius.infra.TypedWriteTransaction;
  34 import org.opendaylight.genius.mdsalutil.ActionInfo;
  35 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  36 import org.opendaylight.genius.mdsalutil.MatchInfo;
  37 import org.opendaylight.genius.mdsalutil.NwConstants;
  38 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  39 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  40 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  41 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  42 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  43 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  44 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  45 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  46 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  47 import org.opendaylight.infrautils.utils.concurrent.JdkFutures;
  48 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  49 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  50 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  51 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  52 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  53 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  54 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  55 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  56 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
  57 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  58 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  59 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.OdlArputilService;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpRequestInput;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpRequestInputBuilder;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.interfaces.InterfaceAddress;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.interfaces.InterfaceAddressBuilder;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.info.router.ports.ports.InternalToExternalPortMap;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.port.info.FloatingIpIdToPortMapping;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelOutput;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
  88 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  89 import org.opendaylight.yangtools.yang.common.RpcResult;
  90 import org.slf4j.Logger;
  91 import org.slf4j.LoggerFactory;
  92
  93 @Singleton
  94 public class VpnFloatingIpHandler implements FloatingIPHandler {
  95     private static final Logger LOG = LoggerFactory.getLogger(VpnFloatingIpHandler.class);
  96     private static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
  97     private static final String FLOWID_PREFIX = "NAT.";
  98
  99     private final DataBroker dataBroker;
 100     private final ManagedNewTransactionRunner txRunner;
 101     private final IMdsalApiManager mdsalManager;
 102     private final VpnRpcService vpnService;
 103     private final IBgpManager bgpManager;
 104     private final FibRpcService fibService;
 105     private final IVpnManager vpnManager;
 106     private final IFibManager fibManager;
 107     private final OdlArputilService arpUtilService;
 108     private final IElanService elanService;
 109     private final EvpnDnatFlowProgrammer evpnDnatFlowProgrammer;
 110     private final INeutronVpnManager nvpnManager;
 111     private final IdManagerService idManager;
 112     private final NatServiceCounters natServiceCounters;
 113
 114     @Inject
 115     public VpnFloatingIpHandler(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 116                                 final VpnRpcService vpnService,
 117                                 final IBgpManager bgpManager,
 118                                 final FibRpcService fibService,
 119                                 final IFibManager fibManager,
 120                                 final OdlArputilService arputilService,
 121                                 final IVpnManager vpnManager,
 122                                 final IElanService elanService,
 123                                 final EvpnDnatFlowProgrammer evpnDnatFlowProgrammer,
 124                                 final INeutronVpnManager nvpnManager,
 125                                 final IdManagerService idManager,
 126                                 NatServiceCounters natServiceCounters) {
 127         this.dataBroker = dataBroker;
 128         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 129         this.mdsalManager = mdsalManager;
 130         this.vpnService = vpnService;
 131         this.bgpManager = bgpManager;
 132         this.fibService = fibService;
 133         this.fibManager = fibManager;
 134         this.arpUtilService = arputilService;
 135         this.vpnManager = vpnManager;
 136         this.elanService = elanService;
 137         this.evpnDnatFlowProgrammer = evpnDnatFlowProgrammer;
 138         this.nvpnManager = nvpnManager;
 139         this.idManager = idManager;
 140         this.natServiceCounters = natServiceCounters;
 141     }
 142
 143     @Override
 144     public void onAddFloatingIp(final BigInteger dpnId, final String routerUuid, final long routerId,
 145                                 final Uuid networkId, final String interfaceName,
 146                                 final InternalToExternalPortMap mapping,
 147                                 TypedReadWriteTransaction<Configuration> confTx) {
 148         String externalIp = mapping.getExternalIp();
 149         String internalIp = mapping.getInternalIp();
 150         Uuid floatingIpId = mapping.getExternalId();
 151         Uuid subnetId = NatUtil.getFloatingIpPortSubnetIdFromFloatingIpId(dataBroker, floatingIpId);
 152         String floatingIpPortMacAddress = NatUtil.getFloatingIpPortMacFromFloatingIpId(dataBroker, floatingIpId);
 153         if (floatingIpPortMacAddress == null) {
 154             LOG.error("onAddFloatingIp: Unable to retrieve floatingIp port MAC address from floatingIpId {} for "
 155                     + "router {} to handle floatingIp {}", floatingIpId, routerUuid, externalIp);
 156             return;
 157         }
 158         Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, subnetId);
 159         final String vpnName = externalSubnet.isPresent() ? subnetId.getValue() :
 160             NatUtil.getAssociatedVPN(dataBroker, networkId);
 161         final String subnetVpnName = externalSubnet.isPresent() ? subnetId.getValue() : null;
 162         if (vpnName == null) {
 163             LOG.error("onAddFloatingIp: No VPN is associated with ext nw {} to handle add floating ip {} configuration "
 164                     + "for router {}", networkId, externalIp, routerId);
 165             return;
 166         }
 167         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
 168         if (rd == null) {
 169             LOG.error("onAddFloatingIp: Unable to retrieve external (internet) VPN RD from external VPN {} for "
 170                     + "router {} to handle floatingIp {}", vpnName, routerId, externalIp);
 171             return;
 172         }
 173         ProviderTypes provType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerUuid, networkId);
 174         if (provType == null) {
 175             return;
 176         }
 177         /*
 178          *  For external network of type GRE, it is required to use "Internet VPN VNI" for intra-DC
 179          *  communication, but we still require "MPLS labels" to reach SNAT/DNAT VMs from external
 180          *  entities via MPLSOverGRE.
 181          *
 182          *  MPLSOverGRE based external networks, the ``opendaylight-vni-ranges`` pool will be
 183          *  used to carve out a unique VNI per Internet VPN (GRE-provider-type) to be used in the
 184          *  datapath for traffic forwarding for ``SNAT-to-DNAT`` and ``DNAT-to-DNAT`` cases within the
 185          *  DataCenter.
 186         */
 187         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanService, provType)) {
 188             NatOverVxlanUtil.validateAndCreateVxlanVniPool(dataBroker, nvpnManager, idManager,
 189                     NatConstants.ODL_VNI_POOL_NAME);
 190         }
 191         String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
 192         LOG.debug("onAddFloatingIp: Nexthop ip for prefix {} is {}", externalIp, nextHopIp);
 193         if (provType == ProviderTypes.VXLAN) {
 194             Uuid floatingIpInterface = NatEvpnUtil.getFloatingIpInterfaceIdFromFloatingIpId(dataBroker, floatingIpId);
 195             evpnDnatFlowProgrammer.onAddFloatingIp(dpnId, routerUuid, routerId, vpnName, internalIp,
 196                     externalIp, networkId, interfaceName, floatingIpInterface.getValue(), floatingIpPortMacAddress,
 197                     rd, nextHopIp, confTx);
 198             return;
 199         }
 200         /*
 201          *  MPLS label will be used to advertise prefixes and in "L3_LFIB_TABLE" (table 20) taking the packet
 202          *  to "INBOUND_NAPT_TABLE" (table 44) and "PDNAT_TABLE" (table 25).
 203          */
 204         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
 205             .setIpPrefix(externalIp).build();
 206         ListenableFuture<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
 207
 208         ListenableFuture<RpcResult<CreateFibEntryOutput>> future = Futures.transformAsync(labelFuture, result -> {
 209             if (result.isSuccessful()) {
 210                 GenerateVpnLabelOutput output = result.getResult();
 211                 long label = output.getLabel();
 212                 LOG.debug("onAddFloatingIp : Generated label {} for prefix {}", label, externalIp);
 213                 FloatingIPListener.updateOperationalDS(dataBroker, routerUuid, interfaceName, label,
 214                         internalIp, externalIp);
 215                 /*
 216                  * For external network of type VXLAN all packets going from VMs within the DC, towards the
 217                  * external gateway device via the External VXLAN Tunnel,we are setting the VXLAN Tunnel ID to
 218                  * the L3VNI value of VPNInstance to which the VM belongs to.
 219                  */
 220                 long l3vni = 0;
 221                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanService, provType)) {
 222                     l3vni = NatOverVxlanUtil.getInternetVpnVni(idManager, vpnName, l3vni).longValue();
 223                 }
 224                 String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
 225                 //Inform BGP
 226                 NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd,
 227                     fibExternalIp, nextHopIp, networkId.getValue(), floatingIpPortMacAddress,
 228                         label, l3vni, RouteOrigin.STATIC, dpnId);
 229
 230                 List<Instruction> instructions = new ArrayList<>();
 231                 List<ActionInfo> actionsInfos = new ArrayList<>();
 232                 actionsInfos.add(new ActionNxResubmit(NwConstants.PDNAT_TABLE));
 233                 instructions.add(new InstructionApplyActions(actionsInfos).buildInstruction(0));
 234
 235                 List<ActionInfo> actionInfoFib = new ArrayList<>();
 236                 List<Instruction> customInstructions = new ArrayList<>();
 237                 actionInfoFib.add(new ActionSetFieldEthernetDestination(new MacAddress(floatingIpPortMacAddress)));
 238                 customInstructions.add(new InstructionApplyActions(actionInfoFib).buildInstruction(0));
 239                 customInstructions.add(new InstructionGotoTable(NwConstants.PDNAT_TABLE).buildInstruction(1));
 240
 241                 ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 242                     innerConfTx -> {
 243                         makeTunnelTableEntry(vpnName, dpnId, label, instructions, innerConfTx, provType);
 244                         makeLFibTableEntry(dpnId, label, floatingIpPortMacAddress, NwConstants.PDNAT_TABLE,
 245                             innerConfTx);
 246                     }), LOG, "Error adding tunnel or FIB table entries");
 247
 248                 CreateFibEntryInput input = new CreateFibEntryInputBuilder().setVpnName(vpnName)
 249                         .setSourceDpid(dpnId).setInstruction(customInstructions)
 250                         .setIpAddress(fibExternalIp).setServiceId(label)
 251                         .setIpAddressSource(CreateFibEntryInput.IpAddressSource.FloatingIP)
 252                         .setInstruction(customInstructions).build();
 253                 //Future<RpcResult<java.lang.Void>> createFibEntry(CreateFibEntryInput input);
 254                 ListenableFuture<RpcResult<CreateFibEntryOutput>> future1 = fibService.createFibEntry(input);
 255                 LOG.debug("onAddFloatingIp : Add Floating Ip {} , found associated to fixed port {}",
 256                         externalIp, interfaceName);
 257                 String networkVpnName =  NatUtil.getAssociatedVPN(dataBroker, networkId);
 258                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 259                     vpnManager.addSubnetMacIntoVpnInstance(networkVpnName, subnetVpnName,
 260                             floatingIpPortMacAddress, dpnId, tx);
 261                     vpnManager.addArpResponderFlowsToExternalNetworkIps(routerUuid,
 262                             Collections.singleton(externalIp),
 263                             floatingIpPortMacAddress, dpnId, networkId);
 264                 });
 265                 return future1;
 266             } else {
 267                 String errMsg = String.format("onAddFloatingIp : Could not retrieve the label for prefix %s "
 268                         + "in VPN %s, %s", externalIp, vpnName, result.getErrors());
 269                 LOG.error(errMsg);
 270                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
 271             }
 272         }, MoreExecutors.directExecutor());
 273
 274         Futures.addCallback(future, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
 275
 276             @Override
 277             public void onFailure(@Nonnull Throwable error) {
 278                 LOG.error("onAddFloatingIp : Error in generate label or fib install process", error);
 279             }
 280
 281             @Override
 282             public void onSuccess(@Nonnull RpcResult<CreateFibEntryOutput> result) {
 283                 if (result.isSuccessful()) {
 284                     LOG.info("onAddFloatingIp : Successfully installed custom FIB routes for prefix {}", externalIp);
 285                 } else {
 286                     LOG.error("onAddFloatingIp : Error in rpc call to create custom Fib entries for prefix {} "
 287                             + "in DPN {}, {}", externalIp, dpnId, result.getErrors());
 288                 }
 289             }
 290         }, MoreExecutors.directExecutor());
 291
 292         // Handle GARP transmission
 293         final IpAddress extrenalAddress = IpAddressBuilder.getDefaultInstance(externalIp);
 294         sendGarpOnInterface(dpnId, networkId, extrenalAddress, floatingIpPortMacAddress);
 295
 296     }
 297
 298     @Override
 299     public void onRemoveFloatingIp(final BigInteger dpnId, String routerUuid, long routerId, final Uuid networkId,
 300                                    InternalToExternalPortMap mapping, final long label,
 301                                    TypedReadWriteTransaction<Configuration> confTx) {
 302         String externalIp = mapping.getExternalIp();
 303         Uuid floatingIpId = mapping.getExternalId();
 304         Uuid subnetId = NatUtil.getFloatingIpPortSubnetIdFromFloatingIpId(confTx, floatingIpId);
 305         Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(confTx, subnetId);
 306         final String vpnName = externalSubnet.isPresent() ? subnetId.getValue() :
 307             NatUtil.getAssociatedVPN(dataBroker, networkId);
 308         if (vpnName == null) {
 309             LOG.error("onRemoveFloatingIp: No VPN associated with ext nw {} to remove floating ip {} configuration "
 310                     + "for router {}", networkId, externalIp, routerUuid);
 311             return;
 312         }
 313
 314         //Remove floating mac from mymac table
 315         LOG.debug("onRemoveFloatingIp: Removing FloatingIp {}", externalIp);
 316         String floatingIpPortMacAddress = NatUtil.getFloatingIpPortMacFromFloatingIpId(confTx, floatingIpId);
 317         if (floatingIpPortMacAddress == null) {
 318             LOG.error("onRemoveFloatingIp: Unable to retrieve floatingIp port MAC address from floatingIpId {} for "
 319                     + "router {} to remove floatingIp {}", floatingIpId, routerUuid, externalIp);
 320             return;
 321         }
 322
 323         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 324             String networkVpnName =  NatUtil.getAssociatedVPN(tx, networkId);
 325             vpnManager.removeSubnetMacFromVpnInstance(networkVpnName, subnetId.getValue(), floatingIpPortMacAddress,
 326                     dpnId, tx);
 327             vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerUuid, Collections.singletonList(externalIp),
 328                     floatingIpPortMacAddress, dpnId, networkId);
 329         }), LOG, "onRemoveFloatingIp");
 330
 331         removeFromFloatingIpPortInfo(floatingIpId);
 332         ProviderTypes provType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerUuid, networkId);
 333         if (provType == null) {
 334             return;
 335         }
 336         if (provType == ProviderTypes.VXLAN) {
 337             Uuid floatingIpInterface = NatEvpnUtil.getFloatingIpInterfaceIdFromFloatingIpId(dataBroker, floatingIpId);
 338             evpnDnatFlowProgrammer.onRemoveFloatingIp(dpnId, vpnName, externalIp, floatingIpInterface.getValue(),
 339                     floatingIpPortMacAddress, routerId, confTx);
 340             return;
 341         }
 342         cleanupFibEntries(dpnId, vpnName, externalIp, label, confTx, provType);
 343     }
 344
 345     @Override
 346     public void cleanupFibEntries(BigInteger dpnId, String vpnName, String externalIp,
 347                                   long label, TypedReadWriteTransaction<Configuration> confTx, ProviderTypes provType) {
 348         //Remove Prefix from BGP
 349         String rd = NatUtil.getVpnRd(confTx, vpnName);
 350         String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
 351         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, fibExternalIp, vpnName, LOG);
 352
 353         //Remove custom FIB routes
 354
 355         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
 356         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(vpnName)
 357             .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
 358             .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.FloatingIP).build();
 359         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
 360
 361         ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture = Futures.transformAsync(future, result -> {
 362             //Release label
 363             if (result.isSuccessful()) {
 364                 /*  check if any floating IP information is available in vpn-to-dpn-list for given dpn id. If exist any
 365                  *  floating IP then do not remove INTERNAL_TUNNEL_TABLE (table=36) -> PDNAT_TABLE (table=25) flow entry
 366                  */
 367                 Boolean removeTunnelFlow = Boolean.TRUE;
 368                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanService, provType)) {
 369                     if (NatUtil.isFloatingIpPresentForDpn(dataBroker, dpnId, rd, vpnName, externalIp, false)) {
 370                         removeTunnelFlow = Boolean.FALSE;
 371                     }
 372                 }
 373                 if (removeTunnelFlow) {
 374                     removeTunnelTableEntry(dpnId, label, confTx);
 375                 }
 376                 removeLFibTableEntry(dpnId, label, confTx);
 377                 RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
 378                         .setVpnName(vpnName).setIpPrefix(externalIp).build();
 379                 return vpnService.removeVpnLabel(labelInput);
 380             } else {
 381                 String errMsg = String.format("onRemoveFloatingIp :RPC call to remove custom FIB entries "
 382                         + "on dpn %s for prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
 383                 LOG.error(errMsg);
 384                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
 385             }
 386         }, MoreExecutors.directExecutor());
 387
 388         Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
 389
 390             @Override
 391             public void onFailure(@Nonnull Throwable error) {
 392                 LOG.error("onRemoveFloatingIp : Error in removing the label or custom fib entries", error);
 393             }
 394
 395             @Override
 396             public void onSuccess(@Nonnull RpcResult<RemoveVpnLabelOutput> result) {
 397                 if (result.isSuccessful()) {
 398                     LOG.debug("onRemoveFloatingIp : Successfully removed the label for the prefix {} from VPN {}",
 399                             externalIp, vpnName);
 400                 } else {
 401                     LOG.error("onRemoveFloatingIp : Error in removing the label for prefix {} from VPN {}, {}",
 402                         externalIp, vpnName, result.getErrors());
 403                 }
 404             }
 405         }, MoreExecutors.directExecutor());
 406     }
 407
 408     private String getFlowRef(BigInteger dpnId, short tableId, long id, String ipAddress) {
 409         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR + id
 410                 + NwConstants.FLOWID_SEPARATOR + ipAddress;
 411     }
 412
 413     private void removeTunnelTableEntry(BigInteger dpnId, long serviceId,
 414             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 415
 416         LOG.debug("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
 417         mdsalManager.removeFlow(confTx, dpnId,
 418             new FlowKey(new FlowId(getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""))),
 419             NwConstants.INTERNAL_TUNNEL_TABLE);
 420         LOG.debug("removeTunnelTableEntry : Terminating service Entry for dpID {} : label : {} removed successfully",
 421                 dpnId, serviceId);
 422     }
 423
 424     private void makeTunnelTableEntry(String vpnName, BigInteger dpnId, long serviceId,
 425             List<Instruction> customInstructions, TypedWriteTransaction<Configuration> confTx, ProviderTypes provType) {
 426         List<MatchInfo> mkMatches = new ArrayList<>();
 427
 428         LOG.info("makeTunnelTableEntry on DpnId = {} and serviceId = {}", dpnId, serviceId);
 429         int flowPriority = 5;
 430         // Increased the 36->25 flow priority. If SNAT is also configured on the same
 431         // DPN, then the traffic will be hijacked to DNAT and if there are no DNAT match,
 432         // then handled back to using using flow 25->44(which will be installed as part of SNAT)
 433         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanService, provType)) {
 434             mkMatches.add(new MatchTunnelId(NatOverVxlanUtil.getInternetVpnVni(idManager, vpnName, serviceId)));
 435             flowPriority = 6;
 436         } else {
 437             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(serviceId)));
 438         }
 439
 440         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
 441             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), flowPriority,
 442             String.format("%s:%d", "TST Flow Entry ", serviceId),
 443             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId)), mkMatches, customInstructions);
 444
 445         mdsalManager.addFlow(confTx, dpnId, terminatingServiceTableFlowEntity);
 446     }
 447
 448     private void makeLFibTableEntry(BigInteger dpId, long serviceId, String floatingIpPortMacAddress, short tableId,
 449                                     TypedWriteTransaction<Configuration> confTx) {
 450         List<MatchInfo> matches = new ArrayList<>();
 451         matches.add(MatchEthernetType.MPLS_UNICAST);
 452         matches.add(new MatchMplsLabel(serviceId));
 453
 454         List<Instruction> instructions = new ArrayList<>();
 455         List<ActionInfo> actionsInfos = new ArrayList<>();
 456         //NAT is required for IPv4 only. Hence always etherType will be IPv4
 457         actionsInfos.add(new ActionPopMpls(NwConstants.ETHTYPE_IPV4));
 458         actionsInfos.add(new ActionSetFieldEthernetDestination(new MacAddress(floatingIpPortMacAddress)));
 459         Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
 460         instructions.add(writeInstruction);
 461         instructions.add(new InstructionGotoTable(tableId).buildInstruction(1));
 462
 463         // Install the flow entry in L3_LFIB_TABLE
 464         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
 465
 466         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
 467             10, flowRef, 0, 0,
 468             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
 469
 470         mdsalManager.addFlow(confTx, dpId, flowEntity);
 471
 472         LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
 473     }
 474
 475     private void removeLFibTableEntry(BigInteger dpnId, long serviceId,
 476             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 477
 478         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
 479
 480         LOG.debug("removeLFibTableEntry : removing LFib entry with flow ref {}", flowRef);
 481
 482         mdsalManager.removeFlow(confTx, dpnId, new FlowKey(new FlowId(flowRef)), NwConstants.L3_LFIB_TABLE);
 483
 484         LOG.debug("removeLFibTableEntry : LFIB Entry for dpID : {} label : {} removed successfully",
 485                 dpnId, serviceId);
 486     }
 487
 488     // TODO Clean up the exception handling
 489     @SuppressWarnings("checkstyle:IllegalCatch")
 490     private void sendGarpOnInterface(final BigInteger dpnId, Uuid networkId, final IpAddress floatingIpAddress,
 491                                      String floatingIpPortMacAddress) {
 492         if (floatingIpAddress.getIpv4Address() == null) {
 493             LOG.error("sendGarpOnInterface : Failed to send GARP for IP. recieved IPv6.");
 494             natServiceCounters.garpFailedIpv6();
 495             return;
 496         }
 497
 498         String interfaceName = elanService.getExternalElanInterface(networkId.getValue(), dpnId);
 499         if (interfaceName == null) {
 500             LOG.warn("sendGarpOnInterface : Failed to send GARP for IP. Failed to retrieve interface name "
 501                     + "from network {} and dpn id {}.", networkId.getValue(), dpnId);
 502             natServiceCounters.garpFailedMissingInterface();
 503             return;
 504         }
 505
 506         try {
 507             // find the external network interface name for dpn
 508             List<InterfaceAddress> interfaceAddresses = new ArrayList<>();
 509             interfaceAddresses.add(new InterfaceAddressBuilder()
 510                 .setInterface(interfaceName)
 511                 .setIpAddress(floatingIpAddress)
 512                 .setMacaddress(new PhysAddress(floatingIpPortMacAddress)).build());
 513
 514             SendArpRequestInput sendArpRequestInput = new SendArpRequestInputBuilder().setIpaddress(floatingIpAddress)
 515                 .setInterfaceAddress(interfaceAddresses).build();
 516
 517             JdkFutures.addErrorLogging(arpUtilService.sendArpRequest(sendArpRequestInput), LOG, "sendArpRequest");
 518             natServiceCounters.garpSent();
 519         } catch (Exception e) {
 520             LOG.error("sendGarpOnInterface : Failed to send GARP request for floating ip {} from interface {}",
 521                 floatingIpAddress.getIpv4Address().getValue(), interfaceName, e);
 522             natServiceCounters.garpFailedSend();
 523         }
 524     }
 525
 526     // TODO Clean up the exception handling
 527     @SuppressWarnings("checkstyle:IllegalCatch")
 528     private void removeFromFloatingIpPortInfo(Uuid floatingIpId) {
 529         InstanceIdentifier id = buildfloatingIpIdToPortMappingIdentifier(floatingIpId);
 530         try {
 531             Optional<FloatingIpIdToPortMapping> optFloatingIpIdToPortMapping =
 532                     SingleTransactionDataBroker.syncReadOptionalAndTreatReadFailedExceptionAsAbsentOptional(dataBroker,
 533                             LogicalDatastoreType.CONFIGURATION, id);
 534             if (optFloatingIpIdToPortMapping.isPresent() && optFloatingIpIdToPortMapping.get().isFloatingIpDeleted()) {
 535                 LOG.debug("Deleting floating IP UUID {} to Floating IP neutron port mapping from Floating "
 536                     + "IP Port Info Config DS", floatingIpId.getValue());
 537                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
 538             }
 539         } catch (Exception e) {
 540             LOG.error("removeFromFloatingIpPortInfo : Deleting floating IP UUID {} to Floating IP neutron port "
 541                     + "mapping from Floating IP Port Info Config DS failed", floatingIpId.getValue(), e);
 542         }
 543     }
 544
 545 }