natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/VpnFloatingIpHandler.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  11 import static org.opendaylight.netvirt.natservice.internal.NatUtil.buildfloatingIpIdToPortMappingIdentifier;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import java.math.BigInteger;
  19 import java.util.ArrayList;
  20 import java.util.Collections;
  21 import java.util.List;
  22 import java.util.concurrent.ExecutionException;
  23 import javax.annotation.Nonnull;
  24 import javax.inject.Inject;
  25 import javax.inject.Singleton;
  26 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  27 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  28 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  29 import org.opendaylight.genius.infra.Datastore.Configuration;
  30 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  31 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  32 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  33 import org.opendaylight.genius.infra.TypedWriteTransaction;
  34 import org.opendaylight.genius.mdsalutil.ActionInfo;
  35 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  36 import org.opendaylight.genius.mdsalutil.MatchInfo;
  37 import org.opendaylight.genius.mdsalutil.NwConstants;
  38 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  39 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  40 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  41 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  42 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  43 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  44 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  45 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  46 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  47 import org.opendaylight.infrautils.utils.concurrent.JdkFutures;
  48 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  49 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  50 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  51 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  52 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  53 import org.opendaylight.netvirt.neutronvpn.interfaces.INeutronVpnManager;
  54 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  55 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  56 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
  57 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  58 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  59 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.OdlArputilService;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpRequestInput;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpRequestInputBuilder;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.interfaces.InterfaceAddress;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.interfaces.InterfaceAddressBuilder;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.info.router.ports.ports.InternalToExternalPortMap;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.port.info.FloatingIpIdToPortMapping;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelOutput;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
  87 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  88 import org.opendaylight.yangtools.yang.common.RpcResult;
  89 import org.slf4j.Logger;
  90 import org.slf4j.LoggerFactory;
  91
  92 @Singleton
  93 public class VpnFloatingIpHandler implements FloatingIPHandler {
  94     private static final Logger LOG = LoggerFactory.getLogger(VpnFloatingIpHandler.class);
  95     private static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
  96     private static final String FLOWID_PREFIX = "NAT.";
  97
  98     private final DataBroker dataBroker;
  99     private final ManagedNewTransactionRunner txRunner;
 100     private final IMdsalApiManager mdsalManager;
 101     private final VpnRpcService vpnService;
 102     private final IBgpManager bgpManager;
 103     private final FibRpcService fibService;
 104     private final IVpnManager vpnManager;
 105     private final IFibManager fibManager;
 106     private final OdlArputilService arpUtilService;
 107     private final IElanService elanService;
 108     private final EvpnDnatFlowProgrammer evpnDnatFlowProgrammer;
 109     private final INeutronVpnManager nvpnManager;
 110     private final NatServiceCounters natServiceCounters;
 111     private final NatOverVxlanUtil natOverVxlanUtil;
 112
 113     @Inject
 114     public VpnFloatingIpHandler(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 115                                 final VpnRpcService vpnService,
 116                                 final IBgpManager bgpManager,
 117                                 final FibRpcService fibService,
 118                                 final IFibManager fibManager,
 119                                 final OdlArputilService arputilService,
 120                                 final IVpnManager vpnManager,
 121                                 final IElanService elanService,
 122                                 final EvpnDnatFlowProgrammer evpnDnatFlowProgrammer,
 123                                 final INeutronVpnManager nvpnManager,
 124                                 final NatOverVxlanUtil natOverVxlanUtil,
 125                                 NatServiceCounters natServiceCounters) {
 126         this.dataBroker = dataBroker;
 127         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 128         this.mdsalManager = mdsalManager;
 129         this.vpnService = vpnService;
 130         this.bgpManager = bgpManager;
 131         this.fibService = fibService;
 132         this.fibManager = fibManager;
 133         this.arpUtilService = arputilService;
 134         this.vpnManager = vpnManager;
 135         this.elanService = elanService;
 136         this.evpnDnatFlowProgrammer = evpnDnatFlowProgrammer;
 137         this.nvpnManager = nvpnManager;
 138         this.natServiceCounters = natServiceCounters;
 139         this.natOverVxlanUtil = natOverVxlanUtil;
 140     }
 141
 142     @Override
 143     public void onAddFloatingIp(final BigInteger dpnId, final String routerUuid, final long routerId,
 144                                 final Uuid networkId, final String interfaceName,
 145                                 final InternalToExternalPortMap mapping,
 146                                 TypedReadWriteTransaction<Configuration> confTx) {
 147         String externalIp = mapping.getExternalIp();
 148         String internalIp = mapping.getInternalIp();
 149         Uuid floatingIpId = mapping.getExternalId();
 150         Uuid subnetId = NatUtil.getFloatingIpPortSubnetIdFromFloatingIpId(dataBroker, floatingIpId);
 151         String floatingIpPortMacAddress = NatUtil.getFloatingIpPortMacFromFloatingIpId(dataBroker, floatingIpId);
 152         if (floatingIpPortMacAddress == null) {
 153             LOG.error("onAddFloatingIp: Unable to retrieve floatingIp port MAC address from floatingIpId {} for "
 154                     + "router {} to handle floatingIp {}", floatingIpId, routerUuid, externalIp);
 155             return;
 156         }
 157         Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, subnetId);
 158         final String vpnName = externalSubnet.isPresent() ? subnetId.getValue() :
 159             NatUtil.getAssociatedVPN(dataBroker, networkId);
 160         final String subnetVpnName = externalSubnet.isPresent() ? subnetId.getValue() : null;
 161         if (vpnName == null) {
 162             LOG.error("onAddFloatingIp: No VPN is associated with ext nw {} to handle add floating ip {} configuration "
 163                     + "for router {}", networkId, externalIp, routerId);
 164             return;
 165         }
 166         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
 167         if (rd == null) {
 168             LOG.error("onAddFloatingIp: Unable to retrieve external (internet) VPN RD from external VPN {} for "
 169                     + "router {} to handle floatingIp {}", vpnName, routerId, externalIp);
 170             return;
 171         }
 172         ProviderTypes provType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerUuid, networkId);
 173         if (provType == null) {
 174             return;
 175         }
 176         /*
 177          *  For external network of type GRE, it is required to use "Internet VPN VNI" for intra-DC
 178          *  communication, but we still require "MPLS labels" to reach SNAT/DNAT VMs from external
 179          *  entities via MPLSOverGRE.
 180          *
 181          *  MPLSOverGRE based external networks, the ``opendaylight-vni-ranges`` pool will be
 182          *  used to carve out a unique VNI per Internet VPN (GRE-provider-type) to be used in the
 183          *  datapath for traffic forwarding for ``SNAT-to-DNAT`` and ``DNAT-to-DNAT`` cases within the
 184          *  DataCenter.
 185         */
 186         String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
 187         LOG.debug("onAddFloatingIp: Nexthop ip for prefix {} is {}", externalIp, nextHopIp);
 188         if (provType == ProviderTypes.VXLAN) {
 189             Uuid floatingIpInterface = NatEvpnUtil.getFloatingIpInterfaceIdFromFloatingIpId(dataBroker, floatingIpId);
 190             evpnDnatFlowProgrammer.onAddFloatingIp(dpnId, routerUuid, routerId, vpnName, internalIp,
 191                     externalIp, networkId, interfaceName, floatingIpInterface.getValue(), floatingIpPortMacAddress,
 192                     rd, nextHopIp, confTx);
 193             return;
 194         }
 195         /*
 196          *  MPLS label will be used to advertise prefixes and in "L3_LFIB_TABLE" (table 20) taking the packet
 197          *  to "INBOUND_NAPT_TABLE" (table 44) and "PDNAT_TABLE" (table 25).
 198          */
 199         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
 200             .setIpPrefix(externalIp).build();
 201         ListenableFuture<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
 202
 203         ListenableFuture<RpcResult<CreateFibEntryOutput>> future = Futures.transformAsync(labelFuture, result -> {
 204             if (result.isSuccessful()) {
 205                 GenerateVpnLabelOutput output = result.getResult();
 206                 long label = output.getLabel();
 207                 LOG.debug("onAddFloatingIp : Generated label {} for prefix {}", label, externalIp);
 208                 FloatingIPListener.updateOperationalDS(dataBroker, routerUuid, interfaceName, label,
 209                         internalIp, externalIp);
 210                 /*
 211                  * For external network of type VXLAN all packets going from VMs within the DC, towards the
 212                  * external gateway device via the External VXLAN Tunnel,we are setting the VXLAN Tunnel ID to
 213                  * the L3VNI value of VPNInstance to which the VM belongs to.
 214                  */
 215                 long l3vni = 0;
 216                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanService, provType)) {
 217                     l3vni = natOverVxlanUtil.getInternetVpnVni(vpnName, l3vni).longValue();
 218                 }
 219                 String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
 220                 //Inform BGP
 221                 NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd,
 222                     fibExternalIp, nextHopIp, networkId.getValue(), floatingIpPortMacAddress,
 223                         label, l3vni, RouteOrigin.STATIC, dpnId);
 224
 225                 List<Instruction> instructions = new ArrayList<>();
 226                 List<ActionInfo> actionsInfos = new ArrayList<>();
 227                 actionsInfos.add(new ActionNxResubmit(NwConstants.PDNAT_TABLE));
 228                 instructions.add(new InstructionApplyActions(actionsInfos).buildInstruction(0));
 229
 230                 List<ActionInfo> actionInfoFib = new ArrayList<>();
 231                 List<Instruction> customInstructions = new ArrayList<>();
 232                 actionInfoFib.add(new ActionSetFieldEthernetDestination(new MacAddress(floatingIpPortMacAddress)));
 233                 customInstructions.add(new InstructionApplyActions(actionInfoFib).buildInstruction(0));
 234                 customInstructions.add(new InstructionGotoTable(NwConstants.PDNAT_TABLE).buildInstruction(1));
 235
 236                 ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 237                     innerConfTx -> {
 238                         makeTunnelTableEntry(vpnName, dpnId, label, instructions, innerConfTx, provType);
 239                         makeLFibTableEntry(dpnId, label, floatingIpPortMacAddress, NwConstants.PDNAT_TABLE,
 240                             innerConfTx);
 241                     }), LOG, "Error adding tunnel or FIB table entries");
 242
 243                 CreateFibEntryInput input = new CreateFibEntryInputBuilder().setVpnName(vpnName)
 244                         .setSourceDpid(dpnId).setInstruction(customInstructions)
 245                         .setIpAddress(fibExternalIp).setServiceId(label)
 246                         .setIpAddressSource(CreateFibEntryInput.IpAddressSource.FloatingIP)
 247                         .setInstruction(customInstructions).build();
 248                 //Future<RpcResult<java.lang.Void>> createFibEntry(CreateFibEntryInput input);
 249                 ListenableFuture<RpcResult<CreateFibEntryOutput>> future1 = fibService.createFibEntry(input);
 250                 LOG.debug("onAddFloatingIp : Add Floating Ip {} , found associated to fixed port {}",
 251                         externalIp, interfaceName);
 252                 String networkVpnName =  NatUtil.getAssociatedVPN(dataBroker, networkId);
 253                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 254                     vpnManager.addSubnetMacIntoVpnInstance(networkVpnName, subnetVpnName,
 255                             floatingIpPortMacAddress, dpnId, tx);
 256                     vpnManager.addArpResponderFlowsToExternalNetworkIps(routerUuid,
 257                             Collections.singleton(externalIp),
 258                             floatingIpPortMacAddress, dpnId, networkId);
 259                 });
 260                 return future1;
 261             } else {
 262                 String errMsg = String.format("onAddFloatingIp : Could not retrieve the label for prefix %s "
 263                         + "in VPN %s, %s", externalIp, vpnName, result.getErrors());
 264                 LOG.error(errMsg);
 265                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
 266             }
 267         }, MoreExecutors.directExecutor());
 268
 269         Futures.addCallback(future, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
 270
 271             @Override
 272             public void onFailure(@Nonnull Throwable error) {
 273                 LOG.error("onAddFloatingIp : Error in generate label or fib install process", error);
 274             }
 275
 276             @Override
 277             public void onSuccess(@Nonnull RpcResult<CreateFibEntryOutput> result) {
 278                 if (result.isSuccessful()) {
 279                     LOG.info("onAddFloatingIp : Successfully installed custom FIB routes for prefix {}", externalIp);
 280                 } else {
 281                     LOG.error("onAddFloatingIp : Error in rpc call to create custom Fib entries for prefix {} "
 282                             + "in DPN {}, {}", externalIp, dpnId, result.getErrors());
 283                 }
 284             }
 285         }, MoreExecutors.directExecutor());
 286
 287         // Handle GARP transmission
 288         final IpAddress extrenalAddress = IpAddressBuilder.getDefaultInstance(externalIp);
 289         sendGarpOnInterface(dpnId, networkId, extrenalAddress, floatingIpPortMacAddress);
 290
 291     }
 292
 293     @Override
 294     public void onRemoveFloatingIp(final BigInteger dpnId, String routerUuid, long routerId, final Uuid networkId,
 295                                    InternalToExternalPortMap mapping, final long label,
 296                                    TypedReadWriteTransaction<Configuration> confTx) {
 297         String externalIp = mapping.getExternalIp();
 298         Uuid floatingIpId = mapping.getExternalId();
 299         Uuid subnetId = NatUtil.getFloatingIpPortSubnetIdFromFloatingIpId(confTx, floatingIpId);
 300         Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(confTx, subnetId);
 301         final String vpnName = externalSubnet.isPresent() ? subnetId.getValue() :
 302             NatUtil.getAssociatedVPN(dataBroker, networkId);
 303         if (vpnName == null) {
 304             LOG.error("onRemoveFloatingIp: No VPN associated with ext nw {} to remove floating ip {} configuration "
 305                     + "for router {}", networkId, externalIp, routerUuid);
 306             return;
 307         }
 308
 309         //Remove floating mac from mymac table
 310         LOG.debug("onRemoveFloatingIp: Removing FloatingIp {}", externalIp);
 311         String floatingIpPortMacAddress = NatUtil.getFloatingIpPortMacFromFloatingIpId(confTx, floatingIpId);
 312         if (floatingIpPortMacAddress == null) {
 313             LOG.error("onRemoveFloatingIp: Unable to retrieve floatingIp port MAC address from floatingIpId {} for "
 314                     + "router {} to remove floatingIp {}", floatingIpId, routerUuid, externalIp);
 315             return;
 316         }
 317
 318         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 319             String networkVpnName =  NatUtil.getAssociatedVPN(tx, networkId);
 320             vpnManager.removeSubnetMacFromVpnInstance(networkVpnName, subnetId.getValue(), floatingIpPortMacAddress,
 321                     dpnId, tx);
 322             vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerUuid, Collections.singletonList(externalIp),
 323                     floatingIpPortMacAddress, dpnId, networkId);
 324         }), LOG, "onRemoveFloatingIp");
 325
 326         removeFromFloatingIpPortInfo(floatingIpId);
 327         ProviderTypes provType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerUuid, networkId);
 328         if (provType == null) {
 329             return;
 330         }
 331         if (provType == ProviderTypes.VXLAN) {
 332             Uuid floatingIpInterface = NatEvpnUtil.getFloatingIpInterfaceIdFromFloatingIpId(dataBroker, floatingIpId);
 333             evpnDnatFlowProgrammer.onRemoveFloatingIp(dpnId, vpnName, externalIp, floatingIpInterface.getValue(),
 334                     floatingIpPortMacAddress, routerId, confTx);
 335             return;
 336         }
 337         cleanupFibEntries(dpnId, vpnName, externalIp, label, confTx, provType);
 338     }
 339
 340     @Override
 341     public void cleanupFibEntries(BigInteger dpnId, String vpnName, String externalIp,
 342                                   long label, TypedReadWriteTransaction<Configuration> confTx, ProviderTypes provType) {
 343         //Remove Prefix from BGP
 344         String rd = NatUtil.getVpnRd(confTx, vpnName);
 345         String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
 346         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, fibExternalIp, vpnName, LOG);
 347
 348         //Remove custom FIB routes
 349
 350         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
 351         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(vpnName)
 352             .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
 353             .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.FloatingIP).build();
 354         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
 355
 356         ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture = Futures.transformAsync(future, result -> {
 357             //Release label
 358             if (result.isSuccessful()) {
 359                 /*  check if any floating IP information is available in vpn-to-dpn-list for given dpn id. If exist any
 360                  *  floating IP then do not remove INTERNAL_TUNNEL_TABLE (table=36) -> PDNAT_TABLE (table=25) flow entry
 361                  */
 362                 Boolean removeTunnelFlow = Boolean.TRUE;
 363                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanService, provType)) {
 364                     if (NatUtil.isFloatingIpPresentForDpn(dataBroker, dpnId, rd, vpnName, externalIp, false)) {
 365                         removeTunnelFlow = Boolean.FALSE;
 366                     }
 367                 }
 368                 if (removeTunnelFlow) {
 369                     removeTunnelTableEntry(dpnId, label, confTx);
 370                 }
 371                 removeLFibTableEntry(dpnId, label, confTx);
 372                 RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
 373                         .setVpnName(vpnName).setIpPrefix(externalIp).build();
 374                 return vpnService.removeVpnLabel(labelInput);
 375             } else {
 376                 String errMsg = String.format("onRemoveFloatingIp :RPC call to remove custom FIB entries "
 377                         + "on dpn %s for prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
 378                 LOG.error(errMsg);
 379                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
 380             }
 381         }, MoreExecutors.directExecutor());
 382
 383         Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
 384
 385             @Override
 386             public void onFailure(@Nonnull Throwable error) {
 387                 LOG.error("onRemoveFloatingIp : Error in removing the label or custom fib entries", error);
 388             }
 389
 390             @Override
 391             public void onSuccess(@Nonnull RpcResult<RemoveVpnLabelOutput> result) {
 392                 if (result.isSuccessful()) {
 393                     LOG.debug("onRemoveFloatingIp : Successfully removed the label for the prefix {} from VPN {}",
 394                             externalIp, vpnName);
 395                 } else {
 396                     LOG.error("onRemoveFloatingIp : Error in removing the label for prefix {} from VPN {}, {}",
 397                         externalIp, vpnName, result.getErrors());
 398                 }
 399             }
 400         }, MoreExecutors.directExecutor());
 401     }
 402
 403     private String getFlowRef(BigInteger dpnId, short tableId, long id, String ipAddress) {
 404         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR + id
 405                 + NwConstants.FLOWID_SEPARATOR + ipAddress;
 406     }
 407
 408     private void removeTunnelTableEntry(BigInteger dpnId, long serviceId,
 409             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 410
 411         LOG.debug("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
 412         mdsalManager.removeFlow(confTx, dpnId,
 413             new FlowKey(new FlowId(getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""))),
 414             NwConstants.INTERNAL_TUNNEL_TABLE);
 415         LOG.debug("removeTunnelTableEntry : Terminating service Entry for dpID {} : label : {} removed successfully",
 416                 dpnId, serviceId);
 417     }
 418
 419     private void makeTunnelTableEntry(String vpnName, BigInteger dpnId, long serviceId,
 420             List<Instruction> customInstructions, TypedWriteTransaction<Configuration> confTx, ProviderTypes provType) {
 421         List<MatchInfo> mkMatches = new ArrayList<>();
 422
 423         LOG.info("makeTunnelTableEntry on DpnId = {} and serviceId = {}", dpnId, serviceId);
 424         int flowPriority = 5;
 425         // Increased the 36->25 flow priority. If SNAT is also configured on the same
 426         // DPN, then the traffic will be hijacked to DNAT and if there are no DNAT match,
 427         // then handled back to using using flow 25->44(which will be installed as part of SNAT)
 428         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanService, provType)) {
 429             mkMatches.add(new MatchTunnelId(natOverVxlanUtil.getInternetVpnVni(vpnName, serviceId)));
 430             flowPriority = 6;
 431         } else {
 432             mkMatches.add(new MatchTunnelId(BigInteger.valueOf(serviceId)));
 433         }
 434
 435         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
 436             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), flowPriority,
 437             String.format("%s:%d", "TST Flow Entry ", serviceId),
 438             0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId)), mkMatches, customInstructions);
 439
 440         mdsalManager.addFlow(confTx, dpnId, terminatingServiceTableFlowEntity);
 441     }
 442
 443     private void makeLFibTableEntry(BigInteger dpId, long serviceId, String floatingIpPortMacAddress, short tableId,
 444                                     TypedWriteTransaction<Configuration> confTx) {
 445         List<MatchInfo> matches = new ArrayList<>();
 446         matches.add(MatchEthernetType.MPLS_UNICAST);
 447         matches.add(new MatchMplsLabel(serviceId));
 448
 449         List<Instruction> instructions = new ArrayList<>();
 450         List<ActionInfo> actionsInfos = new ArrayList<>();
 451         //NAT is required for IPv4 only. Hence always etherType will be IPv4
 452         actionsInfos.add(new ActionPopMpls(NwConstants.ETHTYPE_IPV4));
 453         actionsInfos.add(new ActionSetFieldEthernetDestination(new MacAddress(floatingIpPortMacAddress)));
 454         Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
 455         instructions.add(writeInstruction);
 456         instructions.add(new InstructionGotoTable(tableId).buildInstruction(1));
 457
 458         // Install the flow entry in L3_LFIB_TABLE
 459         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
 460
 461         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
 462             10, flowRef, 0, 0,
 463             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
 464
 465         mdsalManager.addFlow(confTx, dpId, flowEntity);
 466
 467         LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
 468     }
 469
 470     private void removeLFibTableEntry(BigInteger dpnId, long serviceId,
 471             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 472
 473         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
 474
 475         LOG.debug("removeLFibTableEntry : removing LFib entry with flow ref {}", flowRef);
 476
 477         mdsalManager.removeFlow(confTx, dpnId, new FlowKey(new FlowId(flowRef)), NwConstants.L3_LFIB_TABLE);
 478
 479         LOG.debug("removeLFibTableEntry : LFIB Entry for dpID : {} label : {} removed successfully",
 480                 dpnId, serviceId);
 481     }
 482
 483     // TODO Clean up the exception handling
 484     @SuppressWarnings("checkstyle:IllegalCatch")
 485     private void sendGarpOnInterface(final BigInteger dpnId, Uuid networkId, final IpAddress floatingIpAddress,
 486                                      String floatingIpPortMacAddress) {
 487         if (floatingIpAddress.getIpv4Address() == null) {
 488             LOG.error("sendGarpOnInterface : Failed to send GARP for IP. recieved IPv6.");
 489             natServiceCounters.garpFailedIpv6();
 490             return;
 491         }
 492
 493         String interfaceName = elanService.getExternalElanInterface(networkId.getValue(), dpnId);
 494         if (interfaceName == null) {
 495             LOG.warn("sendGarpOnInterface : Failed to send GARP for IP. Failed to retrieve interface name "
 496                     + "from network {} and dpn id {}.", networkId.getValue(), dpnId);
 497             natServiceCounters.garpFailedMissingInterface();
 498             return;
 499         }
 500
 501         try {
 502             // find the external network interface name for dpn
 503             List<InterfaceAddress> interfaceAddresses = new ArrayList<>();
 504             interfaceAddresses.add(new InterfaceAddressBuilder()
 505                 .setInterface(interfaceName)
 506                 .setIpAddress(floatingIpAddress)
 507                 .setMacaddress(new PhysAddress(floatingIpPortMacAddress)).build());
 508
 509             SendArpRequestInput sendArpRequestInput = new SendArpRequestInputBuilder().setIpaddress(floatingIpAddress)
 510                 .setInterfaceAddress(interfaceAddresses).build();
 511
 512             JdkFutures.addErrorLogging(arpUtilService.sendArpRequest(sendArpRequestInput), LOG, "sendArpRequest");
 513             natServiceCounters.garpSent();
 514         } catch (Exception e) {
 515             LOG.error("sendGarpOnInterface : Failed to send GARP request for floating ip {} from interface {}",
 516                 floatingIpAddress.getIpv4Address().getValue(), interfaceName, e);
 517             natServiceCounters.garpFailedSend();
 518         }
 519     }
 520
 521     // TODO Clean up the exception handling
 522     @SuppressWarnings("checkstyle:IllegalCatch")
 523     private void removeFromFloatingIpPortInfo(Uuid floatingIpId) {
 524         InstanceIdentifier id = buildfloatingIpIdToPortMappingIdentifier(floatingIpId);
 525         try {
 526             Optional<FloatingIpIdToPortMapping> optFloatingIpIdToPortMapping =
 527                     SingleTransactionDataBroker.syncReadOptionalAndTreatReadFailedExceptionAsAbsentOptional(dataBroker,
 528                             LogicalDatastoreType.CONFIGURATION, id);
 529             if (optFloatingIpIdToPortMapping.isPresent() && optFloatingIpIdToPortMapping.get().isFloatingIpDeleted()) {
 530                 LOG.debug("Deleting floating IP UUID {} to Floating IP neutron port mapping from Floating "
 531                     + "IP Port Info Config DS", floatingIpId.getValue());
 532                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
 533             }
 534         } catch (Exception e) {
 535             LOG.error("removeFromFloatingIpPortInfo : Deleting floating IP UUID {} to Floating IP neutron port "
 536                     + "mapping from Floating IP Port Info Config DS failed", floatingIpId.getValue(), e);
 537         }
 538     }
 539
 540 }