natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/VpnFloatingIpHandler.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  11 import static org.opendaylight.netvirt.natservice.internal.NatUtil.buildfloatingIpIdToPortMappingIdentifier;
  12
  13 import com.google.common.base.Optional;
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import java.math.BigInteger;
  19 import java.util.ArrayList;
  20 import java.util.Collections;
  21 import java.util.List;
  22 import java.util.concurrent.ExecutionException;
  23 import javax.inject.Inject;
  24 import javax.inject.Singleton;
  25 import org.eclipse.jdt.annotation.NonNull;
  26 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  27 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  28 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  29 import org.opendaylight.genius.infra.Datastore.Configuration;
  30 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  31 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  32 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  33 import org.opendaylight.genius.infra.TypedWriteTransaction;
  34 import org.opendaylight.genius.mdsalutil.ActionInfo;
  35 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  36 import org.opendaylight.genius.mdsalutil.MatchInfo;
  37 import org.opendaylight.genius.mdsalutil.NwConstants;
  38 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  39 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  40 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  41 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  42 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  43 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  44 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  45 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  46 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  47 import org.opendaylight.infrautils.utils.concurrent.JdkFutures;
  48 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  49 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  50 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  51 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  52 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  53 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  54 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  55 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
  56 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  57 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  58 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.OdlArputilService;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpRequestInput;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpRequestInputBuilder;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.interfaces.InterfaceAddress;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.interfaces.InterfaceAddressBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.info.router.ports.ports.InternalToExternalPortMap;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.port.info.FloatingIpIdToPortMapping;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelOutput;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
  86 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  87 import org.opendaylight.yangtools.yang.common.RpcResult;
  88 import org.opendaylight.yangtools.yang.common.Uint32;
  89 import org.opendaylight.yangtools.yang.common.Uint64;
  90 import org.slf4j.Logger;
  91 import org.slf4j.LoggerFactory;
  92
  93 @Singleton
  94 public class VpnFloatingIpHandler implements FloatingIPHandler {
  95     private static final Logger LOG = LoggerFactory.getLogger(VpnFloatingIpHandler.class);
  96     private static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
  97     private static final String FLOWID_PREFIX = "NAT.";
  98
  99     private final DataBroker dataBroker;
 100     private final ManagedNewTransactionRunner txRunner;
 101     private final IMdsalApiManager mdsalManager;
 102     private final VpnRpcService vpnService;
 103     private final IBgpManager bgpManager;
 104     private final FibRpcService fibService;
 105     private final IVpnManager vpnManager;
 106     private final IFibManager fibManager;
 107     private final OdlArputilService arpUtilService;
 108     private final IElanService elanService;
 109     private final EvpnDnatFlowProgrammer evpnDnatFlowProgrammer;
 110     private final NatServiceCounters natServiceCounters;
 111     private final NatOverVxlanUtil natOverVxlanUtil;
 112
 113     @Inject
 114     public VpnFloatingIpHandler(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 115                                 final VpnRpcService vpnService,
 116                                 final IBgpManager bgpManager,
 117                                 final FibRpcService fibService,
 118                                 final IFibManager fibManager,
 119                                 final OdlArputilService arputilService,
 120                                 final IVpnManager vpnManager,
 121                                 final IElanService elanService,
 122                                 final EvpnDnatFlowProgrammer evpnDnatFlowProgrammer,
 123                                 final NatOverVxlanUtil natOverVxlanUtil,
 124                                 NatServiceCounters natServiceCounters) {
 125         this.dataBroker = dataBroker;
 126         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 127         this.mdsalManager = mdsalManager;
 128         this.vpnService = vpnService;
 129         this.bgpManager = bgpManager;
 130         this.fibService = fibService;
 131         this.fibManager = fibManager;
 132         this.arpUtilService = arputilService;
 133         this.vpnManager = vpnManager;
 134         this.elanService = elanService;
 135         this.evpnDnatFlowProgrammer = evpnDnatFlowProgrammer;
 136         this.natServiceCounters = natServiceCounters;
 137         this.natOverVxlanUtil = natOverVxlanUtil;
 138     }
 139
 140     @Override
 141     public void onAddFloatingIp(final Uint64 dpnId, final String routerUuid, final Uint32 routerId,
 142                                 final Uuid networkId, final String interfaceName,
 143                                 final InternalToExternalPortMap mapping,
 144                                 TypedReadWriteTransaction<Configuration> confTx) {
 145         String externalIp = mapping.getExternalIp();
 146         String internalIp = mapping.getInternalIp();
 147         Uuid floatingIpId = mapping.getExternalId();
 148         Uuid subnetId = NatUtil.getFloatingIpPortSubnetIdFromFloatingIpId(dataBroker, floatingIpId);
 149         String floatingIpPortMacAddress = NatUtil.getFloatingIpPortMacFromFloatingIpId(dataBroker, floatingIpId);
 150         if (floatingIpPortMacAddress == null) {
 151             LOG.error("onAddFloatingIp: Unable to retrieve floatingIp port MAC address from floatingIpId {} for "
 152                     + "router {} to handle floatingIp {}", floatingIpId, routerUuid, externalIp);
 153             return;
 154         }
 155         Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, subnetId);
 156         final String vpnName = externalSubnet.isPresent() ? subnetId.getValue() :
 157             NatUtil.getAssociatedVPN(dataBroker, networkId);
 158         final String subnetVpnName = externalSubnet.isPresent() ? subnetId.getValue() : null;
 159         if (vpnName == null) {
 160             LOG.error("onAddFloatingIp: No VPN is associated with ext nw {} to handle add floating ip {} configuration "
 161                     + "for router {}", networkId, externalIp, routerId);
 162             return;
 163         }
 164         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
 165         if (rd == null) {
 166             LOG.error("onAddFloatingIp: Unable to retrieve external (internet) VPN RD from external VPN {} for "
 167                     + "router {} to handle floatingIp {}", vpnName, routerId, externalIp);
 168             return;
 169         }
 170         ProviderTypes provType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerUuid, networkId);
 171         if (provType == null) {
 172             return;
 173         }
 174         /*
 175          *  For external network of type GRE, it is required to use "Internet VPN VNI" for intra-DC
 176          *  communication, but we still require "MPLS labels" to reach SNAT/DNAT VMs from external
 177          *  entities via MPLSOverGRE.
 178          *
 179          *  MPLSOverGRE based external networks, the ``opendaylight-vni-ranges`` pool will be
 180          *  used to carve out a unique VNI per Internet VPN (GRE-provider-type) to be used in the
 181          *  datapath for traffic forwarding for ``SNAT-to-DNAT`` and ``DNAT-to-DNAT`` cases within the
 182          *  DataCenter.
 183         */
 184         String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
 185         LOG.debug("onAddFloatingIp: Nexthop ip for prefix {} is {}", externalIp, nextHopIp);
 186         if (provType == ProviderTypes.VXLAN) {
 187             Uuid floatingIpInterface = NatEvpnUtil.getFloatingIpInterfaceIdFromFloatingIpId(dataBroker, floatingIpId);
 188             evpnDnatFlowProgrammer.onAddFloatingIp(dpnId, routerUuid, routerId, vpnName, internalIp,
 189                     externalIp, networkId, interfaceName, floatingIpInterface.getValue(), floatingIpPortMacAddress,
 190                     rd, nextHopIp, confTx);
 191             return;
 192         }
 193         /*
 194          *  MPLS label will be used to advertise prefixes and in "L3_LFIB_TABLE" (table 20) taking the packet
 195          *  to "INBOUND_NAPT_TABLE" (table 44) and "PDNAT_TABLE" (table 25).
 196          */
 197         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
 198             .setIpPrefix(externalIp).build();
 199         ListenableFuture<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
 200
 201         ListenableFuture<RpcResult<CreateFibEntryOutput>> future = Futures.transformAsync(labelFuture, result -> {
 202             if (result.isSuccessful()) {
 203                 GenerateVpnLabelOutput output = result.getResult();
 204                 Uint32 label = output.getLabel();
 205                 LOG.debug("onAddFloatingIp : Generated label {} for prefix {}", label, externalIp);
 206                 FloatingIPListener.updateOperationalDS(dataBroker, routerUuid, interfaceName, label,
 207                         internalIp, externalIp);
 208                 /*
 209                  * For external network of type VXLAN all packets going from VMs within the DC, towards the
 210                  * external gateway device via the External VXLAN Tunnel,we are setting the VXLAN Tunnel ID to
 211                  * the L3VNI value of VPNInstance to which the VM belongs to.
 212                  */
 213                 Uint32 l3vni = Uint32.ZERO;
 214                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanService, provType)) {
 215                     l3vni = natOverVxlanUtil.getInternetVpnVni(vpnName, l3vni);
 216                 }
 217                 String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
 218                 //Inform BGP
 219                 NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd,
 220                     fibExternalIp, nextHopIp, networkId.getValue(), floatingIpPortMacAddress,
 221                         label, l3vni, RouteOrigin.STATIC, dpnId);
 222
 223                 List<Instruction> instructions = new ArrayList<>();
 224                 List<ActionInfo> actionsInfos = new ArrayList<>();
 225                 actionsInfos.add(new ActionNxResubmit(NwConstants.PDNAT_TABLE));
 226                 instructions.add(new InstructionApplyActions(actionsInfos).buildInstruction(0));
 227
 228                 List<ActionInfo> actionInfoFib = new ArrayList<>();
 229                 List<Instruction> customInstructions = new ArrayList<>();
 230                 actionInfoFib.add(new ActionSetFieldEthernetDestination(new MacAddress(floatingIpPortMacAddress)));
 231                 customInstructions.add(new InstructionApplyActions(actionInfoFib).buildInstruction(0));
 232                 customInstructions.add(new InstructionGotoTable(NwConstants.PDNAT_TABLE).buildInstruction(1));
 233
 234                 ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 235                     innerConfTx -> {
 236                         makeTunnelTableEntry(vpnName, dpnId, label, instructions, innerConfTx, provType);
 237                         makeLFibTableEntry(dpnId, label, floatingIpPortMacAddress, NwConstants.PDNAT_TABLE,
 238                             innerConfTx);
 239                     }), LOG, "Error adding tunnel or FIB table entries");
 240
 241                 CreateFibEntryInput input = new CreateFibEntryInputBuilder().setVpnName(vpnName)
 242                         .setSourceDpid(dpnId).setInstruction(customInstructions)
 243                         .setIpAddress(fibExternalIp).setServiceId(label)
 244                         .setIpAddressSource(CreateFibEntryInput.IpAddressSource.FloatingIP)
 245                         .setInstruction(customInstructions).build();
 246                 //Future<RpcResult<java.lang.Void>> createFibEntry(CreateFibEntryInput input);
 247                 ListenableFuture<RpcResult<CreateFibEntryOutput>> future1 = fibService.createFibEntry(input);
 248                 LOG.debug("onAddFloatingIp : Add Floating Ip {} , found associated to fixed port {}",
 249                         externalIp, interfaceName);
 250                 String networkVpnName =  NatUtil.getAssociatedVPN(dataBroker, networkId);
 251                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 252                     vpnManager.addSubnetMacIntoVpnInstance(networkVpnName, subnetVpnName,
 253                             floatingIpPortMacAddress, dpnId, tx);
 254                     vpnManager.addArpResponderFlowsToExternalNetworkIps(routerUuid,
 255                             Collections.singleton(externalIp),
 256                             floatingIpPortMacAddress, dpnId, networkId);
 257                 });
 258                 return future1;
 259             } else {
 260                 String errMsg = String.format("onAddFloatingIp : Could not retrieve the label for prefix %s "
 261                         + "in VPN %s, %s", externalIp, vpnName, result.getErrors());
 262                 LOG.error(errMsg);
 263                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
 264             }
 265         }, MoreExecutors.directExecutor());
 266
 267         Futures.addCallback(future, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
 268
 269             @Override
 270             public void onFailure(@NonNull Throwable error) {
 271                 LOG.error("onAddFloatingIp : Error in generate label or fib install process", error);
 272             }
 273
 274             @Override
 275             public void onSuccess(@NonNull RpcResult<CreateFibEntryOutput> result) {
 276                 if (result.isSuccessful()) {
 277                     LOG.info("onAddFloatingIp : Successfully installed custom FIB routes for prefix {}", externalIp);
 278                 } else {
 279                     LOG.error("onAddFloatingIp : Error in rpc call to create custom Fib entries for prefix {} "
 280                             + "in DPN {}, {}", externalIp, dpnId, result.getErrors());
 281                 }
 282             }
 283         }, MoreExecutors.directExecutor());
 284
 285         // Handle GARP transmission
 286         final IpAddress extrenalAddress = IpAddressBuilder.getDefaultInstance(externalIp);
 287         sendGarpOnInterface(dpnId, networkId, extrenalAddress, floatingIpPortMacAddress);
 288
 289     }
 290
 291     @Override
 292     public void onRemoveFloatingIp(final Uint64 dpnId, String routerUuid, Uint32 routerId, final Uuid networkId,
 293                                    InternalToExternalPortMap mapping, final Uint32 label,
 294                                    TypedReadWriteTransaction<Configuration> confTx) {
 295         String externalIp = mapping.getExternalIp();
 296         Uuid floatingIpId = mapping.getExternalId();
 297         Uuid subnetId = NatUtil.getFloatingIpPortSubnetIdFromFloatingIpId(confTx, floatingIpId);
 298         Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(confTx, subnetId);
 299         final String vpnName = externalSubnet.isPresent() ? subnetId.getValue() :
 300             NatUtil.getAssociatedVPN(dataBroker, networkId);
 301         if (vpnName == null) {
 302             LOG.error("onRemoveFloatingIp: No VPN associated with ext nw {} to remove floating ip {} configuration "
 303                     + "for router {}", networkId, externalIp, routerUuid);
 304             return;
 305         }
 306
 307         //Remove floating mac from mymac table
 308         LOG.debug("onRemoveFloatingIp: Removing FloatingIp {}", externalIp);
 309         String floatingIpPortMacAddress = NatUtil.getFloatingIpPortMacFromFloatingIpId(confTx, floatingIpId);
 310         if (floatingIpPortMacAddress == null) {
 311             LOG.error("onRemoveFloatingIp: Unable to retrieve floatingIp port MAC address from floatingIpId {} for "
 312                     + "router {} to remove floatingIp {}", floatingIpId, routerUuid, externalIp);
 313             return;
 314         }
 315
 316         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 317             String networkVpnName =  NatUtil.getAssociatedVPN(tx, networkId);
 318             vpnManager.removeSubnetMacFromVpnInstance(networkVpnName, subnetId.getValue(), floatingIpPortMacAddress,
 319                     dpnId, tx);
 320             vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerUuid, Collections.singletonList(externalIp),
 321                     floatingIpPortMacAddress, dpnId, networkId);
 322         }), LOG, "onRemoveFloatingIp");
 323
 324         removeFromFloatingIpPortInfo(floatingIpId);
 325         ProviderTypes provType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerUuid, networkId);
 326         if (provType == null) {
 327             return;
 328         }
 329         if (provType == ProviderTypes.VXLAN) {
 330             Uuid floatingIpInterface = NatEvpnUtil.getFloatingIpInterfaceIdFromFloatingIpId(dataBroker, floatingIpId);
 331             evpnDnatFlowProgrammer.onRemoveFloatingIp(dpnId, vpnName, externalIp, floatingIpInterface.getValue(),
 332                     floatingIpPortMacAddress, routerId);
 333             return;
 334         }
 335         cleanupFibEntries(dpnId, vpnName, externalIp, label, confTx, provType);
 336     }
 337
 338     @Override
 339     public void cleanupFibEntries(Uint64 dpnId, String vpnName, String externalIp,
 340                                   Uint32 label, TypedReadWriteTransaction<Configuration> confTx,
 341                                   ProviderTypes provType) {
 342         //Remove Prefix from BGP
 343         String rd = NatUtil.getVpnRd(confTx, vpnName);
 344         String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
 345         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, fibExternalIp, vpnName);
 346         NatUtil.deletePrefixToInterface(dataBroker, NatUtil.getVpnId(dataBroker, vpnName), fibExternalIp);
 347         //Remove custom FIB routes
 348
 349         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
 350         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(vpnName)
 351             .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
 352             .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.FloatingIP).build();
 353         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
 354
 355         ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture = Futures.transformAsync(future, result -> {
 356             //Release label
 357             if (result.isSuccessful()) {
 358                 /*  check if any floating IP information is available in vpn-to-dpn-list for given dpn id. If exist any
 359                  *  floating IP then do not remove INTERNAL_TUNNEL_TABLE (table=36) -> PDNAT_TABLE (table=25) flow entry
 360                  */
 361                 Boolean removeTunnelFlow = Boolean.TRUE;
 362                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanService, provType)) {
 363                     if (NatUtil.isFloatingIpPresentForDpn(dataBroker, dpnId, rd, vpnName, externalIp,
 364                             false)) {
 365                         removeTunnelFlow = Boolean.FALSE;
 366                     }
 367                 }
 368                 if (removeTunnelFlow) {
 369                     removeTunnelTableEntry(dpnId, label, confTx);
 370                 }
 371                 removeLFibTableEntry(dpnId, label, confTx);
 372                 RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
 373                         .setVpnName(vpnName).setIpPrefix(externalIp).build();
 374                 return vpnService.removeVpnLabel(labelInput);
 375             } else {
 376                 String errMsg = String.format("onRemoveFloatingIp :RPC call to remove custom FIB entries "
 377                         + "on dpn %s for prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
 378                 LOG.error(errMsg);
 379                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
 380             }
 381         }, MoreExecutors.directExecutor());
 382
 383         Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
 384
 385             @Override
 386             public void onFailure(@NonNull Throwable error) {
 387                 LOG.error("onRemoveFloatingIp : Error in removing the label or custom fib entries", error);
 388             }
 389
 390             @Override
 391             public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
 392                 if (result.isSuccessful()) {
 393                     LOG.debug("onRemoveFloatingIp : Successfully removed the label for the prefix {} from VPN {}",
 394                             externalIp, vpnName);
 395                 } else {
 396                     LOG.error("onRemoveFloatingIp : Error in removing the label for prefix {} from VPN {}, {}",
 397                         externalIp, vpnName, result.getErrors());
 398                 }
 399             }
 400         }, MoreExecutors.directExecutor());
 401     }
 402
 403     private String getFlowRef(Uint64 dpnId, short tableId, Uint32 id, String ipAddress) {
 404         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR + id
 405                 + NwConstants.FLOWID_SEPARATOR + ipAddress;
 406     }
 407
 408     private void removeTunnelTableEntry(Uint64 dpnId, Uint32 serviceId,
 409             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 410
 411         LOG.debug("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
 412         mdsalManager.removeFlow(confTx, dpnId,
 413             new FlowKey(new FlowId(getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""))),
 414             NwConstants.INTERNAL_TUNNEL_TABLE);
 415         LOG.debug("removeTunnelTableEntry : Terminating service Entry for dpID {} : label : {} removed successfully",
 416                 dpnId, serviceId);
 417     }
 418
 419     private void makeTunnelTableEntry(String vpnName, Uint64 dpnId, Uint32 serviceId,
 420             List<Instruction> customInstructions, TypedWriteTransaction<Configuration> confTx, ProviderTypes provType) {
 421         List<MatchInfo> mkMatches = new ArrayList<>();
 422
 423         LOG.info("makeTunnelTableEntry on DpnId = {} and serviceId = {}", dpnId, serviceId);
 424         int flowPriority = NatConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY;
 425         // Increased the 36->25 flow priority. If SNAT is also configured on the same
 426         // DPN, then the traffic will be hijacked to DNAT and if there are no DNAT match,
 427         // then handled back to using using flow 25->44(which will be installed as part of SNAT)
 428         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanService, provType)) {
 429             mkMatches.add(new MatchTunnelId(Uint64.valueOf(
 430                     natOverVxlanUtil.getInternetVpnVni(vpnName, serviceId).longValue())));
 431             flowPriority = NatConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY + 1;
 432         } else {
 433             mkMatches.add(new MatchTunnelId(Uint64.valueOf(serviceId)));
 434         }
 435
 436         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
 437             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), flowPriority,
 438             String.format("%s:%s", "TST Flow Entry ", serviceId),
 439             0, 0, Uint64.valueOf(COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId.longValue()))),
 440                 mkMatches, customInstructions);
 441
 442         mdsalManager.addFlow(confTx, dpnId, terminatingServiceTableFlowEntity);
 443     }
 444
 445     private void makeLFibTableEntry(Uint64 dpId, Uint32 serviceId, String floatingIpPortMacAddress, short tableId,
 446                                     TypedWriteTransaction<Configuration> confTx) {
 447         List<MatchInfo> matches = new ArrayList<>();
 448         matches.add(MatchEthernetType.MPLS_UNICAST);
 449         matches.add(new MatchMplsLabel(serviceId.longValue()));
 450
 451         List<Instruction> instructions = new ArrayList<>();
 452         List<ActionInfo> actionsInfos = new ArrayList<>();
 453         //NAT is required for IPv4 only. Hence always etherType will be IPv4
 454         actionsInfos.add(new ActionPopMpls(NwConstants.ETHTYPE_IPV4));
 455         actionsInfos.add(new ActionSetFieldEthernetDestination(new MacAddress(floatingIpPortMacAddress)));
 456         Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
 457         instructions.add(writeInstruction);
 458         instructions.add(new InstructionGotoTable(tableId).buildInstruction(1));
 459
 460         // Install the flow entry in L3_LFIB_TABLE
 461         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
 462
 463         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
 464             10, flowRef, 0, 0,
 465             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructions);
 466
 467         mdsalManager.addFlow(confTx, dpId, flowEntity);
 468
 469         LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
 470     }
 471
 472     private void removeLFibTableEntry(Uint64 dpnId, Uint32 serviceId,
 473             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 474
 475         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
 476
 477         LOG.debug("removeLFibTableEntry : removing LFib entry with flow ref {}", flowRef);
 478
 479         mdsalManager.removeFlow(confTx, dpnId, new FlowKey(new FlowId(flowRef)), NwConstants.L3_LFIB_TABLE);
 480
 481         LOG.debug("removeLFibTableEntry : LFIB Entry for dpID : {} label : {} removed successfully",
 482                 dpnId, serviceId);
 483     }
 484
 485     // TODO Clean up the exception handling
 486     @SuppressWarnings("checkstyle:IllegalCatch")
 487     private void sendGarpOnInterface(final Uint64 dpnId, Uuid networkId, final IpAddress floatingIpAddress,
 488                                      String floatingIpPortMacAddress) {
 489         if (floatingIpAddress.getIpv4Address() == null) {
 490             LOG.error("sendGarpOnInterface : Failed to send GARP for IP. recieved IPv6.");
 491             natServiceCounters.garpFailedIpv6();
 492             return;
 493         }
 494
 495         String interfaceName = elanService.getExternalElanInterface(networkId.getValue(), dpnId);
 496         if (interfaceName == null) {
 497             LOG.warn("sendGarpOnInterface : Failed to send GARP for IP. Failed to retrieve interface name "
 498                     + "from network {} and dpn id {}.", networkId.getValue(), dpnId);
 499             natServiceCounters.garpFailedMissingInterface();
 500             return;
 501         }
 502
 503         try {
 504             // find the external network interface name for dpn
 505             List<InterfaceAddress> interfaceAddresses = new ArrayList<>();
 506             interfaceAddresses.add(new InterfaceAddressBuilder()
 507                 .setInterface(interfaceName)
 508                 .setIpAddress(floatingIpAddress)
 509                 .setMacaddress(new PhysAddress(floatingIpPortMacAddress)).build());
 510
 511             SendArpRequestInput sendArpRequestInput = new SendArpRequestInputBuilder().setIpaddress(floatingIpAddress)
 512                 .setInterfaceAddress(interfaceAddresses).build();
 513
 514             JdkFutures.addErrorLogging(arpUtilService.sendArpRequest(sendArpRequestInput), LOG, "sendArpRequest");
 515             natServiceCounters.garpSent();
 516         } catch (Exception e) {
 517             LOG.error("sendGarpOnInterface : Failed to send GARP request for floating ip {} from interface {}",
 518                 floatingIpAddress.getIpv4Address().getValue(), interfaceName, e);
 519             natServiceCounters.garpFailedSend();
 520         }
 521     }
 522
 523     // TODO Clean up the exception handling
 524     @SuppressWarnings("checkstyle:IllegalCatch")
 525     private void removeFromFloatingIpPortInfo(Uuid floatingIpId) {
 526         InstanceIdentifier id = buildfloatingIpIdToPortMappingIdentifier(floatingIpId);
 527         try {
 528             Optional<FloatingIpIdToPortMapping> optFloatingIpIdToPortMapping =
 529                     SingleTransactionDataBroker.syncReadOptionalAndTreatReadFailedExceptionAsAbsentOptional(dataBroker,
 530                             LogicalDatastoreType.CONFIGURATION, id);
 531             if (optFloatingIpIdToPortMapping.isPresent() && optFloatingIpIdToPortMapping.get().isFloatingIpDeleted()) {
 532                 LOG.debug("Deleting floating IP UUID {} to Floating IP neutron port mapping from Floating "
 533                     + "IP Port Info Config DS", floatingIpId.getValue());
 534                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
 535             }
 536         } catch (Exception e) {
 537             LOG.error("removeFromFloatingIpPortInfo : Deleting floating IP UUID {} to Floating IP neutron port "
 538                     + "mapping from Floating IP Port Info Config DS failed", floatingIpId.getValue(), e);
 539         }
 540     }
 541 }