natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/VpnFloatingIpHandler.java

   1 /*
   2  * Copyright © 2016, 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.natservice.internal;
   9
  10 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  11 import static org.opendaylight.netvirt.natservice.internal.NatUtil.buildfloatingIpIdToPortMappingIdentifier;
  12
  13 import com.google.common.util.concurrent.FutureCallback;
  14 import com.google.common.util.concurrent.Futures;
  15 import com.google.common.util.concurrent.JdkFutureAdapters;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import java.math.BigInteger;
  19 import java.util.ArrayList;
  20 import java.util.Collections;
  21 import java.util.HashMap;
  22 import java.util.List;
  23 import java.util.Map;
  24 import java.util.Optional;
  25 import java.util.concurrent.ExecutionException;
  26 import java.util.concurrent.Future;
  27 import javax.inject.Inject;
  28 import javax.inject.Singleton;
  29 import org.eclipse.jdt.annotation.NonNull;
  30 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  31 import org.opendaylight.genius.infra.Datastore.Configuration;
  32 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  33 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  34 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  35 import org.opendaylight.genius.infra.TypedWriteTransaction;
  36 import org.opendaylight.genius.mdsalutil.ActionInfo;
  37 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  38 import org.opendaylight.genius.mdsalutil.MatchInfo;
  39 import org.opendaylight.genius.mdsalutil.NwConstants;
  40 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  41 import org.opendaylight.genius.mdsalutil.actions.ActionPopMpls;
  42 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  43 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  44 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  45 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  46 import org.opendaylight.genius.mdsalutil.matches.MatchEthernetType;
  47 import org.opendaylight.genius.mdsalutil.matches.MatchMplsLabel;
  48 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  49 import org.opendaylight.infrautils.utils.concurrent.LoggingFutures;
  50 import org.opendaylight.mdsal.binding.api.DataBroker;
  51 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  52 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  53 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  54 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  55 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  56 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  57 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddress;
  58 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpAddressBuilder;
  59 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  60 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.PhysAddress;
  61 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.FlowId;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.FlowKey;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionKey;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.OdlArputilService;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpRequestInput;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.SendArpRequestInputBuilder;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.interfaces.InterfaceAddress;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.arputil.rev160406.interfaces.InterfaceAddressBuilder;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ProviderTypes;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.info.router.ports.ports.InternalToExternalPortMap;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.floating.ip.port.info.FloatingIpIdToPortMapping;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInput;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelInputBuilder;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.GenerateVpnLabelOutput;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInput;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelInputBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.RemoveVpnLabelOutput;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.vpn.rpc.rev160201.VpnRpcService;
  90 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  91 import org.opendaylight.yangtools.yang.common.RpcResult;
  92 import org.opendaylight.yangtools.yang.common.Uint32;
  93 import org.opendaylight.yangtools.yang.common.Uint64;
  94 import org.slf4j.Logger;
  95 import org.slf4j.LoggerFactory;
  96
  97 @Singleton
  98 public class VpnFloatingIpHandler implements FloatingIPHandler {
  99     private static final Logger LOG = LoggerFactory.getLogger(VpnFloatingIpHandler.class);
 100     private static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
 101     private static final String FLOWID_PREFIX = "NAT.";
 102
 103     private final DataBroker dataBroker;
 104     private final ManagedNewTransactionRunner txRunner;
 105     private final IMdsalApiManager mdsalManager;
 106     private final VpnRpcService vpnService;
 107     private final IBgpManager bgpManager;
 108     private final FibRpcService fibService;
 109     private final IVpnManager vpnManager;
 110     private final IFibManager fibManager;
 111     private final OdlArputilService arpUtilService;
 112     private final IElanService elanService;
 113     private final EvpnDnatFlowProgrammer evpnDnatFlowProgrammer;
 114     private final NatServiceCounters natServiceCounters;
 115     private final NatOverVxlanUtil natOverVxlanUtil;
 116
 117     @Inject
 118     public VpnFloatingIpHandler(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 119                                 final VpnRpcService vpnService,
 120                                 final IBgpManager bgpManager,
 121                                 final FibRpcService fibService,
 122                                 final IFibManager fibManager,
 123                                 final OdlArputilService arputilService,
 124                                 final IVpnManager vpnManager,
 125                                 final IElanService elanService,
 126                                 final EvpnDnatFlowProgrammer evpnDnatFlowProgrammer,
 127                                 final NatOverVxlanUtil natOverVxlanUtil,
 128                                 NatServiceCounters natServiceCounters) {
 129         this.dataBroker = dataBroker;
 130         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 131         this.mdsalManager = mdsalManager;
 132         this.vpnService = vpnService;
 133         this.bgpManager = bgpManager;
 134         this.fibService = fibService;
 135         this.fibManager = fibManager;
 136         this.arpUtilService = arputilService;
 137         this.vpnManager = vpnManager;
 138         this.elanService = elanService;
 139         this.evpnDnatFlowProgrammer = evpnDnatFlowProgrammer;
 140         this.natServiceCounters = natServiceCounters;
 141         this.natOverVxlanUtil = natOverVxlanUtil;
 142     }
 143
 144     @Override
 145     public void onAddFloatingIp(final Uint64 dpnId, final String routerUuid, final Uint32 routerId,
 146                                 final Uuid networkId, final String interfaceName,
 147                                 final InternalToExternalPortMap mapping, final String rd,
 148                                 TypedReadWriteTransaction<Configuration> confTx) {
 149         String externalIp = mapping.getExternalIp();
 150         String internalIp = mapping.getInternalIp();
 151         Uuid floatingIpId = mapping.getExternalId();
 152         Uuid subnetId = NatUtil.getFloatingIpPortSubnetIdFromFloatingIpId(dataBroker, floatingIpId);
 153         String floatingIpPortMacAddress = NatUtil.getFloatingIpPortMacFromFloatingIpId(dataBroker, floatingIpId);
 154         if (floatingIpPortMacAddress == null) {
 155             LOG.error("onAddFloatingIp: Unable to retrieve floatingIp port MAC address from floatingIpId {} for "
 156                     + "router {} to handle floatingIp {}", floatingIpId, routerUuid, externalIp);
 157             return;
 158         }
 159         Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(dataBroker, subnetId);
 160         final String vpnName = externalSubnet.isPresent() ? subnetId.getValue() :
 161             NatUtil.getAssociatedVPN(dataBroker, networkId);
 162         final String subnetVpnName = externalSubnet.isPresent() ? subnetId.getValue() : null;
 163         if (vpnName == null) {
 164             LOG.error("onAddFloatingIp: No VPN is associated with ext nw {} to handle add floating ip {} configuration "
 165                     + "for router {}", networkId, externalIp, routerId);
 166             return;
 167         }
 168         if (rd == null) {
 169             LOG.error("onAddFloatingIp: Unable to retrieve external (internet) VPN RD from external VPN {} for "
 170                     + "router {} to handle floatingIp {}", vpnName, routerId, externalIp);
 171             return;
 172         }
 173         ProviderTypes provType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerUuid, networkId);
 174         if (provType == null) {
 175             return;
 176         }
 177         /*
 178          *  For external network of type GRE, it is required to use "Internet VPN VNI" for intra-DC
 179          *  communication, but we still require "MPLS labels" to reach SNAT/DNAT VMs from external
 180          *  entities via MPLSOverGRE.
 181          *
 182          *  MPLSOverGRE based external networks, the ``opendaylight-vni-ranges`` pool will be
 183          *  used to carve out a unique VNI per Internet VPN (GRE-provider-type) to be used in the
 184          *  datapath for traffic forwarding for ``SNAT-to-DNAT`` and ``DNAT-to-DNAT`` cases within the
 185          *  DataCenter.
 186         */
 187         String nextHopIp = NatUtil.getEndpointIpAddressForDPN(dataBroker, dpnId);
 188         LOG.debug("onAddFloatingIp: Nexthop ip for prefix {} is {}", externalIp, nextHopIp);
 189         if (provType == ProviderTypes.VXLAN) {
 190             Uuid floatingIpInterface = NatEvpnUtil.getFloatingIpInterfaceIdFromFloatingIpId(dataBroker, floatingIpId);
 191             evpnDnatFlowProgrammer.onAddFloatingIp(dpnId, routerUuid, routerId, vpnName, internalIp,
 192                     externalIp, networkId, interfaceName, floatingIpInterface.getValue(), floatingIpPortMacAddress,
 193                     rd, nextHopIp, confTx);
 194             return;
 195         }
 196         /*
 197          *  MPLS label will be used to advertise prefixes and in "L3_LFIB_TABLE" (table 20) taking the packet
 198          *  to "INBOUND_NAPT_TABLE" (table 44) and "PDNAT_TABLE" (table 25).
 199          */
 200         GenerateVpnLabelInput labelInput = new GenerateVpnLabelInputBuilder().setVpnName(vpnName)
 201             .setIpPrefix(externalIp).build();
 202         ListenableFuture<RpcResult<GenerateVpnLabelOutput>> labelFuture = vpnService.generateVpnLabel(labelInput);
 203
 204         ListenableFuture<RpcResult<CreateFibEntryOutput>> future = Futures.transformAsync(labelFuture, result -> {
 205             if (result.isSuccessful()) {
 206                 GenerateVpnLabelOutput output = result.getResult();
 207                 Uint32 label = output.getLabel();
 208                 LOG.debug("onAddFloatingIp : Generated label {} for prefix {}", label, externalIp);
 209                 FloatingIPListener.updateOperationalDS(dataBroker, routerUuid, interfaceName, label,
 210                         internalIp, externalIp);
 211                 /*
 212                  * For external network of type VXLAN all packets going from VMs within the DC, towards the
 213                  * external gateway device via the External VXLAN Tunnel,we are setting the VXLAN Tunnel ID to
 214                  * the L3VNI value of VPNInstance to which the VM belongs to.
 215                  */
 216                 Uint32 l3vni = Uint32.ZERO;
 217                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanService, provType)) {
 218                     l3vni = natOverVxlanUtil.getInternetVpnVni(vpnName, l3vni);
 219                 }
 220                 String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
 221                 //Inform BGP
 222                 NatUtil.addPrefixToBGP(dataBroker, bgpManager, fibManager, vpnName, rd,
 223                     fibExternalIp, nextHopIp, networkId.getValue(), floatingIpPortMacAddress,
 224                         label, l3vni, RouteOrigin.STATIC, dpnId);
 225
 226                 List<Instruction> instructions = new ArrayList<>();
 227                 List<ActionInfo> actionsInfos = new ArrayList<>();
 228                 actionsInfos.add(new ActionNxResubmit(NwConstants.PDNAT_TABLE));
 229                 instructions.add(new InstructionApplyActions(actionsInfos).buildInstruction(0));
 230
 231                 List<ActionInfo> actionInfoFib = new ArrayList<>();
 232                 List<Instruction> customInstructions = new ArrayList<>();
 233                 actionInfoFib.add(new ActionSetFieldEthernetDestination(new MacAddress(floatingIpPortMacAddress)));
 234                 customInstructions.add(new InstructionApplyActions(actionInfoFib).buildInstruction(0));
 235                 customInstructions.add(new InstructionGotoTable(NwConstants.PDNAT_TABLE).buildInstruction(1));
 236
 237                 LoggingFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 238                     innerConfTx -> {
 239                         makeTunnelTableEntry(vpnName, dpnId, label, instructions, innerConfTx, provType);
 240                         makeLFibTableEntry(dpnId, label, floatingIpPortMacAddress, NwConstants.PDNAT_TABLE,
 241                             innerConfTx);
 242                     }), LOG, "Error adding tunnel or FIB table entries");
 243
 244                 CreateFibEntryInput input = new CreateFibEntryInputBuilder().setVpnName(vpnName)
 245                         .setSourceDpid(dpnId).setInstruction(customInstructions)
 246                         .setIpAddress(fibExternalIp).setServiceId(label)
 247                         .setIpAddressSource(CreateFibEntryInput.IpAddressSource.FloatingIP)
 248                         .setInstruction(customInstructions).build();
 249                 //Future<RpcResult<java.lang.Void>> createFibEntry(CreateFibEntryInput input);
 250                 ListenableFuture<RpcResult<CreateFibEntryOutput>> future1 = fibService.createFibEntry(input);
 251                 LOG.debug("onAddFloatingIp : Add Floating Ip {} , found associated to fixed port {}",
 252                         externalIp, interfaceName);
 253                 String networkVpnName =  NatUtil.getAssociatedVPN(dataBroker, networkId);
 254                 txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, tx -> {
 255                     vpnManager.addSubnetMacIntoVpnInstance(networkVpnName, subnetVpnName,
 256                             floatingIpPortMacAddress, dpnId, tx);
 257                     vpnManager.addArpResponderFlowsToExternalNetworkIps(routerUuid,
 258                             Collections.singleton(externalIp),
 259                             floatingIpPortMacAddress, dpnId, networkId);
 260                 });
 261                 return future1;
 262             } else {
 263                 String errMsg = String.format("onAddFloatingIp : Could not retrieve the label for prefix %s "
 264                         + "in VPN %s, %s", externalIp, vpnName, result.getErrors());
 265                 LOG.error(errMsg);
 266                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
 267             }
 268         }, MoreExecutors.directExecutor());
 269
 270         Futures.addCallback(future, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
 271
 272             @Override
 273             public void onFailure(@NonNull Throwable error) {
 274                 LOG.error("onAddFloatingIp : Error in generate label or fib install process", error);
 275             }
 276
 277             @Override
 278             public void onSuccess(@NonNull RpcResult<CreateFibEntryOutput> result) {
 279                 if (result.isSuccessful()) {
 280                     LOG.info("onAddFloatingIp : Successfully installed custom FIB routes for prefix {}", externalIp);
 281                 } else {
 282                     LOG.error("onAddFloatingIp : Error in rpc call to create custom Fib entries for prefix {} "
 283                             + "in DPN {}, {}", externalIp, dpnId, result.getErrors());
 284                 }
 285             }
 286         }, MoreExecutors.directExecutor());
 287
 288         // Handle GARP transmission
 289         final IpAddress extrenalAddress = IpAddressBuilder.getDefaultInstance(externalIp);
 290         sendGarpOnInterface(dpnId, networkId, extrenalAddress, floatingIpPortMacAddress);
 291
 292     }
 293
 294     @Override
 295     public void onRemoveFloatingIp(final Uint64 dpnId, String routerUuid, Uint32 routerId, final Uuid networkId,
 296                                    InternalToExternalPortMap mapping, final Uint32 label, final String vrfId,
 297                                    TypedReadWriteTransaction<Configuration> confTx) {
 298         String externalIp = mapping.getExternalIp();
 299         Uuid floatingIpId = mapping.getExternalId();
 300         Uuid subnetId = NatUtil.getFloatingIpPortSubnetIdFromFloatingIpId(confTx, floatingIpId);
 301         Optional<Subnets> externalSubnet = NatUtil.getOptionalExternalSubnets(confTx, subnetId);
 302         final String vpnName = externalSubnet.isPresent() ? subnetId.getValue() :
 303             NatUtil.getAssociatedVPN(dataBroker, networkId);
 304         if (vpnName == null) {
 305             LOG.error("onRemoveFloatingIp: No VPN associated with ext nw {} to remove floating ip {} configuration "
 306                     + "for router {}", networkId, externalIp, routerUuid);
 307             return;
 308         }
 309
 310         //Remove floating mac from mymac table
 311         LOG.debug("onRemoveFloatingIp: Removing FloatingIp {}", externalIp);
 312         String floatingIpPortMacAddress = NatUtil.getFloatingIpPortMacFromFloatingIpId(confTx, floatingIpId);
 313         if (floatingIpPortMacAddress == null) {
 314             LOG.error("onRemoveFloatingIp: Unable to retrieve floatingIp port MAC address from floatingIpId {} for "
 315                     + "router {} to remove floatingIp {}", floatingIpId, routerUuid, externalIp);
 316             return;
 317         }
 318
 319         LoggingFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 320             String networkVpnName =  NatUtil.getAssociatedVPN(tx, networkId);
 321             vpnManager.removeSubnetMacFromVpnInstance(networkVpnName, subnetId.getValue(), floatingIpPortMacAddress,
 322                     dpnId, tx);
 323             vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerUuid, Collections.singletonList(externalIp),
 324                     floatingIpPortMacAddress, dpnId, networkId);
 325         }), LOG, "onRemoveFloatingIp");
 326
 327         removeFromFloatingIpPortInfo(floatingIpId);
 328         ProviderTypes provType = NatEvpnUtil.getExtNwProvTypeFromRouterName(dataBroker, routerUuid, networkId);
 329         if (provType == null) {
 330             return;
 331         }
 332         if (provType == ProviderTypes.VXLAN) {
 333             Uuid floatingIpInterface = NatEvpnUtil.getFloatingIpInterfaceIdFromFloatingIpId(dataBroker, floatingIpId);
 334             evpnDnatFlowProgrammer.onRemoveFloatingIp(dpnId, vpnName, externalIp, floatingIpInterface.getValue(),
 335                     floatingIpPortMacAddress, routerId);
 336             return;
 337         }
 338         cleanupFibEntries(dpnId, vpnName, externalIp, label, vrfId, confTx, provType);
 339     }
 340
 341     @Override
 342     public void cleanupFibEntries(Uint64 dpnId, String vpnName, String externalIp,
 343                                   Uint32 label, final String rd, TypedReadWriteTransaction<Configuration> confTx,
 344                                   ProviderTypes provType) {
 345         //Remove Prefix from BGP
 346         String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
 347         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, fibExternalIp, vpnName);
 348         NatUtil.deletePrefixToInterface(dataBroker, NatUtil.getVpnId(dataBroker, vpnName), fibExternalIp);
 349         //Remove custom FIB routes
 350
 351         //Future<RpcResult<java.lang.Void>> removeFibEntry(RemoveFibEntryInput input);
 352         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(vpnName)
 353             .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(label)
 354             .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.FloatingIP).build();
 355         ListenableFuture<RpcResult<RemoveFibEntryOutput>> future = fibService.removeFibEntry(input);
 356
 357         ListenableFuture<RpcResult<RemoveVpnLabelOutput>> labelFuture = Futures.transformAsync(future, result -> {
 358             //Release label
 359             if (result.isSuccessful()) {
 360                 /*  check if any floating IP information is available in vpn-to-dpn-list for given dpn id. If exist any
 361                  *  floating IP then do not remove INTERNAL_TUNNEL_TABLE (table=36) -> PDNAT_TABLE (table=25) flow entry
 362                  */
 363                 Boolean removeTunnelFlow = Boolean.TRUE;
 364                 if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanService, provType)) {
 365                     if (NatUtil.isFloatingIpPresentForDpn(dataBroker, dpnId, rd, vpnName, externalIp,
 366                             false)) {
 367                         removeTunnelFlow = Boolean.FALSE;
 368                     }
 369                 }
 370                 if (removeTunnelFlow) {
 371                     removeTunnelTableEntry(dpnId, label, confTx);
 372                 }
 373                 removeLFibTableEntry(dpnId, label, confTx);
 374                 RemoveVpnLabelInput labelInput = new RemoveVpnLabelInputBuilder()
 375                         .setVpnName(vpnName).setIpPrefix(externalIp).build();
 376                 Future<RpcResult<RemoveVpnLabelOutput>> labelFuture1 = vpnService.removeVpnLabel(labelInput);
 377                 if (labelFuture1.get() == null || !labelFuture1.get().isSuccessful()) {
 378                     String errMsg = String.format(
 379                             "VpnFloatingIpHandler: RPC call to remove VPN label on dpn %s "
 380                                     + "for prefix %s failed for vpn %s - %s",
 381                             dpnId, externalIp, vpnName, result.getErrors());
 382                     LOG.error(errMsg);
 383                     return Futures.immediateFailedFuture(new RuntimeException(errMsg));
 384                 }
 385                 return JdkFutureAdapters.listenInPoolThread(labelFuture1);
 386             } else {
 387                 String errMsg = String.format("onRemoveFloatingIp :RPC call to remove custom FIB entries "
 388                         + "on dpn %s for prefix %s Failed - %s", dpnId, externalIp, result.getErrors());
 389                 LOG.error(errMsg);
 390                 return Futures.immediateFailedFuture(new RuntimeException(errMsg));
 391             }
 392         }, MoreExecutors.directExecutor());
 393
 394         Futures.addCallback(labelFuture, new FutureCallback<RpcResult<RemoveVpnLabelOutput>>() {
 395
 396             @Override
 397             public void onFailure(@NonNull Throwable error) {
 398                 LOG.error("onRemoveFloatingIp : Error in removing the label or custom fib entries", error);
 399             }
 400
 401             @Override
 402             public void onSuccess(@NonNull RpcResult<RemoveVpnLabelOutput> result) {
 403                 if (result.isSuccessful()) {
 404                     LOG.debug("onRemoveFloatingIp : Successfully removed the label for the prefix {} from VPN {}",
 405                             externalIp, vpnName);
 406                 } else {
 407                     LOG.error("onRemoveFloatingIp : Error in removing the label for prefix {} from VPN {}, {}",
 408                         externalIp, vpnName, result.getErrors());
 409                 }
 410             }
 411         }, MoreExecutors.directExecutor());
 412     }
 413
 414     private String getFlowRef(Uint64 dpnId, short tableId, Uint32 id, String ipAddress) {
 415         return FLOWID_PREFIX + dpnId + NwConstants.FLOWID_SEPARATOR + tableId + NwConstants.FLOWID_SEPARATOR + id
 416                 + NwConstants.FLOWID_SEPARATOR + ipAddress;
 417     }
 418
 419     private void removeTunnelTableEntry(Uint64 dpnId, Uint32 serviceId,
 420             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 421
 422         LOG.debug("removeTunnelTableEntry : called with DpnId = {} and label = {}", dpnId, serviceId);
 423         mdsalManager.removeFlow(confTx, dpnId,
 424             new FlowKey(new FlowId(getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""))),
 425             NwConstants.INTERNAL_TUNNEL_TABLE);
 426         LOG.debug("removeTunnelTableEntry : Terminating service Entry for dpID {} : label : {} removed successfully",
 427                 dpnId, serviceId);
 428     }
 429
 430     private void makeTunnelTableEntry(String vpnName, Uint64 dpnId, Uint32 serviceId,
 431             List<Instruction> customInstructions, TypedWriteTransaction<Configuration> confTx, ProviderTypes provType) {
 432         List<MatchInfo> mkMatches = new ArrayList<>();
 433
 434         LOG.info("makeTunnelTableEntry on DpnId = {} and serviceId = {}", dpnId, serviceId);
 435         int flowPriority = NatConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY;
 436         // Increased the 36->25 flow priority. If SNAT is also configured on the same
 437         // DPN, then the traffic will be hijacked to DNAT and if there are no DNAT match,
 438         // then handled back to using using flow 25->44(which will be installed as part of SNAT)
 439         if (NatUtil.isOpenStackVniSemanticsEnforcedForGreAndVxlan(elanService, provType)) {
 440             mkMatches.add(new MatchTunnelId(Uint64.valueOf(
 441                     natOverVxlanUtil.getInternetVpnVni(vpnName, serviceId).longValue())));
 442             flowPriority = NatConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY + 1;
 443         } else {
 444             mkMatches.add(new MatchTunnelId(Uint64.valueOf(serviceId)));
 445         }
 446         Map<InstructionKey, Instruction> customInstructionsMap = new HashMap<>();
 447         int instructionKey = 0;
 448         for (Instruction instructionObj : customInstructions) {
 449             customInstructionsMap.put(new InstructionKey(++instructionKey), instructionObj);
 450         }
 451         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
 452             getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, serviceId, ""), flowPriority,
 453             String.format("%s:%s", "TST Flow Entry ", serviceId),
 454             0, 0, Uint64.valueOf(COOKIE_TUNNEL.add(BigInteger.valueOf(serviceId.longValue()))),
 455                 mkMatches, customInstructionsMap);
 456
 457         mdsalManager.addFlow(confTx, dpnId, terminatingServiceTableFlowEntity);
 458     }
 459
 460     private void makeLFibTableEntry(Uint64 dpId, Uint32 serviceId, String floatingIpPortMacAddress, short tableId,
 461                                     TypedWriteTransaction<Configuration> confTx) {
 462         List<MatchInfo> matches = new ArrayList<>();
 463         matches.add(MatchEthernetType.MPLS_UNICAST);
 464         matches.add(new MatchMplsLabel(serviceId.longValue()));
 465
 466         Map<InstructionKey, Instruction> instructionMap = new HashMap<>();
 467         int instructionKey = 0;
 468         List<ActionInfo> actionsInfos = new ArrayList<>();
 469         //NAT is required for IPv4 only. Hence always etherType will be IPv4
 470         actionsInfos.add(new ActionPopMpls(NwConstants.ETHTYPE_IPV4));
 471         actionsInfos.add(new ActionSetFieldEthernetDestination(new MacAddress(floatingIpPortMacAddress)));
 472         Instruction writeInstruction = new InstructionApplyActions(actionsInfos).buildInstruction(0);
 473         instructionMap.put(new InstructionKey(++instructionKey), writeInstruction);
 474         instructionMap.put(new InstructionKey(++instructionKey),
 475                 new InstructionGotoTable(tableId).buildInstruction(1));
 476
 477         // Install the flow entry in L3_LFIB_TABLE
 478         String flowRef = getFlowRef(dpId, NwConstants.L3_LFIB_TABLE, serviceId, "");
 479
 480         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.L3_LFIB_TABLE, flowRef,
 481             10, flowRef, 0, 0,
 482             NwConstants.COOKIE_VM_LFIB_TABLE, matches, instructionMap);
 483
 484         mdsalManager.addFlow(confTx, dpId, flowEntity);
 485
 486         LOG.debug("makeLFibTableEntry : LFIB Entry for dpID {} : label : {} modified successfully", dpId, serviceId);
 487     }
 488
 489     private void removeLFibTableEntry(Uint64 dpnId, Uint32 serviceId,
 490             TypedReadWriteTransaction<Configuration> confTx) throws ExecutionException, InterruptedException {
 491
 492         String flowRef = getFlowRef(dpnId, NwConstants.L3_LFIB_TABLE, serviceId, "");
 493
 494         LOG.debug("removeLFibTableEntry : removing LFib entry with flow ref {}", flowRef);
 495
 496         mdsalManager.removeFlow(confTx, dpnId, new FlowKey(new FlowId(flowRef)), NwConstants.L3_LFIB_TABLE);
 497
 498         LOG.debug("removeLFibTableEntry : LFIB Entry for dpID : {} label : {} removed successfully",
 499                 dpnId, serviceId);
 500     }
 501
 502     // TODO Clean up the exception handling
 503     @SuppressWarnings("checkstyle:IllegalCatch")
 504     private void sendGarpOnInterface(final Uint64 dpnId, Uuid networkId, final IpAddress floatingIpAddress,
 505                                      String floatingIpPortMacAddress) {
 506         if (floatingIpAddress.getIpv4Address() == null) {
 507             LOG.error("sendGarpOnInterface : Failed to send GARP for IP. recieved IPv6.");
 508             natServiceCounters.garpFailedIpv6();
 509             return;
 510         }
 511
 512         String interfaceName = elanService.getExternalElanInterface(networkId.getValue(), dpnId);
 513         if (interfaceName == null) {
 514             LOG.warn("sendGarpOnInterface : Failed to send GARP for IP. Failed to retrieve interface name "
 515                     + "from network {} and dpn id {}.", networkId.getValue(), dpnId);
 516             natServiceCounters.garpFailedMissingInterface();
 517             return;
 518         }
 519
 520         try {
 521             // find the external network interface name for dpn
 522             List<InterfaceAddress> interfaceAddresses = new ArrayList<>();
 523             interfaceAddresses.add(new InterfaceAddressBuilder()
 524                 .setInterface(interfaceName)
 525                 .setIpAddress(floatingIpAddress)
 526                 .setMacaddress(new PhysAddress(floatingIpPortMacAddress)).build());
 527
 528             SendArpRequestInput sendArpRequestInput = new SendArpRequestInputBuilder().setIpaddress(floatingIpAddress)
 529                 .setInterfaceAddress(interfaceAddresses).build();
 530
 531             LoggingFutures.addErrorLogging(arpUtilService.sendArpRequest(sendArpRequestInput), LOG, "sendArpRequest");
 532             natServiceCounters.garpSent();
 533         } catch (Exception e) {
 534             LOG.error("sendGarpOnInterface : Failed to send GARP request for floating ip {} from interface {}",
 535                 floatingIpAddress.getIpv4Address().getValue(), interfaceName, e);
 536             natServiceCounters.garpFailedSend();
 537         }
 538     }
 539
 540     // TODO Clean up the exception handling
 541     @SuppressWarnings("checkstyle:IllegalCatch")
 542     private void removeFromFloatingIpPortInfo(Uuid floatingIpId) {
 543         InstanceIdentifier id = buildfloatingIpIdToPortMappingIdentifier(floatingIpId);
 544         try {
 545             Optional<FloatingIpIdToPortMapping> optFloatingIpIdToPortMapping =
 546                     SingleTransactionDataBroker.syncReadOptionalAndTreatReadFailedExceptionAsAbsentOptional(dataBroker,
 547                             LogicalDatastoreType.CONFIGURATION, id);
 548             if (optFloatingIpIdToPortMapping.isPresent() && optFloatingIpIdToPortMapping.get().isFloatingIpDeleted()) {
 549                 LOG.debug("Deleting floating IP UUID {} to Floating IP neutron port mapping from Floating "
 550                     + "IP Port Info Config DS", floatingIpId.getValue());
 551                 MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
 552             }
 553         } catch (Exception e) {
 554             LOG.error("removeFromFloatingIpPortInfo : Deleting floating IP UUID {} to Floating IP neutron port "
 555                     + "mapping from Floating IP Port Info Config DS failed", floatingIpId.getValue(), e);
 556         }
 557     }
 558 }