natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/EvpnDnatFlowProgrammer.java

   1 /*
   2  * Copyright (c) 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.netvirt.natservice.internal;
  10
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13
  14 import com.google.common.util.concurrent.FutureCallback;
  15 import com.google.common.util.concurrent.Futures;
  16 import com.google.common.util.concurrent.ListenableFuture;
  17 import com.google.common.util.concurrent.MoreExecutors;
  18 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  19 import java.math.BigInteger;
  20 import java.util.ArrayList;
  21 import java.util.Collections;
  22 import java.util.HashMap;
  23 import java.util.List;
  24 import java.util.Map;
  25 import java.util.Optional;
  26 import java.util.concurrent.ExecutionException;
  27 import javax.inject.Inject;
  28 import javax.inject.Singleton;
  29 import org.eclipse.jdt.annotation.NonNull;
  30 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  31 import org.opendaylight.genius.infra.Datastore.Configuration;
  32 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  33 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  34 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  35 import org.opendaylight.genius.infra.TypedWriteTransaction;
  36 import org.opendaylight.genius.mdsalutil.ActionInfo;
  37 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  38 import org.opendaylight.genius.mdsalutil.MatchInfo;
  39 import org.opendaylight.genius.mdsalutil.NwConstants;
  40 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  41 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  42 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  43 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  44 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  45 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  46 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  47 import org.opendaylight.mdsal.binding.api.DataBroker;
  48 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  49 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  50 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  51 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  52 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  53 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  54 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  55 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.InstructionKey;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOpBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.Adjacencies;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.Adjacency;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.AdjacencyKey;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.VpnInterface;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.VpnInterfaceBuilder;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.vpn._interface.VpnInstanceNames;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  78 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  79 import org.opendaylight.yangtools.yang.common.RpcResult;
  80 import org.opendaylight.yangtools.yang.common.Uint32;
  81 import org.opendaylight.yangtools.yang.common.Uint64;
  82 import org.slf4j.Logger;
  83 import org.slf4j.LoggerFactory;
  84
  85 @Singleton
  86 public class EvpnDnatFlowProgrammer {
  87     private static final Logger LOG = LoggerFactory.getLogger(EvpnDnatFlowProgrammer.class);
  88
  89     private static final Uint64 COOKIE_TUNNEL = Uint64.valueOf("9000000", 16).intern();
  90
  91     private final DataBroker dataBroker;
  92     private final ManagedNewTransactionRunner txRunner;
  93     private final IMdsalApiManager mdsalManager;
  94     private final IBgpManager bgpManager;
  95     private final IFibManager fibManager;
  96     private final FibRpcService fibService;
  97     private final IVpnManager vpnManager;
  98     private final NatOverVxlanUtil natOverVxlanUtil;
  99
 100     @Inject
 101     public EvpnDnatFlowProgrammer(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
 102                            final IBgpManager bgpManager,
 103                            final IFibManager fibManager,
 104                            final FibRpcService fibService,
 105                            final IVpnManager vpnManager,
 106                            final NatOverVxlanUtil natOverVxlanUtil) {
 107         this.dataBroker = dataBroker;
 108         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 109         this.mdsalManager = mdsalManager;
 110         this.bgpManager = bgpManager;
 111         this.fibManager = fibManager;
 112         this.fibService = fibService;
 113         this.vpnManager = vpnManager;
 114         this.natOverVxlanUtil = natOverVxlanUtil;
 115     }
 116
 117     public void onAddFloatingIp(final Uint64 dpnId, final String routerName, final Uint32 routerId,
 118                                 final String vpnName,
 119                                 final String internalIp, final String externalIp, final Uuid networkId,
 120                                 final String interfaceName,
 121                                 final String floatingIpInterface,
 122                                 final String floatingIpPortMacAddress,
 123                                 final String rd,
 124                                 final String nextHopIp, final TypedReadWriteTransaction<Configuration> confTx) {
 125     /*
 126      *  1) Install the flow INTERNAL_TUNNEL_TABLE (table=36)-> PDNAT_TABLE (table=25) (SNAT VM on DPN1 is
 127      *     responding back to FIP VM on DPN2) {SNAT to DNAT traffic on different Hypervisor}
 128      *
 129      *  2) Install the flow L3_FIB_TABLE (table=21)-> PDNAT_TABLE (table=25) (FIP VM1 to FIP VM2
 130      *    Traffic on Same Hypervisor) {DNAT to DNAT on Same Hypervisor}
 131      *
 132      *  3) Install the flow L3_GW_MAC_TABLE (table=19)-> PDNAT_TABLE (table=25)
 133      *    (DC-GW is responding back to FIP VM) {DNAT Reverse traffic})
 134      *
 135      */
 136         Uint32 vpnId = NatUtil.getVpnId(dataBroker, vpnName);
 137         if (vpnId == NatConstants.INVALID_ID) {
 138             LOG.error("onAddFloatingIp : Invalid Vpn Id is found for Vpn Name {}", vpnName);
 139             return;
 140         }
 141         Uint32 l3Vni = NatEvpnUtil.getL3Vni(dataBroker, rd);
 142         if (l3Vni == NatConstants.DEFAULT_L3VNI_VALUE) {
 143             LOG.debug("onAddFloatingIp : L3VNI value is not configured in Internet VPN {} and RD {} "
 144                     + "Carve-out L3VNI value from OpenDaylight VXLAN VNI Pool and continue with installing "
 145                     + "DNAT flows for FloatingIp {}", vpnName, rd, externalIp);
 146             l3Vni = natOverVxlanUtil.getInternetVpnVni(vpnName, routerId);
 147         }
 148         FloatingIPListener.updateOperationalDS(dataBroker, routerName, interfaceName, NatConstants.DEFAULT_LABEL_VALUE,
 149                 internalIp, externalIp);
 150         String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
 151         //Inform to FIB and BGP
 152         NatEvpnUtil.addRoutesForVxLanProvType(dataBroker, bgpManager, fibManager, vpnName, rd, fibExternalIp,
 153                 nextHopIp, l3Vni, floatingIpInterface, floatingIpPortMacAddress,
 154                 confTx, RouteOrigin.STATIC, dpnId, networkId);
 155
 156         /* Install the flow table L3_FIB_TABLE (table=21)-> PDNAT_TABLE (table=25)
 157          * (SNAT to DNAT reverse traffic: If the DPN has both SNAT and  DNAT configured )
 158          */
 159         List<ActionInfo> actionInfoFib = new ArrayList<>();
 160         actionInfoFib.add(new ActionSetFieldEthernetDestination(new MacAddress(floatingIpPortMacAddress)));
 161         List<Instruction> instructionsFib = new ArrayList<>();
 162         instructionsFib.add(new InstructionApplyActions(actionInfoFib).buildInstruction(0));
 163         instructionsFib.add(new InstructionGotoTable(NwConstants.PDNAT_TABLE).buildInstruction(1));
 164
 165         CreateFibEntryInput input = new CreateFibEntryInputBuilder().setVpnName(vpnName)
 166                 .setSourceDpid(dpnId).setIpAddress(fibExternalIp)
 167                 .setServiceId(l3Vni).setIpAddressSource(CreateFibEntryInput.IpAddressSource.FloatingIP)
 168                 .setInstruction(instructionsFib).build();
 169
 170         ListenableFuture<RpcResult<CreateFibEntryOutput>> futureVxlan = fibService.createFibEntry(input);
 171         LOG.debug("onAddFloatingIp : Add Floating Ip {} , found associated to fixed port {}",
 172                 externalIp, interfaceName);
 173         if (floatingIpPortMacAddress != null) {
 174             ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 175                 vpnManager.addSubnetMacIntoVpnInstance(vpnName, null, floatingIpPortMacAddress, dpnId, tx);
 176                 vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 177                         Collections.singleton(externalIp),
 178                         floatingIpPortMacAddress, dpnId, networkId);
 179             }), LOG, "Error processing floating IP port with MAC address {}", floatingIpPortMacAddress);
 180         }
 181         final Uint32 finalL3Vni = l3Vni;
 182         Futures.addCallback(futureVxlan, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
 183
 184             @Override
 185             public void onFailure(@NonNull Throwable error) {
 186                 LOG.error("onAddFloatingIp : Error {} in custom fib routes install process for Floating "
 187                         + "IP Prefix {} on DPN {}", error, externalIp, dpnId);
 188             }
 189
 190             @Override
 191             public void onSuccess(@NonNull RpcResult<CreateFibEntryOutput> result) {
 192                 if (result.isSuccessful()) {
 193                     ListenableFutures.addErrorLogging(
 194                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, innerConfTx -> {
 195                             LOG.info("onAddFloatingIp : Successfully installed custom FIB routes for Floating "
 196                                 + "IP Prefix {} on DPN {}", externalIp, dpnId);
 197                             List<Instruction> instructions = new ArrayList<>();
 198                             List<ActionInfo> actionsInfos = new ArrayList<>();
 199                             List<Instruction> customInstructions = new ArrayList<>();
 200                             customInstructions.add(
 201                                 new InstructionGotoTable(NwConstants.PDNAT_TABLE).buildInstruction(0));
 202                             actionsInfos.add(new ActionNxResubmit(NwConstants.PDNAT_TABLE));
 203                             instructions.add(new InstructionApplyActions(actionsInfos).buildInstruction(0));
 204                             /* If more than one floatingIp is available in vpn-to-dpn-list for given dpn id, do not
 205                             call for
 206                              * installing INTERNAL_TUNNEL_TABLE (table=36) -> PDNAT_TABLE (table=25) flow entry with
 207                              * same tunnel_id
 208                              * again and again.
 209                              */
 210                             if (!NatUtil.isFloatingIpPresentForDpn(dataBroker, dpnId, rd, vpnName, externalIp, true)) {
 211                                 makeTunnelTableEntry(dpnId, finalL3Vni, instructions, innerConfTx);
 212                             }
 213                             /* Install the flow L3_GW_MAC_TABLE (table=19)-> PDNAT_TABLE (table=25)
 214                              * (DNAT reverse traffic: If the traffic is Initiated from DC-GW to FIP VM (DNAT forward
 215                              * traffic))
 216                              */
 217                             NatEvpnUtil.makeL3GwMacTableEntry(dpnId, vpnId, floatingIpPortMacAddress,
 218                                 customInstructions,
 219                                 mdsalManager, innerConfTx);
 220                         }), LOG, "Error installing DNAT flows");
 221                 } else {
 222                     LOG.error("onAddFloatingIp : Error {} in rpc call to create custom Fib entries for Floating "
 223                             + "IP Prefix {} on DPN {}", result.getErrors(), externalIp, dpnId);
 224                 }
 225             }
 226         }, MoreExecutors.directExecutor());
 227
 228         //Read the FIP vpn-interface details from Configuration l3vpn:vpn-interfaces model and write into Operational DS
 229         InstanceIdentifier<VpnInterface> vpnIfIdentifier = NatUtil.getVpnInterfaceIdentifier(floatingIpInterface);
 230         Optional<VpnInterface> optionalVpnInterface =
 231                 SingleTransactionDataBroker.syncReadOptionalAndTreatReadFailedExceptionAsAbsentOptional(dataBroker,
 232                         LogicalDatastoreType.CONFIGURATION, vpnIfIdentifier);
 233         if (optionalVpnInterface.isPresent()) {
 234             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, tx -> {
 235                 for (VpnInstanceNames vpnInstance : optionalVpnInterface.get().nonnullVpnInstanceNames().values()) {
 236                     if (!vpnName.equals(vpnInstance.getVpnName())) {
 237                         continue;
 238                     }
 239                     VpnInterfaceBuilder vpnIfBuilder = new VpnInterfaceBuilder(optionalVpnInterface.get());
 240                     Adjacencies adjs = vpnIfBuilder.augmentation(Adjacencies.class);
 241                     VpnInterfaceOpDataEntryBuilder vpnIfOpDataEntryBuilder = new VpnInterfaceOpDataEntryBuilder();
 242                     vpnIfOpDataEntryBuilder.withKey(new VpnInterfaceOpDataEntryKey(interfaceName, vpnName));
 243
 244                     Map<AdjacencyKey, Adjacency> keyAdjacencyMap =
 245                         adjs != null && adjs.getAdjacency() != null ? adjs.getAdjacency()
 246                                 : new HashMap<AdjacencyKey, Adjacency>();
 247                     List<Adjacency> adjacencyListToImport = new ArrayList<>();
 248                     for (Adjacency adj : keyAdjacencyMap.values()) {
 249                         Subnetmap sn = VpnHelper.getSubnetmapFromItsUuid(dataBroker, adj.getSubnetId());
 250                         if (!VpnHelper.isSubnetPartOfVpn(sn, vpnName)) {
 251                             continue;
 252                         }
 253                         adjacencyListToImport.add(adj);
 254                     }
 255                     AdjacenciesOp adjacenciesOp = new AdjacenciesOpBuilder()
 256                             .setAdjacency(adjacencyListToImport).build();
 257                     vpnIfOpDataEntryBuilder.addAugmentation(AdjacenciesOp.class, adjacenciesOp);
 258
 259                     LOG.debug("onAddFloatingIp : Add vpnInterface {} to Operational l3vpn:vpn-interfaces-op-data ",
 260                             floatingIpInterface);
 261                     InstanceIdentifier<VpnInterfaceOpDataEntry> vpnIfIdentifierOpDataEntry =
 262                             NatUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 263                     tx.mergeParentStructurePut(vpnIfIdentifierOpDataEntry, vpnIfOpDataEntryBuilder.build());
 264                     break;
 265                 }
 266             }), LOG, "onAddFloatingIp : Could not write Interface {}, vpnName {}", interfaceName, vpnName);
 267         } else {
 268             LOG.debug("onAddFloatingIp : No vpnInterface {} found in Configuration l3vpn:vpn-interfaces ",
 269                            floatingIpInterface);
 270         }
 271     }
 272
 273     public void onRemoveFloatingIp(final Uint64 dpnId, final String vpnName, final String externalIp,
 274                                    final String floatingIpInterface, final String floatingIpPortMacAddress,
 275                                    final Uint32 routerId) {
 276     /*
 277      *  1) Remove the flow INTERNAL_TUNNEL_TABLE (table=36)-> PDNAT_TABLE (table=25) (SNAT VM on DPN1 is
 278      *     responding back to FIP VM on DPN2) {SNAT to DNAT traffic on different Hypervisor}
 279      *
 280      *  2) Remove the flow L3_FIB_TABLE (table=21)-> PDNAT_TABLE (table=25) (FIP VM1 to FIP VM2
 281      *    Traffic on Same Hypervisor) {DNAT to DNAT on Same Hypervisor}
 282      *
 283      *  3) Remove the flow L3_GW_MAC_TABLE (table=19)-> PDNAT_TABLE (table=25)
 284      *    (DC-GW is responding back to FIP VM) {DNAT Reverse traffic})
 285      *
 286      */
 287         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
 288         if (rd == null) {
 289             LOG.error("onRemoveFloatingIp : Could not retrieve RD value from VPN Name {}  ", vpnName);
 290             return;
 291         }
 292         Uint32 vpnId = NatUtil.getVpnId(dataBroker, vpnName);
 293         if (vpnId == NatConstants.INVALID_ID) {
 294             LOG.error("onRemoveFloatingIp : Invalid Vpn Id is found for Vpn Name {}", vpnName);
 295             return;
 296         }
 297         Uint32 l3Vni = NatEvpnUtil.getL3Vni(dataBroker, rd);
 298         if (l3Vni == NatConstants.DEFAULT_L3VNI_VALUE) {
 299             LOG.debug("onRemoveFloatingIp : L3VNI value is not configured in Internet VPN {} and RD {} "
 300                     + "Carve-out L3VNI value from OpenDaylight VXLAN VNI Pool and continue with installing "
 301                     + "DNAT flows for FloatingIp {}", vpnName, rd, externalIp);
 302             l3Vni = natOverVxlanUtil.getInternetVpnVni(vpnName, routerId);
 303         }
 304         String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
 305
 306         //Remove Prefix from BGP
 307         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, fibExternalIp, vpnName);
 308
 309         //Remove custom FIB routes flow for L3_FIB_TABLE (table=21)-> PDNAT_TABLE (table=25)
 310         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(vpnName)
 311                 .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(l3Vni)
 312                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.FloatingIP).build();
 313         ListenableFuture<RpcResult<RemoveFibEntryOutput>> futureVxlan = fibService.removeFibEntry(input);
 314         final Uint32 finalL3Vni = l3Vni;
 315         Futures.addCallback(futureVxlan, new FutureCallback<RpcResult<RemoveFibEntryOutput>>() {
 316
 317             @Override
 318             public void onFailure(@NonNull Throwable error) {
 319                 LOG.error("onRemoveFloatingIp : Error {} in custom fib routes remove process for Floating "
 320                         + "IP Prefix {} on DPN {}", error, externalIp, dpnId);
 321             }
 322
 323             @Override
 324             public void onSuccess(@NonNull RpcResult<RemoveFibEntryOutput> result) {
 325                 if (result.isSuccessful()) {
 326                     ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 327                         innerConfTx -> {
 328                             LOG.info("onRemoveFloatingIp : Successfully removed custom FIB routes for Floating "
 329                                 + "IP Prefix {} on DPN {}", externalIp, dpnId);
 330                             /*  check if any floating IP information is available in vpn-to-dpn-list for given dpn id.
 331                              *  If exist any floating IP then do not remove
 332                              *  INTERNAL_TUNNEL_TABLE (table=36) -> PDNAT_TABLE (table=25) flow entry.
 333                              */
 334                             if (!NatUtil.isFloatingIpPresentForDpn(dataBroker, dpnId, rd, vpnName, externalIp, false)) {
 335                                 //Remove the flow for INTERNAL_TUNNEL_TABLE (table=36)-> PDNAT_TABLE (table=25)
 336                                 removeTunnelTableEntry(dpnId, finalL3Vni, innerConfTx);
 337                             }
 338                             //Remove the flow for L3_GW_MAC_TABLE (table=19)-> PDNAT_TABLE (table=25)
 339                             NatEvpnUtil.removeL3GwMacTableEntry(dpnId, vpnId, floatingIpPortMacAddress, mdsalManager,
 340                                 innerConfTx);
 341                         }), LOG, "Error removing flows");
 342                 } else {
 343                     LOG.error("onRemoveFloatingIp : Error {} in rpc call to remove custom Fib entries for Floating "
 344                             + "IP Prefix {} on DPN {}", result.getErrors(), externalIp, dpnId);
 345                 }
 346             }
 347         }, MoreExecutors.directExecutor());
 348         //Read the FIP vpn-interface details from Operational l3vpn:vpn-interfaces model and delete from Operational DS
 349         InstanceIdentifier<VpnInterface> vpnIfIdentifier = NatUtil.getVpnInterfaceIdentifier(floatingIpInterface);
 350         Optional<VpnInterface> optionalVpnInterface =
 351                 SingleTransactionDataBroker.syncReadOptionalAndTreatReadFailedExceptionAsAbsentOptional(dataBroker,
 352                         LogicalDatastoreType.CONFIGURATION, vpnIfIdentifier);
 353         if (optionalVpnInterface.isPresent()) {
 354             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, tx -> {
 355                 for (VpnInstanceNames vpnInstance : optionalVpnInterface.get().nonnullVpnInstanceNames().values()) {
 356                     if (!vpnName.equals(vpnInstance.getVpnName())) {
 357                         continue;
 358                     }
 359                     InstanceIdentifier<VpnInterfaceOpDataEntry> vpnOpIfIdentifier = NatUtil
 360                             .getVpnInterfaceOpDataEntryIdentifier(floatingIpInterface, vpnName);
 361                     tx.delete(vpnOpIfIdentifier);
 362                     break;
 363                 }
 364             }), LOG, "onRemoveFloatingIp : Could not remove vpnInterface {}, vpnName {} from Operational "
 365                     + "odl-l3vpn:vpn-interface-op-data", floatingIpInterface, vpnName);
 366
 367             LOG.debug("onRemoveFloatingIp : Remove vpnInterface {} vpnName {} "
 368                      + "to Operational odl-l3vpn:vpn-interface-op-data", floatingIpInterface, vpnName);
 369         } else {
 370             LOG.debug("onRemoveFloatingIp : No vpnInterface {} found "
 371                     + "in Operational odl-l3vpn:vpn-interface-op-data", floatingIpInterface);
 372         }
 373     }
 374
 375     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
 376             justification = "https://github.com/spotbugs/spotbugs/issues/811")
 377     private void makeTunnelTableEntry(Uint64 dpnId, Uint32 l3Vni, List<Instruction> customInstructions,
 378                                       TypedWriteTransaction<Configuration> confTx) {
 379         LOG.debug("makeTunnelTableEntry : Create terminating service table {} --> table {} flow on DpnId {} "
 380                 + "with l3Vni {} as matching parameter", NwConstants.INTERNAL_TUNNEL_TABLE, NwConstants.PDNAT_TABLE,
 381                 dpnId, l3Vni);
 382         List<MatchInfo> mkMatches = new ArrayList<>();
 383         mkMatches.add(new MatchTunnelId(Uint64.valueOf(l3Vni)));
 384         Map<InstructionKey, Instruction> customInstructionsMap = new HashMap<InstructionKey, Instruction>();
 385         int instructionKey = 0;
 386         for (Instruction instructionObj : customInstructions) {
 387             customInstructionsMap.put(new InstructionKey(++instructionKey), instructionObj);
 388         }
 389         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
 390                 NatEvpnUtil.getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, l3Vni, NatConstants.DNAT_FLOW_NAME),
 391                 NatConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY + 1,
 392                 String.format("%s:%s", "TST Flow Entry ", l3Vni),
 393                 0, 0,
 394                 Uint64.valueOf(COOKIE_TUNNEL.toJava().add(BigInteger.valueOf(l3Vni.longValue()))),
 395                 mkMatches, customInstructionsMap);
 396         mdsalManager.addFlow(confTx, dpnId, terminatingServiceTableFlowEntity);
 397         LOG.debug("makeTunnelTableEntry : Successfully installed terminating service table flow {} on DpnId {}",
 398                 terminatingServiceTableFlowEntity, dpnId);
 399     }
 400
 401     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
 402             justification = "https://github.com/spotbugs/spotbugs/issues/811")
 403     private void removeTunnelTableEntry(Uint64 dpnId, Uint32 l3Vni, TypedReadWriteTransaction<Configuration> confTx)
 404             throws ExecutionException, InterruptedException {
 405         LOG.debug("removeTunnelTableEntry : Remove terminating service table {} --> table {} flow on DpnId {} "
 406                 + "with l3Vni {} as matching parameter", NwConstants.INTERNAL_TUNNEL_TABLE, NwConstants.PDNAT_TABLE,
 407                 dpnId, l3Vni);
 408         List<MatchInfo> mkMatches = new ArrayList<>();
 409         mkMatches.add(new MatchTunnelId(Uint64.valueOf(l3Vni)));
 410         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
 411                 NatEvpnUtil.getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, l3Vni, NatConstants.DNAT_FLOW_NAME),
 412                 NatConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY + 1,
 413                 String.format("%s:%s", "TST Flow Entry ", l3Vni), 0, 0,
 414                 Uint64.valueOf(COOKIE_TUNNEL.toJava().add(BigInteger.valueOf(l3Vni.longValue()))),
 415                 mkMatches, null);
 416         mdsalManager.removeFlow(confTx, dpnId, flowEntity);
 417         LOG.debug("removeTunnelTableEntry : Successfully removed terminating service table flow {} on DpnId {}",
 418                 flowEntity, dpnId);
 419     }
 420 }