natservice/impl/src/main/java/org/opendaylight/netvirt/natservice/internal/EvpnDnatFlowProgrammer.java

   1 /*
   2  * Copyright (c) 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.netvirt.natservice.internal;
  10
  11 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  12 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  13 import static org.opendaylight.mdsal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  14
  15 import com.google.common.util.concurrent.FutureCallback;
  16 import com.google.common.util.concurrent.Futures;
  17 import com.google.common.util.concurrent.ListenableFuture;
  18 import com.google.common.util.concurrent.MoreExecutors;
  19 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  20 import java.math.BigInteger;
  21 import java.util.ArrayList;
  22 import java.util.Collections;
  23 import java.util.List;
  24 import java.util.Optional;
  25 import java.util.concurrent.ExecutionException;
  26 import javax.inject.Inject;
  27 import javax.inject.Singleton;
  28 import org.eclipse.jdt.annotation.NonNull;
  29 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  30 import org.opendaylight.genius.infra.Datastore.Configuration;
  31 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  32 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  33 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  34 import org.opendaylight.genius.infra.TypedWriteTransaction;
  35 import org.opendaylight.genius.mdsalutil.ActionInfo;
  36 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  37 import org.opendaylight.genius.mdsalutil.MatchInfo;
  38 import org.opendaylight.genius.mdsalutil.NwConstants;
  39 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  40 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  41 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  42 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  43 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  44 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  45 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  46 import org.opendaylight.mdsal.binding.api.DataBroker;
  47 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  48 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  49 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  50 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  51 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  52 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  53 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  54 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOpBuilder;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.Adjacencies;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.Adjacency;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.VpnInterface;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.VpnInterfaceBuilder;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.vpn._interface.VpnInstanceNames;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  75 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  76 import org.opendaylight.yangtools.yang.common.RpcResult;
  77 import org.opendaylight.yangtools.yang.common.Uint32;
  78 import org.opendaylight.yangtools.yang.common.Uint64;
  79 import org.slf4j.Logger;
  80 import org.slf4j.LoggerFactory;
  81
  82 @Singleton
  83 public class EvpnDnatFlowProgrammer {
  84     private static final Logger LOG = LoggerFactory.getLogger(EvpnDnatFlowProgrammer.class);
  85
  86     private static final Uint64 COOKIE_TUNNEL = Uint64.valueOf("9000000", 16).intern();
  87
  88     private final DataBroker dataBroker;
  89     private final ManagedNewTransactionRunner txRunner;
  90     private final IMdsalApiManager mdsalManager;
  91     private final IBgpManager bgpManager;
  92     private final IFibManager fibManager;
  93     private final FibRpcService fibService;
  94     private final IVpnManager vpnManager;
  95     private final NatOverVxlanUtil natOverVxlanUtil;
  96
  97     @Inject
  98     public EvpnDnatFlowProgrammer(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
  99                            final IBgpManager bgpManager,
 100                            final IFibManager fibManager,
 101                            final FibRpcService fibService,
 102                            final IVpnManager vpnManager,
 103                            final NatOverVxlanUtil natOverVxlanUtil) {
 104         this.dataBroker = dataBroker;
 105         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 106         this.mdsalManager = mdsalManager;
 107         this.bgpManager = bgpManager;
 108         this.fibManager = fibManager;
 109         this.fibService = fibService;
 110         this.vpnManager = vpnManager;
 111         this.natOverVxlanUtil = natOverVxlanUtil;
 112     }
 113
 114     public void onAddFloatingIp(final Uint64 dpnId, final String routerName, final Uint32 routerId,
 115                                 final String vpnName,
 116                                 final String internalIp, final String externalIp, final Uuid networkId,
 117                                 final String interfaceName,
 118                                 final String floatingIpInterface,
 119                                 final String floatingIpPortMacAddress,
 120                                 final String rd,
 121                                 final String nextHopIp, final TypedReadWriteTransaction<Configuration> confTx) {
 122     /*
 123      *  1) Install the flow INTERNAL_TUNNEL_TABLE (table=36)-> PDNAT_TABLE (table=25) (SNAT VM on DPN1 is
 124      *     responding back to FIP VM on DPN2) {SNAT to DNAT traffic on different Hypervisor}
 125      *
 126      *  2) Install the flow L3_FIB_TABLE (table=21)-> PDNAT_TABLE (table=25) (FIP VM1 to FIP VM2
 127      *    Traffic on Same Hypervisor) {DNAT to DNAT on Same Hypervisor}
 128      *
 129      *  3) Install the flow L3_GW_MAC_TABLE (table=19)-> PDNAT_TABLE (table=25)
 130      *    (DC-GW is responding back to FIP VM) {DNAT Reverse traffic})
 131      *
 132      */
 133         Uint32 vpnId = NatUtil.getVpnId(dataBroker, vpnName);
 134         if (vpnId == NatConstants.INVALID_ID) {
 135             LOG.error("onAddFloatingIp : Invalid Vpn Id is found for Vpn Name {}", vpnName);
 136             return;
 137         }
 138         Uint32 l3Vni = NatEvpnUtil.getL3Vni(dataBroker, rd);
 139         if (l3Vni == NatConstants.DEFAULT_L3VNI_VALUE) {
 140             LOG.debug("onAddFloatingIp : L3VNI value is not configured in Internet VPN {} and RD {} "
 141                     + "Carve-out L3VNI value from OpenDaylight VXLAN VNI Pool and continue with installing "
 142                     + "DNAT flows for FloatingIp {}", vpnName, rd, externalIp);
 143             l3Vni = natOverVxlanUtil.getInternetVpnVni(vpnName, routerId);
 144         }
 145         FloatingIPListener.updateOperationalDS(dataBroker, routerName, interfaceName, NatConstants.DEFAULT_LABEL_VALUE,
 146                 internalIp, externalIp);
 147         String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
 148         //Inform to FIB and BGP
 149         NatEvpnUtil.addRoutesForVxLanProvType(dataBroker, bgpManager, fibManager, vpnName, rd, fibExternalIp,
 150                 nextHopIp, l3Vni, floatingIpInterface, floatingIpPortMacAddress,
 151                 confTx, RouteOrigin.STATIC, dpnId, networkId);
 152
 153         /* Install the flow table L3_FIB_TABLE (table=21)-> PDNAT_TABLE (table=25)
 154          * (SNAT to DNAT reverse traffic: If the DPN has both SNAT and  DNAT configured )
 155          */
 156         List<ActionInfo> actionInfoFib = new ArrayList<>();
 157         actionInfoFib.add(new ActionSetFieldEthernetDestination(new MacAddress(floatingIpPortMacAddress)));
 158         List<Instruction> instructionsFib = new ArrayList<>();
 159         instructionsFib.add(new InstructionApplyActions(actionInfoFib).buildInstruction(0));
 160         instructionsFib.add(new InstructionGotoTable(NwConstants.PDNAT_TABLE).buildInstruction(1));
 161
 162         CreateFibEntryInput input = new CreateFibEntryInputBuilder().setVpnName(vpnName)
 163                 .setSourceDpid(dpnId).setIpAddress(fibExternalIp)
 164                 .setServiceId(l3Vni).setIpAddressSource(CreateFibEntryInput.IpAddressSource.FloatingIP)
 165                 .setInstruction(instructionsFib).build();
 166
 167         ListenableFuture<RpcResult<CreateFibEntryOutput>> futureVxlan = fibService.createFibEntry(input);
 168         LOG.debug("onAddFloatingIp : Add Floating Ip {} , found associated to fixed port {}",
 169                 externalIp, interfaceName);
 170         if (floatingIpPortMacAddress != null) {
 171             ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, tx -> {
 172                 vpnManager.addSubnetMacIntoVpnInstance(vpnName, null, floatingIpPortMacAddress, dpnId, tx);
 173                 vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 174                         Collections.singleton(externalIp),
 175                         floatingIpPortMacAddress, dpnId, networkId);
 176             }), LOG, "Error processing floating IP port with MAC address {}", floatingIpPortMacAddress);
 177         }
 178         final Uint32 finalL3Vni = l3Vni;
 179         Futures.addCallback(futureVxlan, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
 180
 181             @Override
 182             public void onFailure(@NonNull Throwable error) {
 183                 LOG.error("onAddFloatingIp : Error {} in custom fib routes install process for Floating "
 184                         + "IP Prefix {} on DPN {}", error, externalIp, dpnId);
 185             }
 186
 187             @Override
 188             public void onSuccess(@NonNull RpcResult<CreateFibEntryOutput> result) {
 189                 if (result.isSuccessful()) {
 190                     ListenableFutures.addErrorLogging(
 191                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, innerConfTx -> {
 192                             LOG.info("onAddFloatingIp : Successfully installed custom FIB routes for Floating "
 193                                 + "IP Prefix {} on DPN {}", externalIp, dpnId);
 194                             List<Instruction> instructions = new ArrayList<>();
 195                             List<ActionInfo> actionsInfos = new ArrayList<>();
 196                             List<Instruction> customInstructions = new ArrayList<>();
 197                             customInstructions.add(
 198                                 new InstructionGotoTable(NwConstants.PDNAT_TABLE).buildInstruction(0));
 199                             actionsInfos.add(new ActionNxResubmit(NwConstants.PDNAT_TABLE));
 200                             instructions.add(new InstructionApplyActions(actionsInfos).buildInstruction(0));
 201                             /* If more than one floatingIp is available in vpn-to-dpn-list for given dpn id, do not
 202                             call for
 203                              * installing INTERNAL_TUNNEL_TABLE (table=36) -> PDNAT_TABLE (table=25) flow entry with
 204                              * same tunnel_id
 205                              * again and again.
 206                              */
 207                             if (!NatUtil.isFloatingIpPresentForDpn(dataBroker, dpnId, rd, vpnName, externalIp, true)) {
 208                                 makeTunnelTableEntry(dpnId, finalL3Vni, instructions, innerConfTx);
 209                             }
 210                             /* Install the flow L3_GW_MAC_TABLE (table=19)-> PDNAT_TABLE (table=25)
 211                              * (DNAT reverse traffic: If the traffic is Initiated from DC-GW to FIP VM (DNAT forward
 212                              * traffic))
 213                              */
 214                             NatEvpnUtil.makeL3GwMacTableEntry(dpnId, vpnId, floatingIpPortMacAddress,
 215                                 customInstructions,
 216                                 mdsalManager, innerConfTx);
 217                         }), LOG, "Error installing DNAT flows");
 218                 } else {
 219                     LOG.error("onAddFloatingIp : Error {} in rpc call to create custom Fib entries for Floating "
 220                             + "IP Prefix {} on DPN {}", result.getErrors(), externalIp, dpnId);
 221                 }
 222             }
 223         }, MoreExecutors.directExecutor());
 224
 225         //Read the FIP vpn-interface details from Configuration l3vpn:vpn-interfaces model and write into Operational DS
 226         InstanceIdentifier<VpnInterface> vpnIfIdentifier = NatUtil.getVpnInterfaceIdentifier(floatingIpInterface);
 227         Optional<VpnInterface> optionalVpnInterface =
 228                 SingleTransactionDataBroker.syncReadOptionalAndTreatReadFailedExceptionAsAbsentOptional(dataBroker,
 229                         LogicalDatastoreType.CONFIGURATION, vpnIfIdentifier);
 230         if (optionalVpnInterface.isPresent()) {
 231             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, tx -> {
 232                 for (VpnInstanceNames vpnInstance : optionalVpnInterface.get().nonnullVpnInstanceNames()) {
 233                     if (!vpnName.equals(vpnInstance.getVpnName())) {
 234                         continue;
 235                     }
 236                     VpnInterfaceBuilder vpnIfBuilder = new VpnInterfaceBuilder(optionalVpnInterface.get());
 237                     Adjacencies adjs = vpnIfBuilder.augmentation(Adjacencies.class);
 238                     VpnInterfaceOpDataEntryBuilder vpnIfOpDataEntryBuilder = new VpnInterfaceOpDataEntryBuilder();
 239                     vpnIfOpDataEntryBuilder.withKey(new VpnInterfaceOpDataEntryKey(interfaceName, vpnName));
 240
 241                     List<Adjacency> adjacencyList =
 242                         adjs != null && adjs.getAdjacency() != null ? adjs.getAdjacency() : new ArrayList<>();
 243                     List<Adjacency> adjacencyListToImport = new ArrayList<>();
 244                     for (Adjacency adj : adjacencyList) {
 245                         Subnetmap sn = VpnHelper.getSubnetmapFromItsUuid(dataBroker, adj.getSubnetId());
 246                         if (!VpnHelper.isSubnetPartOfVpn(sn, vpnName)) {
 247                             continue;
 248                         }
 249                         adjacencyListToImport.add(adj);
 250                     }
 251                     AdjacenciesOp adjacenciesOp = new AdjacenciesOpBuilder()
 252                             .setAdjacency(adjacencyListToImport).build();
 253                     vpnIfOpDataEntryBuilder.addAugmentation(AdjacenciesOp.class, adjacenciesOp);
 254
 255                     LOG.debug("onAddFloatingIp : Add vpnInterface {} to Operational l3vpn:vpn-interfaces-op-data ",
 256                             floatingIpInterface);
 257                     InstanceIdentifier<VpnInterfaceOpDataEntry> vpnIfIdentifierOpDataEntry =
 258                             NatUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 259                     tx.put(vpnIfIdentifierOpDataEntry, vpnIfOpDataEntryBuilder.build(), CREATE_MISSING_PARENTS);
 260                     break;
 261                 }
 262             }), LOG, "onAddFloatingIp : Could not write Interface {}, vpnName {}", interfaceName, vpnName);
 263         } else {
 264             LOG.debug("onAddFloatingIp : No vpnInterface {} found in Configuration l3vpn:vpn-interfaces ",
 265                            floatingIpInterface);
 266         }
 267     }
 268
 269     public void onRemoveFloatingIp(final Uint64 dpnId, final String vpnName, final String externalIp,
 270                                    final String floatingIpInterface, final String floatingIpPortMacAddress,
 271                                    final Uint32 routerId) {
 272     /*
 273      *  1) Remove the flow INTERNAL_TUNNEL_TABLE (table=36)-> PDNAT_TABLE (table=25) (SNAT VM on DPN1 is
 274      *     responding back to FIP VM on DPN2) {SNAT to DNAT traffic on different Hypervisor}
 275      *
 276      *  2) Remove the flow L3_FIB_TABLE (table=21)-> PDNAT_TABLE (table=25) (FIP VM1 to FIP VM2
 277      *    Traffic on Same Hypervisor) {DNAT to DNAT on Same Hypervisor}
 278      *
 279      *  3) Remove the flow L3_GW_MAC_TABLE (table=19)-> PDNAT_TABLE (table=25)
 280      *    (DC-GW is responding back to FIP VM) {DNAT Reverse traffic})
 281      *
 282      */
 283         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
 284         if (rd == null) {
 285             LOG.error("onRemoveFloatingIp : Could not retrieve RD value from VPN Name {}  ", vpnName);
 286             return;
 287         }
 288         Uint32 vpnId = NatUtil.getVpnId(dataBroker, vpnName);
 289         if (vpnId == NatConstants.INVALID_ID) {
 290             LOG.error("onRemoveFloatingIp : Invalid Vpn Id is found for Vpn Name {}", vpnName);
 291             return;
 292         }
 293         Uint32 l3Vni = NatEvpnUtil.getL3Vni(dataBroker, rd);
 294         if (l3Vni == NatConstants.DEFAULT_L3VNI_VALUE) {
 295             LOG.debug("onRemoveFloatingIp : L3VNI value is not configured in Internet VPN {} and RD {} "
 296                     + "Carve-out L3VNI value from OpenDaylight VXLAN VNI Pool and continue with installing "
 297                     + "DNAT flows for FloatingIp {}", vpnName, rd, externalIp);
 298             l3Vni = natOverVxlanUtil.getInternetVpnVni(vpnName, routerId);
 299         }
 300         String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
 301
 302         //Remove Prefix from BGP
 303         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, fibExternalIp, vpnName);
 304
 305         //Remove custom FIB routes flow for L3_FIB_TABLE (table=21)-> PDNAT_TABLE (table=25)
 306         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(vpnName)
 307                 .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(l3Vni)
 308                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.FloatingIP).build();
 309         ListenableFuture<RpcResult<RemoveFibEntryOutput>> futureVxlan = fibService.removeFibEntry(input);
 310         final Uint32 finalL3Vni = l3Vni;
 311         Futures.addCallback(futureVxlan, new FutureCallback<RpcResult<RemoveFibEntryOutput>>() {
 312
 313             @Override
 314             public void onFailure(@NonNull Throwable error) {
 315                 LOG.error("onRemoveFloatingIp : Error {} in custom fib routes remove process for Floating "
 316                         + "IP Prefix {} on DPN {}", error, externalIp, dpnId);
 317             }
 318
 319             @Override
 320             public void onSuccess(@NonNull RpcResult<RemoveFibEntryOutput> result) {
 321                 if (result.isSuccessful()) {
 322                     ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
 323                         innerConfTx -> {
 324                             LOG.info("onRemoveFloatingIp : Successfully removed custom FIB routes for Floating "
 325                                 + "IP Prefix {} on DPN {}", externalIp, dpnId);
 326                             /*  check if any floating IP information is available in vpn-to-dpn-list for given dpn id.
 327                              *  If exist any floating IP then do not remove
 328                              *  INTERNAL_TUNNEL_TABLE (table=36) -> PDNAT_TABLE (table=25) flow entry.
 329                              */
 330                             if (!NatUtil.isFloatingIpPresentForDpn(dataBroker, dpnId, rd, vpnName, externalIp, false)) {
 331                                 //Remove the flow for INTERNAL_TUNNEL_TABLE (table=36)-> PDNAT_TABLE (table=25)
 332                                 removeTunnelTableEntry(dpnId, finalL3Vni, innerConfTx);
 333                             }
 334                             //Remove the flow for L3_GW_MAC_TABLE (table=19)-> PDNAT_TABLE (table=25)
 335                             NatEvpnUtil.removeL3GwMacTableEntry(dpnId, vpnId, floatingIpPortMacAddress, mdsalManager,
 336                                 innerConfTx);
 337                         }), LOG, "Error removing flows");
 338                 } else {
 339                     LOG.error("onRemoveFloatingIp : Error {} in rpc call to remove custom Fib entries for Floating "
 340                             + "IP Prefix {} on DPN {}", result.getErrors(), externalIp, dpnId);
 341                 }
 342             }
 343         }, MoreExecutors.directExecutor());
 344         //Read the FIP vpn-interface details from Operational l3vpn:vpn-interfaces model and delete from Operational DS
 345         InstanceIdentifier<VpnInterface> vpnIfIdentifier = NatUtil.getVpnInterfaceIdentifier(floatingIpInterface);
 346         Optional<VpnInterface> optionalVpnInterface =
 347                 SingleTransactionDataBroker.syncReadOptionalAndTreatReadFailedExceptionAsAbsentOptional(dataBroker,
 348                         LogicalDatastoreType.CONFIGURATION, vpnIfIdentifier);
 349         if (optionalVpnInterface.isPresent()) {
 350             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, tx -> {
 351                 for (VpnInstanceNames vpnInstance : optionalVpnInterface.get().nonnullVpnInstanceNames()) {
 352                     if (!vpnName.equals(vpnInstance.getVpnName())) {
 353                         continue;
 354                     }
 355                     InstanceIdentifier<VpnInterfaceOpDataEntry> vpnOpIfIdentifier = NatUtil
 356                             .getVpnInterfaceOpDataEntryIdentifier(floatingIpInterface, vpnName);
 357                     tx.delete(vpnOpIfIdentifier);
 358                     break;
 359                 }
 360             }), LOG, "onRemoveFloatingIp : Could not remove vpnInterface {}, vpnName {} from Operational "
 361                     + "odl-l3vpn:vpn-interface-op-data", floatingIpInterface, vpnName);
 362
 363             LOG.debug("onRemoveFloatingIp : Remove vpnInterface {} vpnName {} "
 364                      + "to Operational odl-l3vpn:vpn-interface-op-data", floatingIpInterface, vpnName);
 365         } else {
 366             LOG.debug("onRemoveFloatingIp : No vpnInterface {} found "
 367                     + "in Operational odl-l3vpn:vpn-interface-op-data", floatingIpInterface);
 368         }
 369     }
 370
 371     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
 372             justification = "https://github.com/spotbugs/spotbugs/issues/811")
 373     private void makeTunnelTableEntry(Uint64 dpnId, Uint32 l3Vni, List<Instruction> customInstructions,
 374                                       TypedWriteTransaction<Configuration> confTx) {
 375         LOG.debug("makeTunnelTableEntry : Create terminating service table {} --> table {} flow on DpnId {} "
 376                 + "with l3Vni {} as matching parameter", NwConstants.INTERNAL_TUNNEL_TABLE, NwConstants.PDNAT_TABLE,
 377                 dpnId, l3Vni);
 378         List<MatchInfo> mkMatches = new ArrayList<>();
 379         mkMatches.add(new MatchTunnelId(Uint64.valueOf(l3Vni)));
 380         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
 381                 NatEvpnUtil.getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, l3Vni, NatConstants.DNAT_FLOW_NAME),
 382                 NatConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY + 1,
 383                 String.format("%s:%s", "TST Flow Entry ", l3Vni),
 384                 0, 0,
 385                 Uint64.valueOf(COOKIE_TUNNEL.toJava().add(BigInteger.valueOf(l3Vni.longValue()))),
 386                 mkMatches, customInstructions);
 387         mdsalManager.addFlow(confTx, dpnId, terminatingServiceTableFlowEntity);
 388         LOG.debug("makeTunnelTableEntry : Successfully installed terminating service table flow {} on DpnId {}",
 389                 terminatingServiceTableFlowEntity, dpnId);
 390     }
 391
 392     @SuppressFBWarnings(value = "UPM_UNCALLED_PRIVATE_METHOD",
 393             justification = "https://github.com/spotbugs/spotbugs/issues/811")
 394     private void removeTunnelTableEntry(Uint64 dpnId, Uint32 l3Vni, TypedReadWriteTransaction<Configuration> confTx)
 395             throws ExecutionException, InterruptedException {
 396         LOG.debug("removeTunnelTableEntry : Remove terminating service table {} --> table {} flow on DpnId {} "
 397                 + "with l3Vni {} as matching parameter", NwConstants.INTERNAL_TUNNEL_TABLE, NwConstants.PDNAT_TABLE,
 398                 dpnId, l3Vni);
 399         List<MatchInfo> mkMatches = new ArrayList<>();
 400         mkMatches.add(new MatchTunnelId(Uint64.valueOf(l3Vni)));
 401         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
 402                 NatEvpnUtil.getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, l3Vni, NatConstants.DNAT_FLOW_NAME),
 403                 NatConstants.DEFAULT_VPN_INTERNAL_TUNNEL_TABLE_PRIORITY + 1,
 404                 String.format("%s:%s", "TST Flow Entry ", l3Vni), 0, 0,
 405                 Uint64.valueOf(COOKIE_TUNNEL.toJava().add(BigInteger.valueOf(l3Vni.longValue()))),
 406                 mkMatches, null);
 407         mdsalManager.removeFlow(confTx, dpnId, flowEntity);
 408         LOG.debug("removeTunnelTableEntry : Successfully removed terminating service table flow {} on DpnId {}",
 409                 flowEntity, dpnId);
 410     }
 411 }