git.opendaylight Code Review - netvirt.git/blob

   1 /*
   2  * Copyright (c) 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.netvirt.natservice.internal;
  10
  11 import com.google.common.base.Optional;
  12 import com.google.common.util.concurrent.FutureCallback;
  13 import com.google.common.util.concurrent.Futures;
  14 import com.google.common.util.concurrent.ListenableFuture;
  15 import com.google.common.util.concurrent.MoreExecutors;
  16 import java.math.BigInteger;
  17 import java.util.ArrayList;
  18 import java.util.Collections;
  19 import java.util.List;
  20 import javax.annotation.Nonnull;
  21 import javax.inject.Inject;
  22 import javax.inject.Singleton;
  23 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  24 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  25 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  26 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  27 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  28 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  29 import org.opendaylight.genius.mdsalutil.ActionInfo;
  30 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  31 import org.opendaylight.genius.mdsalutil.MatchInfo;
  32 import org.opendaylight.genius.mdsalutil.NwConstants;
  33 import org.opendaylight.genius.mdsalutil.actions.ActionNxResubmit;
  34 import org.opendaylight.genius.mdsalutil.actions.ActionSetFieldEthernetDestination;
  35 import org.opendaylight.genius.mdsalutil.instructions.InstructionApplyActions;
  36 import org.opendaylight.genius.mdsalutil.instructions.InstructionGotoTable;
  37 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  38 import org.opendaylight.genius.mdsalutil.matches.MatchTunnelId;
  39 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  40 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  41 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  42 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  43 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  44 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  45 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  46 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceBuilder;
  47 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  48 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.MacAddress;
  49 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.inventory.rev130819.tables.table.Flow;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInput;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryInputBuilder;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.CreateFibEntryOutput;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.FibRpcService;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInput;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryInputBuilder;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fib.rpc.rev160121.RemoveFibEntryOutput;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOpBuilder;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  68 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  69 import org.opendaylight.yangtools.yang.common.RpcResult;
  70 import org.slf4j.Logger;
  71 import org.slf4j.LoggerFactory;
  72
  73 @Singleton
  74 public class EvpnDnatFlowProgrammer {
  75     private static final Logger LOG = LoggerFactory.getLogger(EvpnDnatFlowProgrammer.class);
  76
  77     private static final BigInteger COOKIE_TUNNEL = new BigInteger("9000000", 16);
  78
  79     private final DataBroker dataBroker;
  80     private final ManagedNewTransactionRunner txRunner;
  81     private final IMdsalApiManager mdsalManager;
  82     private final IBgpManager bgpManager;
  83     private final IFibManager fibManager;
  84     private final FibRpcService fibService;
  85     private final IVpnManager vpnManager;
  86     private final IdManagerService idManager;
  87
  88     @Inject
  89     public EvpnDnatFlowProgrammer(final DataBroker dataBroker, final IMdsalApiManager mdsalManager,
  90                            final IBgpManager bgpManager,
  91                            final IFibManager fibManager,
  92                            final FibRpcService fibService,
  93                            final IVpnManager vpnManager,
  94                            final IdManagerService idManager) {
  95         this.dataBroker = dataBroker;
  96         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
  97         this.mdsalManager = mdsalManager;
  98         this.bgpManager = bgpManager;
  99         this.fibManager = fibManager;
 100         this.fibService = fibService;
 101         this.vpnManager = vpnManager;
 102         this.idManager = idManager;
 103     }
 104
 105     public void onAddFloatingIp(final BigInteger dpnId, final String routerName, final long routerId,
 106                                 final String vpnName,
 107                                 final String internalIp, final String externalIp, final Uuid networkId,
 108                                 final String interfaceName,
 109                                 final String floatingIpInterface,
 110                                 final String floatingIpPortMacAddress,
 111                                 final String rd,
 112                                 final String nextHopIp, final WriteTransaction writeFlowInvTx) {
 113     /*
 114      *  1) Install the flow INTERNAL_TUNNEL_TABLE (table=36)-> PDNAT_TABLE (table=25) (SNAT VM on DPN1 is
 115      *     responding back to FIP VM on DPN2) {SNAT to DNAT traffic on different Hypervisor}
 116      *
 117      *  2) Install the flow L3_FIB_TABLE (table=21)-> PDNAT_TABLE (table=25) (FIP VM1 to FIP VM2
 118      *    Traffic on Same Hypervisor) {DNAT to DNAT on Same Hypervisor}
 119      *
 120      *  3) Install the flow L3_GW_MAC_TABLE (table=19)-> PDNAT_TABLE (table=25)
 121      *    (DC-GW is responding back to FIP VM) {DNAT Reverse traffic})
 122      *
 123      */
 124         long vpnId = NatUtil.getVpnId(dataBroker, vpnName);
 125         if (vpnId == NatConstants.INVALID_ID) {
 126             LOG.error("onAddFloatingIp : Invalid Vpn Id is found for Vpn Name {}", vpnName);
 127             return;
 128         }
 129         long l3Vni = NatEvpnUtil.getL3Vni(dataBroker, rd);
 130         if (l3Vni == NatConstants.DEFAULT_L3VNI_VALUE) {
 131             LOG.debug("onAddFloatingIp : L3VNI value is not configured in Internet VPN {} and RD {} "
 132                     + "Carve-out L3VNI value from OpenDaylight VXLAN VNI Pool and continue with installing "
 133                     + "DNAT flows for FloatingIp {}", vpnName, rd, externalIp);
 134             l3Vni = NatOverVxlanUtil.getInternetVpnVni(idManager, vpnName, routerId).longValue();
 135         }
 136         FloatingIPListener.updateOperationalDS(dataBroker, routerName, interfaceName, NatConstants.DEFAULT_LABEL_VALUE,
 137                 internalIp, externalIp);
 138         String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
 139         //Inform to FIB and BGP
 140         NatEvpnUtil.addRoutesForVxLanProvType(dataBroker, bgpManager, fibManager, vpnName, rd, fibExternalIp,
 141                 nextHopIp, l3Vni, floatingIpInterface, floatingIpPortMacAddress,
 142                 writeFlowInvTx, RouteOrigin.STATIC, dpnId);
 143
 144         /* Install the flow table L3_FIB_TABLE (table=21)-> PDNAT_TABLE (table=25)
 145          * (SNAT to DNAT reverse traffic: If the DPN has both SNAT and  DNAT configured )
 146          */
 147         List<ActionInfo> actionInfoFib = new ArrayList<>();
 148         actionInfoFib.add(new ActionSetFieldEthernetDestination(new MacAddress(floatingIpPortMacAddress)));
 149         List<Instruction> instructionsFib = new ArrayList<>();
 150         instructionsFib.add(new InstructionApplyActions(actionInfoFib).buildInstruction(0));
 151         instructionsFib.add(new InstructionGotoTable(NwConstants.PDNAT_TABLE).buildInstruction(1));
 152
 153         CreateFibEntryInput input = new CreateFibEntryInputBuilder().setVpnName(vpnName)
 154                 .setSourceDpid(dpnId).setIpAddress(fibExternalIp)
 155                 .setServiceId(l3Vni).setIpAddressSource(CreateFibEntryInput.IpAddressSource.FloatingIP)
 156                 .setInstruction(instructionsFib).build();
 157
 158         ListenableFuture<RpcResult<CreateFibEntryOutput>> futureVxlan = fibService.createFibEntry(input);
 159         LOG.debug("onAddFloatingIp : Add Floating Ip {} , found associated to fixed port {}",
 160                 externalIp, interfaceName);
 161         if (floatingIpPortMacAddress != null) {
 162             ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
 163                 vpnManager.addSubnetMacIntoVpnInstance(vpnName, null, floatingIpPortMacAddress, dpnId, tx);
 164                 vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 165                         Collections.singleton(externalIp),
 166                         floatingIpPortMacAddress, dpnId, networkId, tx);
 167             }), LOG, "Error processing floating IP port with MAC address {}", floatingIpPortMacAddress);
 168         }
 169         final long finalL3Vni = l3Vni;
 170         Futures.addCallback(futureVxlan, new FutureCallback<RpcResult<CreateFibEntryOutput>>() {
 171
 172             @Override
 173             public void onFailure(@Nonnull Throwable error) {
 174                 LOG.error("onAddFloatingIp : Error {} in custom fib routes install process for Floating "
 175                         + "IP Prefix {} on DPN {}", error, externalIp, dpnId);
 176             }
 177
 178             @Override
 179             public void onSuccess(@Nonnull RpcResult<CreateFibEntryOutput> result) {
 180                 if (result.isSuccessful()) {
 181                     LOG.info("onAddFloatingIp : Successfully installed custom FIB routes for Floating "
 182                             + "IP Prefix {} on DPN {}", externalIp, dpnId);
 183                     List<Instruction> instructions = new ArrayList<>();
 184                     List<ActionInfo> actionsInfos = new ArrayList<>();
 185                     List<Instruction> customInstructions = new ArrayList<>();
 186                     customInstructions.add(new InstructionGotoTable(NwConstants.PDNAT_TABLE).buildInstruction(0));
 187                     actionsInfos.add(new ActionNxResubmit(NwConstants.PDNAT_TABLE));
 188                     instructions.add(new InstructionApplyActions(actionsInfos).buildInstruction(0));
 189                  /* If more than one floatingIp is available in vpn-to-dpn-list for given dpn id, do not call for
 190                   * installing INTERNAL_TUNNEL_TABLE (table=36) -> PDNAT_TABLE (table=25) flow entry with same tunnel_id
 191                   * again and again.
 192                   */
 193                     if (!NatUtil.isFloatingIpPresentForDpn(dataBroker, dpnId, rd, vpnName, externalIp, true)) {
 194                         makeTunnelTableEntry(dpnId, finalL3Vni, instructions, writeFlowInvTx);
 195                     }
 196                  /* Install the flow L3_GW_MAC_TABLE (table=19)-> PDNAT_TABLE (table=25)
 197                   * (DNAT reverse traffic: If the traffic is Initiated from DC-GW to FIP VM (DNAT forward traffic))
 198                   */
 199                     NatEvpnUtil.makeL3GwMacTableEntry(dpnId, vpnId, floatingIpPortMacAddress, customInstructions,
 200                             mdsalManager, writeFlowInvTx);
 201                     NatUtil.waitForTransactionToComplete(writeFlowInvTx);
 202                 } else {
 203                     LOG.error("onAddFloatingIp : Error {} in rpc call to create custom Fib entries for Floating "
 204                             + "IP Prefix {} on DPN {}", result.getErrors(), externalIp, dpnId);
 205                 }
 206             }
 207         }, MoreExecutors.directExecutor());
 208
 209         //Read the FIP vpn-interface details from Configuration l3vpn:vpn-interfaces model and write into Operational DS
 210         InstanceIdentifier<VpnInterface> vpnIfIdentifier = NatUtil.getVpnInterfaceIdentifier(floatingIpInterface);
 211         Optional<VpnInterface> optionalVpnInterface =
 212                 SingleTransactionDataBroker.syncReadOptionalAndTreatReadFailedExceptionAsAbsentOptional(dataBroker,
 213                         LogicalDatastoreType.CONFIGURATION, vpnIfIdentifier);
 214         if (optionalVpnInterface.isPresent()) {
 215             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 216                 for (VpnInstanceNames vpnInstance : optionalVpnInterface.get().getVpnInstanceNames()) {
 217                     if (!vpnName.equals(vpnInstance.getVpnName())) {
 218                         continue;
 219                     }
 220                     VpnInterfaceBuilder vpnIfBuilder = new VpnInterfaceBuilder(optionalVpnInterface.get());
 221                     Adjacencies adjs = vpnIfBuilder.getAugmentation(Adjacencies.class);
 222                     VpnInterfaceOpDataEntryBuilder vpnIfOpDataEntryBuilder = new VpnInterfaceOpDataEntryBuilder();
 223                     vpnIfOpDataEntryBuilder.setKey(new VpnInterfaceOpDataEntryKey(interfaceName, vpnName));
 224
 225                     List<Adjacency> adjacencyList = adjs != null ? adjs.getAdjacency() : new ArrayList<>();
 226                     List<Adjacency> adjacencyListToImport = new ArrayList<>();
 227                     for (Adjacency adj : adjacencyList) {
 228                         Subnetmap sn = VpnHelper.getSubnetmapFromItsUuid(dataBroker, adj.getSubnetId());
 229                         if (!VpnHelper.isSubnetPartOfVpn(sn, vpnName)) {
 230                             continue;
 231                         }
 232                         adjacencyListToImport.add(adj);
 233                     }
 234                     AdjacenciesOp adjacenciesOp = new AdjacenciesOpBuilder()
 235                             .setAdjacency(adjacencyListToImport).build();
 236                     vpnIfOpDataEntryBuilder.addAugmentation(AdjacenciesOp.class, adjacenciesOp);
 237
 238                     LOG.debug("onAddFloatingIp : Add vpnInterface {} to Operational l3vpn:vpn-interfaces-op-data ",
 239                             floatingIpInterface);
 240                     InstanceIdentifier<VpnInterfaceOpDataEntry> vpnIfIdentifierOpDataEntry =
 241                             NatUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 242                     tx.put(LogicalDatastoreType.OPERATIONAL, vpnIfIdentifierOpDataEntry,
 243                             vpnIfOpDataEntryBuilder.build(), WriteTransaction.CREATE_MISSING_PARENTS);
 244                     break;
 245                 }
 246             }), LOG, "onAddFloatingIp : Could not write Interface {}, vpnName {}", interfaceName, vpnName);
 247         } else {
 248             LOG.debug("onAddFloatingIp : No vpnInterface {} found in Configuration l3vpn:vpn-interfaces ",
 249                            floatingIpInterface);
 250         }
 251     }
 252
 253     public void onRemoveFloatingIp(final BigInteger dpnId, final String vpnName, final String externalIp,
 254                                    final String floatingIpInterface, final String floatingIpPortMacAddress,
 255                                    final long routerId, WriteTransaction removeFlowInvTx) {
 256     /*
 257      *  1) Remove the flow INTERNAL_TUNNEL_TABLE (table=36)-> PDNAT_TABLE (table=25) (SNAT VM on DPN1 is
 258      *     responding back to FIP VM on DPN2) {SNAT to DNAT traffic on different Hypervisor}
 259      *
 260      *  2) Remove the flow L3_FIB_TABLE (table=21)-> PDNAT_TABLE (table=25) (FIP VM1 to FIP VM2
 261      *    Traffic on Same Hypervisor) {DNAT to DNAT on Same Hypervisor}
 262      *
 263      *  3) Remove the flow L3_GW_MAC_TABLE (table=19)-> PDNAT_TABLE (table=25)
 264      *    (DC-GW is responding back to FIP VM) {DNAT Reverse traffic})
 265      *
 266      */
 267         String rd = NatUtil.getVpnRd(dataBroker, vpnName);
 268         if (rd == null) {
 269             LOG.error("onRemoveFloatingIp : Could not retrieve RD value from VPN Name {}  ", vpnName);
 270             return;
 271         }
 272         long vpnId = NatUtil.getVpnId(dataBroker, vpnName);
 273         if (vpnId == NatConstants.INVALID_ID) {
 274             LOG.error("onRemoveFloatingIp : Invalid Vpn Id is found for Vpn Name {}", vpnName);
 275             return;
 276         }
 277         long l3Vni = NatEvpnUtil.getL3Vni(dataBroker, rd);
 278         if (l3Vni == NatConstants.DEFAULT_L3VNI_VALUE) {
 279             LOG.debug("onRemoveFloatingIp : L3VNI value is not configured in Internet VPN {} and RD {} "
 280                     + "Carve-out L3VNI value from OpenDaylight VXLAN VNI Pool and continue with installing "
 281                     + "DNAT flows for FloatingIp {}", vpnName, rd, externalIp);
 282             l3Vni = NatOverVxlanUtil.getInternetVpnVni(idManager, vpnName, routerId).longValue();
 283         }
 284         String fibExternalIp = NatUtil.validateAndAddNetworkMask(externalIp);
 285
 286         //Remove Prefix from BGP
 287         NatUtil.removePrefixFromBGP(bgpManager, fibManager, rd, fibExternalIp, vpnName, LOG);
 288
 289         //Remove custom FIB routes flow for L3_FIB_TABLE (table=21)-> PDNAT_TABLE (table=25)
 290         RemoveFibEntryInput input = new RemoveFibEntryInputBuilder().setVpnName(vpnName)
 291                 .setSourceDpid(dpnId).setIpAddress(fibExternalIp).setServiceId(l3Vni)
 292                 .setIpAddressSource(RemoveFibEntryInput.IpAddressSource.FloatingIP).build();
 293         ListenableFuture<RpcResult<RemoveFibEntryOutput>> futureVxlan = fibService.removeFibEntry(input);
 294         final long finalL3Vni = l3Vni;
 295         Futures.addCallback(futureVxlan, new FutureCallback<RpcResult<RemoveFibEntryOutput>>() {
 296
 297             @Override
 298             public void onFailure(@Nonnull Throwable error) {
 299                 LOG.error("onRemoveFloatingIp : Error {} in custom fib routes remove process for Floating "
 300                         + "IP Prefix {} on DPN {}", error, externalIp, dpnId);
 301             }
 302
 303             @Override
 304             public void onSuccess(@Nonnull RpcResult<RemoveFibEntryOutput> result) {
 305                 if (result.isSuccessful()) {
 306                     LOG.info("onRemoveFloatingIp : Successfully removed custom FIB routes for Floating "
 307                             + "IP Prefix {} on DPN {}", externalIp, dpnId);
 308                      /*  check if any floating IP information is available in vpn-to-dpn-list for given dpn id.
 309                       *  If exist any floating IP then do not remove
 310                       *  INTERNAL_TUNNEL_TABLE (table=36) -> PDNAT_TABLE (table=25) flow entry.
 311                       */
 312                     if (!NatUtil.isFloatingIpPresentForDpn(dataBroker, dpnId, rd, vpnName, externalIp, false)) {
 313                         //Remove the flow for INTERNAL_TUNNEL_TABLE (table=36)-> PDNAT_TABLE (table=25)
 314                         removeTunnelTableEntry(dpnId, finalL3Vni, removeFlowInvTx);
 315                     }
 316                     //Remove the flow for L3_GW_MAC_TABLE (table=19)-> PDNAT_TABLE (table=25)
 317                     NatEvpnUtil.removeL3GwMacTableEntry(dpnId, vpnId, floatingIpPortMacAddress, mdsalManager,
 318                             removeFlowInvTx);
 319                     NatUtil.waitForTransactionToComplete(removeFlowInvTx);
 320                 } else {
 321                     LOG.error("onRemoveFloatingIp : Error {} in rpc call to remove custom Fib entries for Floating "
 322                             + "IP Prefix {} on DPN {}", result.getErrors(), externalIp, dpnId);
 323                 }
 324             }
 325         }, MoreExecutors.directExecutor());
 326         //Read the FIP vpn-interface details from Operational l3vpn:vpn-interfaces model and delete from Operational DS
 327         InstanceIdentifier<VpnInterface> vpnIfIdentifier = NatUtil.getVpnInterfaceIdentifier(floatingIpInterface);
 328         Optional<VpnInterface> optionalVpnInterface =
 329                 SingleTransactionDataBroker.syncReadOptionalAndTreatReadFailedExceptionAsAbsentOptional(dataBroker,
 330                         LogicalDatastoreType.CONFIGURATION, vpnIfIdentifier);
 331         if (optionalVpnInterface.isPresent()) {
 332             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 333                 for (VpnInstanceNames vpnInstance : optionalVpnInterface.get().getVpnInstanceNames()) {
 334                     if (!vpnName.equals(vpnInstance.getVpnName())) {
 335                         continue;
 336                     }
 337                     InstanceIdentifier<VpnInterfaceOpDataEntry> vpnOpIfIdentifier = NatUtil
 338                             .getVpnInterfaceOpDataEntryIdentifier(floatingIpInterface, vpnName);
 339                     tx.delete(LogicalDatastoreType.OPERATIONAL, vpnOpIfIdentifier);
 340                     break;
 341                 }
 342             }), LOG, "onRemoveFloatingIp : Could not remove vpnInterface {}, vpnName {} from Operational "
 343                     + "odl-l3vpn:vpn-interface-op-data", floatingIpInterface, vpnName);
 344
 345             LOG.debug("onRemoveFloatingIp : Remove vpnInterface {} vpnName {} "
 346                      + "to Operational odl-l3vpn:vpn-interface-op-data", floatingIpInterface, vpnName);
 347         } else {
 348             LOG.debug("onRemoveFloatingIp : No vpnInterface {} found "
 349                     + "in Operational odl-l3vpn:vpn-interface-op-data", floatingIpInterface);
 350         }
 351     }
 352
 353     private void makeTunnelTableEntry(BigInteger dpnId, long l3Vni, List<Instruction> customInstructions,
 354                                       WriteTransaction writeFlowInvTx) {
 355         LOG.debug("makeTunnelTableEntry : Create terminating service table {} --> table {} flow on DpnId {} "
 356                 + "with l3Vni {} as matching parameter", NwConstants.INTERNAL_TUNNEL_TABLE, NwConstants.PDNAT_TABLE,
 357                 dpnId, l3Vni);
 358         List<MatchInfo> mkMatches = new ArrayList<>();
 359         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(l3Vni)));
 360         Flow terminatingServiceTableFlowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
 361                 NatEvpnUtil.getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, l3Vni, NatConstants.DNAT_FLOW_NAME), 6,
 362                 String.format("%s:%d", "TST Flow Entry ", l3Vni),
 363                 0, 0, COOKIE_TUNNEL.add(BigInteger.valueOf(l3Vni)), mkMatches, customInstructions);
 364         mdsalManager.addFlowToTx(dpnId, terminatingServiceTableFlowEntity, writeFlowInvTx);
 365         LOG.debug("makeTunnelTableEntry : Successfully installed terminating service table flow {} on DpnId {}",
 366                 terminatingServiceTableFlowEntity, dpnId);
 367     }
 368
 369     private void removeTunnelTableEntry(BigInteger dpnId, long l3Vni, WriteTransaction removeFlowInvTx) {
 370         LOG.debug("removeTunnelTableEntry : Remove terminating service table {} --> table {} flow on DpnId {} "
 371                 + "with l3Vni {} as matching parameter", NwConstants.INTERNAL_TUNNEL_TABLE, NwConstants.PDNAT_TABLE,
 372                 dpnId, l3Vni);
 373         List<MatchInfo> mkMatches = new ArrayList<>();
 374         mkMatches.add(new MatchTunnelId(BigInteger.valueOf(l3Vni)));
 375         Flow flowEntity = MDSALUtil.buildFlowNew(NwConstants.INTERNAL_TUNNEL_TABLE,
 376                 NatEvpnUtil.getFlowRef(dpnId, NwConstants.INTERNAL_TUNNEL_TABLE, l3Vni, NatConstants.DNAT_FLOW_NAME),
 377                 6, String.format("%s:%d", "TST Flow Entry ", l3Vni), 0, 0,
 378                 COOKIE_TUNNEL.add(BigInteger.valueOf(l3Vni)), mkMatches, null);
 379         mdsalManager.removeFlowToTx(dpnId, flowEntity, removeFlowInvTx);
 380         LOG.debug("removeTunnelTableEntry : Successfully removed terminating service table flow {} on DpnId {}",
 381                 flowEntity, dpnId);
 382     }
 383 }