renderers/vpp/src/main/java/org/opendaylight/groupbasedpolicy/renderer/vpp/policy/ForwardingManager.java

   1 /*
   2  * Copyright (c) 2016 Cisco Systems, Inc. and others. All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8
   9 package org.opendaylight.groupbasedpolicy.renderer.vpp.policy;
  10
  11 import java.util.ArrayList;
  12 import java.util.HashMap;
  13 import java.util.List;
  14 import java.util.Map;
  15 import java.util.Set;
  16 import java.util.concurrent.ExecutionException;
  17 import java.util.concurrent.TimeUnit;
  18 import java.util.concurrent.TimeoutException;
  19
  20 import javax.annotation.Nonnull;
  21 import javax.annotation.Nullable;
  22
  23 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  24 import org.opendaylight.controller.md.sal.binding.api.ReadOnlyTransaction;
  25 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  26 import org.opendaylight.groupbasedpolicy.renderer.vpp.api.BridgeDomainManager;
  27 import org.opendaylight.groupbasedpolicy.renderer.vpp.iface.AclManager;
  28 import org.opendaylight.groupbasedpolicy.renderer.vpp.iface.InterfaceManager;
  29 import org.opendaylight.groupbasedpolicy.renderer.vpp.policy.acl.AccessListUtil;
  30 import org.opendaylight.groupbasedpolicy.renderer.vpp.nat.NatManager;
  31 import org.opendaylight.groupbasedpolicy.renderer.vpp.util.KeyFactory;
  32 import org.opendaylight.groupbasedpolicy.util.DataStoreHelper;
  33 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.inet.types.rev130715.IpPrefix;
  34 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.nat.rev150908.nat.config.nat.instances.nat.instance.mapping.table.MappingEntryBuilder;
  35 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.base_endpoint.rev160427.common.endpoint.fields.NetworkContainment;
  36 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.base_endpoint.rev160427.common.endpoint.fields.network.containment.Containment;
  37 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.base_endpoint.rev160427.common.endpoint.fields.network.containment.containment.ForwardingContextContainment;
  38 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.base_endpoint.rev160427.common.endpoint.fields.network.containment.containment.NetworkDomainContainment;
  39 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.base_endpoint.rev160427.has.absolute.location.absolute.location.LocationType;
  40 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.base_endpoint.rev160427.has.absolute.location.absolute.location.location.type.ExternalLocationCase;
  41 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.ContextId;
  42 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.NetworkDomainId;
  43 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.common.rev140421.TenantId;
  44 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.forwarding.l2_l3.rev160427.L2FloodDomain;
  45 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.forwarding.l2_l3.rev160427.SubnetAugmentRenderer;
  46 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.forwarding.rev160427.forwarding.fields.Parent;
  47 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.forwarding.rev160427.NetworkDomain;
  48 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.renderer.rev151103.NatAddressRenderer;
  49 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.renderer.rev151103.renderers.renderer.renderer.policy.Configuration;
  50 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.renderer.rev151103.renderers.renderer.renderer.policy.configuration.Endpoints;
  51 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.renderer.rev151103.renderers.renderer.renderer.policy.configuration.endpoints.AddressEndpointWithLocation;
  52 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.renderer.rev151103.renderers.renderer.renderer.policy.configuration.renderer.endpoints.RendererEndpointKey;
  53 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.renderer.rev151103.renderers.renderer.renderer.policy.configuration.renderer.forwarding.RendererForwardingByTenant;
  54 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.renderer.rev151103.renderers.renderer.renderer.policy.configuration.renderer.forwarding.renderer.forwarding.by.tenant.RendererForwardingContext;
  55 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.renderer.rev151103.renderers.renderer.renderer.policy.configuration.renderer.forwarding.renderer.forwarding.by.tenant.RendererForwardingContextKey;
  56 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.renderer.rev151103.renderers.renderer.renderer.policy.configuration.renderer.forwarding.renderer.forwarding.by.tenant.RendererNetworkDomain;
  57 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.renderer.rev151103.renderers.renderer.renderer.policy.configuration.renderer.forwarding.renderer.forwarding.by.tenant.RendererNetworkDomainKey;
  58 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.vpp_renderer.rev160425.Config;
  59 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.vpp_renderer.rev160425.VlanNetwork;
  60 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.vpp_renderer.rev160425._interface.attributes.InterfaceTypeChoice;
  61 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.vpp_renderer.rev160425._interface.attributes._interface.type.choice.LoopbackCase;
  62 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.vpp_renderer.rev160425.config.GbpBridgeDomain;
  63 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.vpp_renderer.rev160425.config.GbpBridgeDomainKey;
  64 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.vpp_renderer.rev160425.config.VppEndpoint;
  65 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.vpp_renderer.rev160425.config.VppEndpointKey;
  66 import org.opendaylight.yang.gen.v1.urn.opendaylight.groupbasedpolicy.vpp_renderer.rev160425.renderers.renderer.renderer.nodes.renderer.node.PhysicalInterface;
  67 import org.opendaylight.yang.gen.v1.urn.opendaylight.l2.types.rev130827.VlanId;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.params.xml.ns.yang.v3po.rev161214.VxlanVni;
  69 import org.opendaylight.yang.gen.v1.urn.tbd.params.xml.ns.yang.network.topology.rev131021.NodeId;
  70 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  71 import org.slf4j.Logger;
  72 import org.slf4j.LoggerFactory;
  73
  74 import com.google.common.annotations.VisibleForTesting;
  75 import com.google.common.base.Optional;
  76 import com.google.common.base.Preconditions;
  77 import com.google.common.base.Strings;
  78 import com.google.common.collect.SetMultimap;
  79 import com.google.common.collect.Table;
  80
  81 public final class ForwardingManager {
  82
  83     private static final Logger LOG = LoggerFactory.getLogger(ForwardingManager.class);
  84     @VisibleForTesting
  85     private byte WAIT_FOR_BD_PROCESSING = 60; // seconds
  86     private long lastVxlanVni = 1L;
  87     private final Map<String, VxlanVni> vxlanVniByBridgeDomain = new HashMap<>();
  88     private final InterfaceManager ifaceManager;
  89     private final AclManager aclManager;
  90     private final BridgeDomainManager bdManager;
  91     private final NatManager natManager;
  92     private final DataBroker dataBroker;
  93
  94     public ForwardingManager(@Nonnull InterfaceManager ifaceManager, @Nonnull AclManager aclManager,
  95         @Nonnull NatManager natManager, @Nonnull BridgeDomainManager bdManager, @Nonnull DataBroker dataBroker) {
  96         this.ifaceManager = Preconditions.checkNotNull(ifaceManager);
  97         this.bdManager = Preconditions.checkNotNull(bdManager);
  98         this.natManager = Preconditions.checkNotNull(natManager);
  99         this.dataBroker = Preconditions.checkNotNull(dataBroker);
 100         this.aclManager = Preconditions.checkNotNull(aclManager);
 101     }
 102
 103     public Optional<GbpBridgeDomain> readGbpBridgeDomainConfig(String name) {
 104         InstanceIdentifier<GbpBridgeDomain> bdIid = InstanceIdentifier.builder(Config.class)
 105             .child(GbpBridgeDomain.class, new GbpBridgeDomainKey(name))
 106             .build();
 107         ReadOnlyTransaction rTx = dataBroker.newReadOnlyTransaction();
 108         Optional<GbpBridgeDomain> optBd = DataStoreHelper.readFromDs(LogicalDatastoreType.CONFIGURATION, bdIid, rTx);
 109         rTx.close();
 110         return optBd;
 111     }
 112
 113     public void createBridgeDomainOnNodes(SetMultimap<String, NodeId> vppNodesByBridgeDomain) {
 114         for (String bd : vppNodesByBridgeDomain.keySet()) {
 115             Optional<GbpBridgeDomain> bdConfig = readGbpBridgeDomainConfig(bd);
 116             Set<NodeId> vppNodes = vppNodesByBridgeDomain.get(bd);
 117             if (bdConfig.isPresent()) {
 118                 if (bdConfig.get().getType().equals(VlanNetwork.class)) {
 119                     createVlanBridgeDomains(bd, bdConfig.get().getVlan(), vppNodes);
 120                 }
 121             } else {
 122                 VxlanVni vxlanVni = vxlanVniByBridgeDomain.get(bd);
 123                 if (vxlanVni == null) {
 124                     vxlanVni = new VxlanVni(lastVxlanVni++);
 125                     vxlanVniByBridgeDomain.put(bd, vxlanVni);
 126                 }
 127                 createVxlanBridgeDomains(bd, vxlanVni, vppNodes);
 128             }
 129         }
 130     }
 131
 132     private void createVxlanBridgeDomains(final String bd, final VxlanVni vni, final Set<NodeId> vppNodes) {
 133         for (NodeId vppNode : vppNodes) {
 134             try {
 135                 LOG.debug("Creating VXLAN bridge-domain {} on node {} with VNI {}", bd, vppNode.getValue(),
 136                         vni);
 137                 bdManager.createVxlanBridgeDomainOnVppNode(bd, vni, vppNode).get(WAIT_FOR_BD_PROCESSING,
 138                         TimeUnit.SECONDS);
 139             } catch (InterruptedException | ExecutionException e) {
 140                 LOG.warn("VXLAN Bridge domain {} was not created on node {}", bd, vppNode.getValue(), e);
 141             } catch (TimeoutException e) {
 142                 LOG.warn("Probably, VXLAN Bridge domain {} was not created on node {} because BridgeDomainManager "
 143                         + "did not respond by {} seconds. Check VBD log for more details",
 144                         bd, vppNode.getValue(), WAIT_FOR_BD_PROCESSING, e);
 145             }
 146         }
 147     }
 148
 149     private void createVlanBridgeDomains(final String bd, final VlanId vlanId, final Set<NodeId> vppNodes) {
 150         for (NodeId vppNode : vppNodes) {
 151             try {
 152                 LOG.debug("Creating VLAN bridge-domain {} on node {} with VLAN ID {}", bd, vppNode.getValue(),
 153                         vlanId.getValue());
 154                 bdManager.createVlanBridgeDomainOnVppNode(bd, vlanId, vppNode).get(WAIT_FOR_BD_PROCESSING,
 155                         TimeUnit.SECONDS);
 156             } catch (InterruptedException | ExecutionException e) {
 157                 LOG.warn("VLAN Bridge domain {} was not created on node {}", bd, vppNode.getValue(), e);
 158             } catch (TimeoutException e) {
 159                 LOG.warn("Probably, VLAN Bridge domain {} was not created on node {} because BridgeDomainManager "
 160                         + "did not respond by {} seconds. Check VBD log for more details",
 161                         bd, vppNode.getValue(), WAIT_FOR_BD_PROCESSING, e);
 162             }
 163         }
 164     }
 165
 166     public void removeBridgeDomainOnNodes(final SetMultimap<String, NodeId> vppNodesByBridgeDomain) {
 167         for (String bd : vppNodesByBridgeDomain.keySet()) {
 168             Set<NodeId> vppNodes = vppNodesByBridgeDomain.get(bd);
 169             for (NodeId vppNode : vppNodes) {
 170                 try {
 171                     bdManager.removeBridgeDomainFromVppNode(bd, vppNode).get(WAIT_FOR_BD_PROCESSING,
 172                             TimeUnit.SECONDS);
 173                 } catch (InterruptedException | ExecutionException e) {
 174                     LOG.warn("Bridge domain {} was not removed from node {}", bd, vppNode.getValue(), e);
 175                 } catch (TimeoutException e) {
 176                     LOG.warn("Probably, bridge domain {} was not removed from node {} because BridgeDomainManager "
 177                             + "did not respond by {} seconds. Check VBD log for more details",
 178                             bd, vppNode.getValue(), WAIT_FOR_BD_PROCESSING, e);
 179                 }
 180             }
 181         }
 182     }
 183
 184     public void createForwardingForEndpoint(RendererEndpointKey rEpKey, PolicyContext policyCtx) {
 185         AddressEndpointWithLocation rEp = policyCtx.getAddrEpByKey().get(KeyFactory.addressEndpointKey(rEpKey));
 186         ExternalLocationCase rEpLoc = resolveAndValidateLocation(rEp);
 187         if (Strings.isNullOrEmpty(rEpLoc.getExternalNodeConnector())) {
 188             // TODO add it to the status for renderer manager
 189             LOG.info("Renderer endpoint does not have external-node-connector therefore it is ignored {}", rEp);
 190             return;
 191         }
 192
 193         if (Strings.isNullOrEmpty(rEpLoc.getExternalNode())) {
 194             java.util.Optional<String> optL2FloodDomain = resolveL2FloodDomain(rEp, policyCtx);
 195             if (!optL2FloodDomain.isPresent()) {
 196                 // TODO add it to the status for renderer manager
 197                 LOG.info("Renderer endpoint does not have l2FloodDomain as network containment {}", rEp);
 198                 return;
 199             }
 200             String l2FloodDomain = optL2FloodDomain.get();
 201             try {
 202                 ifaceManager.addBridgeDomainToInterface(l2FloodDomain, rEp, AccessListUtil.resolveAclsOnInterface(
 203                         rEpKey, policyCtx), isBviForEndpoint(rEp)).get();
 204                 aclManager.updateAclsForPeers(policyCtx, rEpKey);
 205                 LOG.debug("Interface added to bridge-domain {} for endpoint {}", l2FloodDomain, rEp);
 206             } catch (InterruptedException | ExecutionException e) {
 207                 // TODO add it to the status for renderer manager
 208                 LOG.warn("Interface was not added to bridge-domain {} for endpoint {}", l2FloodDomain, rEp, e);
 209             }
 210         } else {
 211             LOG.debug("Forwarding is not created - Location of renderer endpoint contains "
 212                     + "external-node therefore VPP renderer assumes that interface for endpoint is "
 213                     + "already assigned in bridge-domain representing external-node. {}", rEp);
 214         }
 215     }
 216
 217     private boolean isBviForEndpoint(AddressEndpointWithLocation rEp) {
 218         VppEndpointKey vppEndpointKey =
 219             new VppEndpointKey(rEp.getAddress(), rEp.getAddressType(), rEp.getContextId(), rEp.getContextType());
 220         ReadOnlyTransaction rTx = dataBroker.newReadOnlyTransaction();
 221         Optional<VppEndpoint> vppEndpointOptional =
 222             DataStoreHelper.readFromDs(LogicalDatastoreType.CONFIGURATION,
 223                 InstanceIdentifier.builder(Config.class).child(VppEndpoint.class, vppEndpointKey).build(), rTx);
 224         if (vppEndpointOptional.isPresent()) {
 225             InterfaceTypeChoice interfaceTypeChoice = vppEndpointOptional.get().getInterfaceTypeChoice();
 226             if (interfaceTypeChoice instanceof LoopbackCase) {
 227                 LOG.trace("Vpp renderer endpoint {} IS a BVI interface.", rEp.getKey());
 228                 return ((LoopbackCase) interfaceTypeChoice).isBvi();
 229             }
 230         }
 231         rTx.close();
 232         LOG.trace("Vpp renderer endpoint {} IS NOT a BVI interface.", rEp.getKey());
 233         return false;
 234     }
 235
 236     public void removeForwardingForEndpoint(RendererEndpointKey rEpKey, PolicyContext policyCtx) {
 237         AddressEndpointWithLocation rEp = policyCtx.getAddrEpByKey().get(KeyFactory.addressEndpointKey(rEpKey));
 238         ExternalLocationCase rEpLoc = resolveAndValidateLocation(rEp);
 239         if (Strings.isNullOrEmpty(rEpLoc.getExternalNodeConnector())) {
 240             // nothing was created for endpoint therefore nothing is removed
 241             return;
 242         }
 243         if (!Strings.isNullOrEmpty(rEpLoc.getExternalNode())) {
 244             try {
 245                 ifaceManager.deleteBridgeDomainFromInterface(rEp).get();
 246                 LOG.debug("bridge-domain was deleted from interface for endpoint {}", rEp);
 247             } catch (InterruptedException | ExecutionException e) {
 248                 // TODO add it to the status for renderer manager
 249                 LOG.warn("bridge-domain was not deleted from interface for endpoint {}", rEp, e);
 250             }
 251         } else {
 252             LOG.debug("Forwarding is not removed - Location of renderer endpoint does not contain "
 253                     + "external-node therefore VPP renderer assumes that interface for endpoint is not "
 254                     + "assigned to bridge-domain representing external-node. {}", rEp);
 255         }
 256     }
 257
 258     public static ExternalLocationCase resolveAndValidateLocation(AddressEndpointWithLocation addrEpWithLoc) {
 259         LocationType locationType = addrEpWithLoc.getAbsoluteLocation().getLocationType();
 260         if (!(locationType instanceof ExternalLocationCase)) {
 261             throw new IllegalStateException("Endpoint does not have external location " + addrEpWithLoc);
 262         }
 263         ExternalLocationCase result = (ExternalLocationCase) locationType;
 264         if (result.getExternalNodeMountPoint() == null) {
 265             throw new IllegalStateException("Endpoint does not have external-node-mount-point " + addrEpWithLoc);
 266         }
 267         return result;
 268     }
 269
 270     public static java.util.Optional<String> resolveL2FloodDomain(@Nonnull AddressEndpointWithLocation ep,
 271             @Nonnull PolicyContext policyCtx) {
 272         NetworkContainment netCont = ep.getNetworkContainment();
 273         if (netCont == null) {
 274             return java.util.Optional.empty();
 275         }
 276         Containment containment = netCont.getContainment();
 277         if (containment instanceof ForwardingContextContainment) {
 278             ForwardingContextContainment fwCtxCont = (ForwardingContextContainment) containment;
 279             if (L2FloodDomain.class.equals(fwCtxCont.getContextType())) {
 280                 return fwCtxCont.getContextId() == null ? java.util.Optional.empty() : java.util.Optional
 281                     .of(fwCtxCont.getContextId().getValue());
 282             }
 283         }
 284         if (containment instanceof NetworkDomainContainment) {
 285             final NetworkDomainContainment netDomainCont = (NetworkDomainContainment) containment;
 286             final TenantId tenantId = ep.getTenant();
 287             final NetworkDomainId domainId = netDomainCont.getNetworkDomainId();
 288             final Class<? extends NetworkDomain> domainKey = netDomainCont.getNetworkDomainType();
 289             final RendererNetworkDomainKey rendererNetworkDomainKey = new RendererNetworkDomainKey(domainId, domainKey);
 290             final RendererNetworkDomain rendererNetworkDomain =
 291                     policyCtx.getNetworkDomainTable().get(tenantId, rendererNetworkDomainKey);
 292             if (rendererNetworkDomain == null) {
 293                 LOG.debug("Network domain not found. Containment: {}", containment);
 294                 return java.util.Optional.empty();
 295             }
 296             java.util.Optional<String> optL2Fd = getForwardingCtxForParent(ep.getTenant(),
 297                     rendererNetworkDomain.getParent(), policyCtx.getForwardingCtxTable())
 298                         .filter(fwdCtx -> L2FloodDomain.class.equals(fwdCtx.getContextType()))
 299                         .map(RendererForwardingContext::getContextId)
 300                         .map(ContextId::getValue);
 301             if (!optL2Fd.isPresent()) {
 302                 LOG.debug("network-domain-containment in endpoint does not have L2-flood-domain as parent. "
 303                         + "This case is not supported in VPP renderer. {}", ep);
 304             }
 305             return optL2Fd;
 306         }
 307         return java.util.Optional.empty();
 308     }
 309
 310     private static @Nonnull java.util.Optional<RendererForwardingContext> getForwardingCtxForParent(
 311             @Nullable TenantId tenant, @Nullable Parent parent,
 312             Table<TenantId, RendererForwardingContextKey, RendererForwardingContext> forwardingCtxTable) {
 313         if (tenant == null || parent == null) {
 314             return java.util.Optional.empty();
 315         }
 316         if (parent.getContextId() != null && parent.getContextType() != null) {
 317             return java.util.Optional.ofNullable(forwardingCtxTable.get(tenant,
 318                     new RendererForwardingContextKey(parent.getContextId(), parent.getContextType())));
 319         }
 320         return java.util.Optional.empty();
 321     }
 322
 323     public void syncNatEntries(PolicyContext policyCtx) {
 324         Configuration cfg = policyCtx.getPolicy().getConfiguration();
 325         if(cfg != null) {
 326             List<MappingEntryBuilder> natEntries = resolveNatTableEntries(cfg.getEndpoints());
 327             if (natEntries.isEmpty()) {
 328                 LOG.trace("NAT entries are empty, skipping processing.");
 329                 return;
 330             }
 331             LOG.trace("Syncing NAT entries {}", natEntries);
 332             if (cfg.getRendererForwarding() != null) {
 333                 for (RendererForwardingByTenant fwd : cfg.getRendererForwarding().getRendererForwardingByTenant()) {
 334                     List<InstanceIdentifier<PhysicalInterface>> physIfacesIid =
 335                         resolvePhysicalInterfacesForNat(fwd.getRendererNetworkDomain());
 336                     natManager.submitNatChanges(physIfacesIid, natEntries, policyCtx, true);
 337                 }
 338             }
 339         }
 340     }
 341
 342     public void deleteNatEntries(PolicyContext policyCtx) {
 343         Configuration cfg = policyCtx.getPolicy().getConfiguration();
 344         if(cfg != null) {
 345             List<MappingEntryBuilder> natEntries = resolveNatTableEntries(cfg.getEndpoints());
 346             if (natEntries.isEmpty()) {
 347                 LOG.trace("NAT entries are empty,nothing to delete, skipping processing.");
 348                 return;
 349             }
 350             LOG.trace("Deleting NAT entries {}", natEntries);
 351             if (cfg.getRendererForwarding() != null) {
 352                 for (RendererForwardingByTenant fwd : cfg.getRendererForwarding().getRendererForwardingByTenant()) {
 353                     List<InstanceIdentifier<PhysicalInterface>> physIfacesIid =
 354                         resolvePhysicalInterfacesForNat(fwd.getRendererNetworkDomain());
 355                     natManager.submitNatChanges(physIfacesIid, natEntries, policyCtx, false);
 356                 }
 357             }
 358         }
 359     }
 360
 361     public List<InstanceIdentifier<PhysicalInterface>> resolvePhysicalInterfacesForNat(
 362             List<RendererNetworkDomain> rendNetDomains) {
 363         List<InstanceIdentifier<PhysicalInterface>> physIfaces = new ArrayList<>();
 364         for (RendererNetworkDomain rendDomain : rendNetDomains) {
 365             Optional<IpPrefix> resolvedIpPrefix = resolveIpPrefix(rendDomain);
 366             if (resolvedIpPrefix.isPresent()) {
 367                 Optional<InstanceIdentifier<PhysicalInterface>> resPhIface = natManager.resolvePhysicalInterface(resolvedIpPrefix.get());
 368                 if (resPhIface.isPresent()) {
 369                     physIfaces.add(resPhIface.get());
 370                 }
 371             }
 372         }
 373         return physIfaces;
 374     }
 375
 376     private Optional<IpPrefix> resolveIpPrefix(RendererNetworkDomain rendDomain) {
 377         SubnetAugmentRenderer subnetAug = rendDomain.getAugmentation(SubnetAugmentRenderer.class);
 378         if (subnetAug.getSubnet() != null) {
 379             return Optional.of(subnetAug.getSubnet().getIpPrefix());
 380         }
 381         return Optional.absent();
 382     }
 383
 384     public List<MappingEntryBuilder> resolveNatTableEntries(Endpoints endpoints) {
 385         List<MappingEntryBuilder> sNatEntries = new ArrayList<>();
 386         for (AddressEndpointWithLocation addrEp : endpoints.getAddressEndpointWithLocation()) {
 387             if (addrEp.getAugmentation(NatAddressRenderer.class) == null) {
 388                 continue;
 389             }
 390             NatAddressRenderer natAddr = addrEp.getAugmentation(NatAddressRenderer.class);
 391             if (natAddr.getNatAddress() == null && natAddr.getNatAddress().getIpv4Address() == null) {
 392                 LOG.warn("Only Ipv4 SNAT is currently supported. Cannot apply SNAT for [{},{}]", addrEp.getAddress(),
 393                         natAddr.getNatAddress());
 394                 continue;
 395             }
 396             Optional<MappingEntryBuilder> entry = natManager.resolveSnatEntry(addrEp.getAddress(), natAddr.getNatAddress()
 397                 .getIpv4Address());
 398             if (entry.isPresent()) {
 399                 sNatEntries.add(entry.get());
 400             }
 401         }
 402         return sNatEntries;
 403     }
 404
 405     @VisibleForTesting
 406     void setTimer(byte time) {
 407         WAIT_FOR_BD_PROCESSING = time;
 408     }
 409 }