vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnManagerImpl.java

   1 /*
   2  * Copyright (c) 2015 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  12 import java.util.ArrayList;
  13 import java.util.Collection;
  14 import java.util.Collections;
  15 import java.util.Iterator;
  16 import java.util.List;
  17 import java.util.Objects;
  18 import java.util.Set;
  19 import java.util.concurrent.ExecutionException;
  20 import java.util.concurrent.Future;
  21 import java.util.concurrent.locks.ReentrantLock;
  22 import javax.annotation.PostConstruct;
  23 import javax.inject.Inject;
  24 import javax.inject.Singleton;
  25 import org.eclipse.jdt.annotation.NonNull;
  26 import org.eclipse.jdt.annotation.Nullable;
  27 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  28 import org.opendaylight.controller.md.sal.binding.api.ReadWriteTransaction;
  29 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  30 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  31 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  32 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
  33 import org.opendaylight.genius.infra.Datastore.Configuration;
  34 import org.opendaylight.genius.infra.Datastore.Operational;
  35 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  36 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  37 import org.opendaylight.genius.infra.TypedReadTransaction;
  38 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  39 import org.opendaylight.genius.infra.TypedWriteTransaction;
  40 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  41 import org.opendaylight.genius.mdsalutil.FlowEntity;
  42 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  43 import org.opendaylight.genius.mdsalutil.NwConstants;
  44 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  45 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  46 import org.opendaylight.genius.utils.JvmGlobalLocks;
  47 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  48 import org.opendaylight.infrautils.utils.function.InterruptibleCheckedConsumer;
  49 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  50 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput;
  51 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput.ArpReponderInputBuilder;
  52 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderUtil;
  53 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  54 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  55 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  56 import org.opendaylight.netvirt.neutronvpn.api.enums.IpVersionChoice;
  57 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  58 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  59 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  60 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  61 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  62 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  63 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  64 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  65 import org.opendaylight.serviceutils.upgrade.UpgradeState;
  66 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
  67 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.vpntargets.VpnTarget;
  68 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
  69 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  70 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInput;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInputBuilder;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolOutput;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.SubnetsAssociatedToRouteTargets;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.RouteTarget;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.RouteTargetKey;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.route.target.AssociatedSubnet;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.route.target.associated.subnet.AssociatedVpn;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  89 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  90 import org.opendaylight.yangtools.yang.common.RpcError;
  91 import org.opendaylight.yangtools.yang.common.RpcResult;
  92 import org.opendaylight.yangtools.yang.common.Uint32;
  93 import org.opendaylight.yangtools.yang.common.Uint64;
  94 import org.slf4j.Logger;
  95 import org.slf4j.LoggerFactory;
  96
  97 @Singleton
  98 public class VpnManagerImpl implements IVpnManager {
  99
 100     private static final Logger LOG = LoggerFactory.getLogger(VpnManagerImpl.class);
 101     private final DataBroker dataBroker;
 102     private final ManagedNewTransactionRunner txRunner;
 103     private final IdManagerService idManager;
 104     private final IMdsalApiManager mdsalManager;
 105     private final IElanService elanService;
 106     private final IInterfaceManager interfaceManager;
 107     private final VpnSubnetRouteHandler vpnSubnetRouteHandler;
 108     private final OdlInterfaceRpcService ifaceMgrRpcService;
 109     private final IVpnLinkService ivpnLinkService;
 110     private final IFibManager fibManager;
 111     private final IBgpManager bgpManager;
 112     private final InterVpnLinkCache interVpnLinkCache;
 113     private final DataTreeEventCallbackRegistrar eventCallbacks;
 114     private final UpgradeState upgradeState;
 115     private final ItmRpcService itmRpcService;
 116     private final VpnUtil vpnUtil;
 117
 118     @Inject
 119     public VpnManagerImpl(final DataBroker dataBroker,
 120                           final IdManagerService idManagerService,
 121                           final IMdsalApiManager mdsalManager,
 122                           final IElanService elanService,
 123                           final IInterfaceManager interfaceManager,
 124                           final VpnSubnetRouteHandler vpnSubnetRouteHandler,
 125                           final OdlInterfaceRpcService ifaceMgrRpcService,
 126                           final IVpnLinkService ivpnLinkService,
 127                           final IFibManager fibManager,
 128                           final IBgpManager bgpManager,
 129                           final InterVpnLinkCache interVpnLinkCache,
 130                           final DataTreeEventCallbackRegistrar dataTreeEventCallbackRegistrar,
 131                           final UpgradeState upgradeState,
 132                           final ItmRpcService itmRpcService,
 133                           final VpnUtil vpnUtil) {
 134         this.dataBroker = dataBroker;
 135         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 136         this.idManager = idManagerService;
 137         this.mdsalManager = mdsalManager;
 138         this.elanService = elanService;
 139         this.interfaceManager = interfaceManager;
 140         this.vpnSubnetRouteHandler = vpnSubnetRouteHandler;
 141         this.ifaceMgrRpcService = ifaceMgrRpcService;
 142         this.ivpnLinkService = ivpnLinkService;
 143         this.fibManager = fibManager;
 144         this.bgpManager = bgpManager;
 145         this.interVpnLinkCache = interVpnLinkCache;
 146         this.eventCallbacks = dataTreeEventCallbackRegistrar;
 147         this.upgradeState = upgradeState;
 148         this.itmRpcService = itmRpcService;
 149         this.vpnUtil = vpnUtil;
 150     }
 151
 152     @PostConstruct
 153     public void start() {
 154         LOG.info("{} start", getClass().getSimpleName());
 155         createIdPool();
 156     }
 157
 158     private void createIdPool() {
 159         CreateIdPoolInput createPool = new CreateIdPoolInputBuilder()
 160             .setPoolName(VpnConstants.VPN_IDPOOL_NAME)
 161             .setLow(VpnConstants.VPN_IDPOOL_LOW)
 162             .setHigh(VpnConstants.VPN_IDPOOL_HIGH)
 163             .build();
 164         try {
 165             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPool);
 166             if (result != null && result.get().isSuccessful()) {
 167                 LOG.info("Created IdPool for VPN Service");
 168             }
 169         } catch (InterruptedException | ExecutionException e) {
 170             LOG.error("Failed to create idPool for VPN Service", e);
 171         }
 172
 173         // Now an IdPool for InterVpnLink endpoint's pseudo ports
 174         CreateIdPoolInput createPseudoLporTagPool =
 175             new CreateIdPoolInputBuilder().setPoolName(VpnConstants.PSEUDO_LPORT_TAG_ID_POOL_NAME)
 176                 .setLow(VpnConstants.LOWER_PSEUDO_LPORT_TAG)
 177                 .setHigh(VpnConstants.UPPER_PSEUDO_LPORT_TAG)
 178                 .build();
 179         try {
 180             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPseudoLporTagPool);
 181             if (result.get().isSuccessful()) {
 182                 LOG.debug("Created IdPool for Pseudo Port tags");
 183             } else {
 184                 Collection<RpcError> errors = result.get().getErrors();
 185                 StringBuilder errMsg = new StringBuilder();
 186                 for (RpcError err : errors) {
 187                     errMsg.append(err.getMessage()).append("\n");
 188                 }
 189                 LOG.error("IdPool creation for PseudoPort tags failed. Reasons: {}", errMsg);
 190             }
 191         } catch (InterruptedException | ExecutionException e) {
 192             LOG.error("Failed to create idPool for Pseudo Port tags", e);
 193         }
 194     }
 195
 196     @Override
 197     public void addExtraRoute(String vpnName, String destination, String nextHop, String rd, @Nullable String routerID,
 198         Uint32 l3vni, RouteOrigin origin, @Nullable String intfName, @Nullable Adjacency operationalAdj,
 199         VrfEntry.EncapType encapType, Set<String> prefixListForRefreshFib,
 200         @NonNull TypedWriteTransaction<Configuration> confTx) {
 201         //add extra route to vpn mapping; advertise with nexthop as tunnel ip
 202         vpnUtil.syncUpdate(LogicalDatastoreType.OPERATIONAL,
 203                 VpnExtraRouteHelper.getVpnToExtrarouteVrfIdIdentifier(vpnName, rd != null ? rd : routerID,
 204                         destination),
 205                 VpnUtil.getVpnToExtraroute(destination, Collections.singletonList(nextHop)));
 206
 207         Uint64 dpnId = null;
 208         if (intfName != null && !intfName.isEmpty()) {
 209             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
 210             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 211             if (nextHopIp == null || nextHopIp.isEmpty()) {
 212                 LOG.error("addExtraRoute: NextHop for interface {} is null / empty."
 213                         + " Failed advertising extra route for rd {} prefix {} dpn {}", intfName, rd, destination,
 214                         dpnId);
 215                 return;
 216             }
 217             nextHop = nextHopIp;
 218         }
 219
 220         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 221
 222         // TODO: This is a limitation to be stated in docs. When configuring static route to go to
 223         // another VPN, there can only be one nexthop or, at least, the nexthop to the interVpnLink should be in
 224         // first place.
 225         Optional<InterVpnLinkDataComposite> optVpnLink = interVpnLinkCache.getInterVpnLinkByEndpoint(nextHop);
 226         if (optVpnLink.isPresent() && optVpnLink.get().isActive()) {
 227             InterVpnLinkDataComposite interVpnLink = optVpnLink.get();
 228             // If the nexthop is the endpoint of Vpn2, then prefix must be advertised to Vpn1 in DC-GW, with nexthops
 229             // pointing to the DPNs where Vpn1 is instantiated. LFIB in these DPNS must have a flow entry, with lower
 230             // priority, where if Label matches then sets the lportTag of the Vpn2 endpoint and goes to LportDispatcher
 231             // This is like leaking one of the Vpn2 routes towards Vpn1
 232             String srcVpnUuid = interVpnLink.getVpnNameByIpAddress(nextHop);
 233             String dstVpnUuid = interVpnLink.getOtherVpnNameByIpAddress(nextHop);
 234             String dstVpnRd = vpnUtil.getVpnRd(dstVpnUuid);
 235             Uint32 newLabel = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
 236                     VpnUtil.getNextHopLabelKey(dstVpnRd, destination));
 237             if (newLabel.longValue() == 0) {
 238                 LOG.error("addExtraRoute: Unable to fetch label from Id Manager. Bailing out of adding intervpnlink"
 239                         + " route for destination {}", destination);
 240                 return;
 241             }
 242             ivpnLinkService.leakRoute(interVpnLink, srcVpnUuid, dstVpnUuid, destination, newLabel, RouteOrigin.STATIC);
 243         } else {
 244             Optional<Routes> optVpnExtraRoutes = VpnExtraRouteHelper
 245                     .getVpnExtraroutes(dataBroker, vpnName, rd != null ? rd : routerID, destination);
 246             if (optVpnExtraRoutes.isPresent()) {
 247                 List<String> nhList = optVpnExtraRoutes.get().getNexthopIpList();
 248                 if (nhList != null && nhList.size() > 1) {
 249                     // If nhList is greater than one for vpnextraroute, a call to populatefib doesn't update vrfentry.
 250                     prefixListForRefreshFib.add(destination);
 251                 } else {
 252                     L3vpnInput input = new L3vpnInput().setNextHop(operationalAdj).setNextHopIp(nextHop)
 253                             .setL3vni(l3vni.longValue())
 254                             .setPrimaryRd(primaryRd).setVpnName(vpnName).setDpnId(dpnId)
 255                             .setEncapType(encapType).setRd(rd).setRouteOrigin(origin);
 256                     L3vpnRegistry.getRegisteredPopulator(encapType).populateFib(input, confTx);
 257                 }
 258             }
 259         }
 260     }
 261
 262     @Override
 263     public void delExtraRoute(String vpnName, String destination, String nextHop, String rd, @Nullable String routerID,
 264                               @Nullable String intfName, @NonNull TypedWriteTransaction<Configuration> confTx,
 265                               @NonNull TypedWriteTransaction<Operational> operTx) {
 266         Uint64 dpnId = null;
 267         String tunnelIp = nextHop;
 268         if (intfName != null && !intfName.isEmpty()) {
 269             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
 270             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 271             if (nextHopIp == null || nextHopIp.isEmpty()) {
 272                 LOG.error("delExtraRoute: NextHop for interface {} is null / empty."
 273                         + " Failed advertising extra route for rd {} prefix {} dpn {}", intfName, rd, destination,
 274                         dpnId);
 275             }
 276             tunnelIp = nextHopIp;
 277         }
 278         if (rd != null) {
 279             String primaryRd = vpnUtil.getVpnRd(vpnName);
 280             removePrefixFromBGP(vpnName, primaryRd, rd, intfName, destination,
 281                                 nextHop, tunnelIp, dpnId, confTx, operTx);
 282             LOG.info("delExtraRoute: Removed extra route {} from interface {} for rd {}", destination, intfName, rd);
 283         } else {
 284             // add FIB route directly
 285             fibManager.removeOrUpdateFibEntry(routerID, destination, tunnelIp, confTx);
 286             LOG.info("delExtraRoute: Removed extra route {} from interface {} for rd {}", destination, intfName,
 287                     routerID);
 288         }
 289     }
 290
 291     @Override
 292     @SuppressWarnings("checkstyle:IllegalCatch")
 293     public void removePrefixFromBGP(String vpnName, String primaryRd, String extraRouteRd, String vpnInterfaceName,
 294                                     String prefix, String nextHop, String nextHopTunnelIp, Uint64 dpnId,
 295                                     TypedWriteTransaction<Configuration> confTx,
 296                                     TypedWriteTransaction<Operational> operTx) {
 297         String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 298         // FIXME: separate out to somehow?
 299         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnNamePrefixKey);
 300         LOG.info("removing prefix {} for nexthop {} in VPN {} rd {}", prefix, nextHop, vpnName, extraRouteRd);
 301         try {
 302             lock.lock();
 303             if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, extraRouteRd, vpnInterfaceName, prefix,
 304                     nextHop, nextHopTunnelIp, operTx)) {
 305                 return;
 306             }
 307             fibManager.removeOrUpdateFibEntry(primaryRd, prefix, nextHopTunnelIp, confTx);
 308             if (VpnUtil.isEligibleForBgp(extraRouteRd, vpnName, dpnId, null /*networkName*/)) {
 309                 // TODO: Might be needed to include nextHop here
 310                 bgpManager.withdrawPrefix(extraRouteRd, prefix);
 311             }
 312             LOG.info("removePrefixFromBGP: VPN WITHDRAW: Removed Fib Entry rd {} prefix {} nexthop {}", extraRouteRd,
 313                     prefix, nextHop);
 314         } catch (RuntimeException e) {
 315             LOG.error("removePrefixFromBGP: Delete prefix {} rd {} nextHop {} failed", prefix, extraRouteRd, nextHop);
 316         } finally {
 317             lock.unlock();
 318         }
 319     }
 320
 321     @Override
 322     public boolean isVPNConfigured() {
 323         InstanceIdentifier<VpnInstances> vpnsIdentifier = InstanceIdentifier.builder(VpnInstances.class).build();
 324         try {
 325             Optional<VpnInstances> optionalVpns =
 326                     SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
 327                             vpnsIdentifier);
 328             if (!optionalVpns.isPresent()
 329                     || optionalVpns.get().getVpnInstance() == null
 330                     || optionalVpns.get().getVpnInstance().isEmpty()) {
 331                 LOG.trace("isVPNConfigured: No VPNs configured.");
 332                 return false;
 333             }
 334         } catch (ReadFailedException e) {
 335             throw new RuntimeException("Error reading VPN " + vpnsIdentifier, e);
 336         }
 337         LOG.trace("isVPNConfigured: VPNs are configured on the system.");
 338         return true;
 339     }
 340
 341     @Override
 342     public void addSubnetMacIntoVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 343             Uint64 dpnId, TypedWriteTransaction<Configuration> confTx)
 344             throws ExecutionException, InterruptedException {
 345         setupSubnetMacInVpnInstance(vpnName, subnetVpnName, srcMacAddress, dpnId,
 346             (vpnId, dpId, subnetVpnId) -> addGwMac(srcMacAddress, confTx, vpnId, dpId, subnetVpnId));
 347     }
 348
 349     @Override
 350     public void removeSubnetMacFromVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 351             Uint64 dpnId, TypedReadWriteTransaction<Configuration> confTx)
 352             throws ExecutionException, InterruptedException {
 353         setupSubnetMacInVpnInstance(vpnName, subnetVpnName, srcMacAddress, dpnId,
 354             (vpnId, dpId, subnetVpnId) -> removeGwMac(srcMacAddress, confTx, vpnId, dpId, subnetVpnId));
 355     }
 356
 357     @FunctionalInterface
 358     private interface VpnInstanceSubnetMacSetupMethod {
 359         void process(Uint32 vpnId, Uint64 dpId, Uint32 subnetVpnId) throws InterruptedException, ExecutionException;
 360     }
 361
 362     private void setupSubnetMacInVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 363             Uint64 dpnId, VpnInstanceSubnetMacSetupMethod consumer)
 364             throws ExecutionException, InterruptedException {
 365         if (vpnName == null) {
 366             LOG.warn("Cannot setup subnet MAC {} on DPN {}, null vpnName", srcMacAddress, dpnId);
 367             return;
 368         }
 369
 370         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 371         Uint32 subnetVpnId = vpnUtil.getVpnId(subnetVpnName);
 372         if (dpnId.equals(Uint64.ZERO)) {
 373             /* Apply the MAC on all DPNs in a VPN */
 374             for (Uint64 dpId : vpnUtil.getDpnsOnVpn(vpnName)) {
 375                 consumer.process(vpnId, dpId, subnetVpnId);
 376             }
 377         } else {
 378             consumer.process(vpnId, dpnId, subnetVpnId);
 379         }
 380     }
 381
 382     private void addGwMac(String srcMacAddress, TypedWriteTransaction<Configuration> tx, Uint32 vpnId, Uint64 dpId,
 383         Uint32 subnetVpnId) {
 384         FlowEntity flowEntity = vpnUtil.buildL3vpnGatewayFlow(dpId, srcMacAddress, vpnId, subnetVpnId);
 385         mdsalManager.addFlow(tx, flowEntity);
 386     }
 387
 388     // TODO skitt Fix the exception handling here
 389     @SuppressWarnings("checkstyle:IllegalCatch")
 390     @SuppressFBWarnings("REC_CATCH_EXCEPTION")
 391     private void removeGwMac(String srcMacAddress, TypedReadWriteTransaction<Configuration> tx, Uint32 vpnId,
 392             Uint64 dpId, Uint32 subnetVpnId) throws ExecutionException, InterruptedException {
 393         mdsalManager.removeFlow(tx, dpId,
 394             VpnUtil.getL3VpnGatewayFlowRef(NwConstants.L3_GW_MAC_TABLE, dpId, vpnId, srcMacAddress, subnetVpnId),
 395             NwConstants.L3_GW_MAC_TABLE);
 396     }
 397
 398     @Override
 399     public void addRouterGwMacFlow(String routerName, String routerGwMac, Uint64 dpnId, Uuid extNetworkId,
 400             String subnetVpnName, TypedWriteTransaction<Configuration> confTx)
 401             throws ExecutionException, InterruptedException {
 402         setupRouterGwMacFlow(routerName, routerGwMac, dpnId, extNetworkId,
 403             vpnId -> addSubnetMacIntoVpnInstance(vpnId, subnetVpnName, routerGwMac, dpnId, confTx), "Installing");
 404     }
 405
 406     @Override
 407     public void removeRouterGwMacFlow(String routerName, String routerGwMac, Uint64 dpnId, Uuid extNetworkId,
 408             String subnetVpnName, TypedReadWriteTransaction<Configuration> confTx)
 409             throws ExecutionException, InterruptedException {
 410         setupRouterGwMacFlow(routerName, routerGwMac, dpnId, extNetworkId,
 411             vpnId -> removeSubnetMacFromVpnInstance(vpnId, subnetVpnName, routerGwMac, dpnId, confTx), "Removing");
 412     }
 413
 414     private void setupRouterGwMacFlow(String routerName, String routerGwMac, Uint64 dpnId, Uuid extNetworkId,
 415             InterruptibleCheckedConsumer<String, ExecutionException> consumer, String operation)
 416             throws ExecutionException, InterruptedException {
 417         if (routerGwMac == null) {
 418             LOG.warn("Failed to handle router GW flow in GW-MAC table. MAC address is missing for router-id {}",
 419                 routerName);
 420             return;
 421         }
 422
 423         if (dpnId == null || Uint64.ZERO.equals(dpnId)) {
 424             LOG.info("setupRouterGwMacFlow: DPN id is missing for router-id {}",
 425                 routerName);
 426             return;
 427         }
 428
 429         Uuid vpnId = vpnUtil.getExternalNetworkVpnId(extNetworkId);
 430         if (vpnId == null) {
 431             LOG.warn("Network {} is not associated with VPN", extNetworkId.getValue());
 432             return;
 433         }
 434
 435         LOG.info("{} router GW MAC flow for router-id {} on switch {}", operation, routerName, dpnId);
 436         consumer.accept(vpnId.getValue());
 437     }
 438
 439     @Override
 440     public void addArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 441             Uint64 dpnId, Uuid extNetworkId) {
 442
 443         if (dpnId == null || Uint64.ZERO.equals(dpnId)) {
 444             LOG.warn("Failed to install arp responder flows for router {}. DPN id is missing.", id);
 445             return;
 446         }
 447
 448         String extInterfaceName = elanService.getExternalElanInterface(extNetworkId.getValue(), dpnId);
 449         if (extInterfaceName != null) {
 450             doAddArpResponderFlowsToExternalNetworkIps(
 451                     id, fixedIps, macAddress, dpnId, extInterfaceName);
 452             return;
 453         }
 454
 455         LOG.warn("Failed to install responder flows for {}. No external interface found for DPN id {}", id, dpnId);
 456
 457         if (!upgradeState.isUpgradeInProgress()) {
 458             return;
 459         }
 460
 461         // The following through the end of the function deals with an upgrade scenario where the neutron configuration
 462         // is restored before the OVS switches reconnect. In such a case, the elan-dpn-interfaces entries will be
 463         // missing from the operational data store. In order to mitigate this we use DataTreeEventCallbackRegistrar
 464         // to wait for the exact operational md-sal object we need to contain the external interface we need.
 465
 466         LOG.info("Upgrade in process, waiting for an external interface to appear on dpn {} for elan {}",
 467                 dpnId, extNetworkId.getValue());
 468
 469         InstanceIdentifier<DpnInterfaces> dpnInterfacesIid =
 470                             elanService.getElanDpnInterfaceOperationalDataPath(extNetworkId.getValue(), dpnId);
 471
 472         eventCallbacks.onAddOrUpdate(LogicalDatastoreType.OPERATIONAL, dpnInterfacesIid, (unused, alsoUnused) -> {
 473             LOG.info("Reattempting write of arp responder for external interfaces for external network {}",
 474                     extNetworkId);
 475             DpnInterfaces dpnInterfaces = elanService.getElanInterfaceInfoByElanDpn(extNetworkId.getValue(), dpnId);
 476             if (dpnInterfaces == null) {
 477                 LOG.error("Could not retrieve DpnInterfaces for {}, {}", extNetworkId.getValue(), dpnId);
 478                 return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 479             }
 480
 481             String extIfc = null;
 482             @Nullable List<String> interfaces = dpnInterfaces.getInterfaces();
 483             if (interfaces != null) {
 484                 for (String dpnInterface : interfaces) {
 485                     if (interfaceManager.isExternalInterface(dpnInterface)) {
 486                         extIfc = dpnInterface;
 487                         break;
 488                     }
 489                 }
 490             }
 491
 492             if (extIfc == null) {
 493                 if (upgradeState.isUpgradeInProgress()) {
 494                     LOG.info("External interface not found yet in elan {} on dpn {}, keep waiting",
 495                             extNetworkId.getValue(), dpnInterfaces);
 496                     return DataTreeEventCallbackRegistrar.NextAction.CALL_AGAIN;
 497                 } else {
 498                     return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 499                 }
 500             }
 501
 502             final String extIfcFinal = extIfc;
 503             doAddArpResponderFlowsToExternalNetworkIps(id, fixedIps, macAddress, dpnId, extIfcFinal);
 504
 505             return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 506         });
 507
 508     }
 509
 510     @Override
 511     public void addArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 512             Uint64 dpnId, String extInterfaceName, int lportTag) {
 513         if (fixedIps == null || fixedIps.isEmpty()) {
 514             LOG.debug("No external IPs defined for {}", id);
 515             return;
 516         }
 517
 518         LOG.info("Installing ARP responder flows for {} fixed-ips {} on switch {}", id, fixedIps, dpnId);
 519
 520         for (String fixedIp : fixedIps) {
 521             IpVersionChoice ipVersionChoice = VpnUtil.getIpVersionFromString(fixedIp);
 522             if (ipVersionChoice == IpVersionChoice.IPV6) {
 523                 continue;
 524             }
 525             installArpResponderFlowsToExternalNetworkIp(macAddress, dpnId, extInterfaceName, lportTag,
 526                     fixedIp);
 527         }
 528     }
 529
 530     private void doAddArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 531             Uint64 dpnId, String extInterfaceName) {
 532         Interface extInterfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, extInterfaceName);
 533         if (extInterfaceState == null) {
 534             LOG.debug("No interface state found for interface {}. Delaying responder flows for {}", extInterfaceName,
 535                     id);
 536             return;
 537         }
 538
 539         Integer lportTag = extInterfaceState.getIfIndex();
 540         if (lportTag == null) {
 541             LOG.debug("No Lport tag found for interface {}. Delaying flows for router-id {}", extInterfaceName, id);
 542             return;
 543         }
 544
 545         if (macAddress == null) {
 546
 547             LOG.debug("Failed to install arp responder flows for router-gateway-ip {} for router {}."
 548                     + "External Gw MacAddress is missing.", fixedIps,  id);
 549             return;
 550         }
 551
 552         addArpResponderFlowsToExternalNetworkIps(id, fixedIps, macAddress, dpnId, extInterfaceName, lportTag);
 553     }
 554
 555     @Override
 556     public void removeArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 557             Uint64 dpnId, Uuid extNetworkId) {
 558
 559         if (dpnId == null || Uint64.ZERO.equals(dpnId)) {
 560             LOG.warn("Failed to remove arp responder flows for router {}. DPN id is missing.", id);
 561             return;
 562         }
 563
 564         String extInterfaceName = elanService.getExternalElanInterface(extNetworkId.getValue(), dpnId);
 565         if (extInterfaceName == null) {
 566             LOG.warn("Failed to remove responder flows for {}. No external interface found for DPN id {}", id, dpnId);
 567             return;
 568         }
 569
 570         Interface extInterfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, extInterfaceName);
 571         if (extInterfaceState == null) {
 572             LOG.debug("No interface state found for interface {}. Delaying responder flows for {}", extInterfaceName,
 573                     id);
 574             return;
 575         }
 576
 577         Integer lportTag = extInterfaceState.getIfIndex();
 578         if (lportTag == null) {
 579             LOG.debug("No Lport tag found for interface {}. Delaying flows for router-id {}", extInterfaceName, id);
 580             return;
 581         }
 582
 583         if (macAddress == null) {
 584
 585             LOG.debug("Failed to remove arp responder flows for router-gateway-ip {} for router {}."
 586                     + "External Gw MacAddress is missing.", fixedIps,  id);
 587             return;
 588         }
 589
 590         removeArpResponderFlowsToExternalNetworkIps(id, fixedIps, dpnId,
 591                 extInterfaceName, lportTag);
 592     }
 593
 594     @Override
 595     public void removeArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps,
 596             Uint64 dpnId, String extInterfaceName, int lportTag) {
 597         if (fixedIps == null || fixedIps.isEmpty()) {
 598             LOG.debug("No external IPs defined for {}", id);
 599             return;
 600         }
 601
 602         LOG.info("Removing ARP responder flows for {} fixed-ips {} on switch {}", id, fixedIps, dpnId);
 603
 604         for (String fixedIp : fixedIps) {
 605             removeArpResponderFlowsToExternalNetworkIp(dpnId, lportTag, fixedIp, extInterfaceName);
 606         }
 607     }
 608
 609     @Override
 610     public String getPrimaryRdFromVpnInstance(VpnInstance vpnInstance) {
 611         return VpnUtil.getPrimaryRd(vpnInstance);
 612     }
 613
 614     private void installArpResponderFlowsToExternalNetworkIp(String macAddress, Uint64 dpnId,
 615             String extInterfaceName, int lportTag, String fixedIp) {
 616         // reset the split-horizon bit to allow traffic to be sent back to the
 617         // provider port
 618         List<Instruction> instructions = new ArrayList<>();
 619         instructions.add(
 620                 new InstructionWriteMetadata(Uint64.ZERO,
 621                         MetaDataUtil.METADATA_MASK_SH_FLAG).buildInstruction(1));
 622         instructions.addAll(
 623                 ArpResponderUtil.getExtInterfaceInstructions(interfaceManager, itmRpcService, extInterfaceName,
 624                         fixedIp, macAddress));
 625         ArpReponderInputBuilder builder = new ArpReponderInputBuilder().setDpId(dpnId.toJava())
 626                 .setInterfaceName(extInterfaceName).setSpa(fixedIp).setSha(macAddress).setLportTag(lportTag);
 627         builder.setInstructions(instructions);
 628         elanService.addArpResponderFlow(builder.buildForInstallFlow());
 629     }
 630
 631     private void removeArpResponderFlowsToExternalNetworkIp(Uint64 dpnId, Integer lportTag, String fixedIp,
 632             String extInterfaceName) {
 633         ArpResponderInput arpInput = new ArpReponderInputBuilder()
 634                 .setDpId(dpnId.toJava()).setInterfaceName(extInterfaceName)
 635                 .setSpa(fixedIp).setLportTag(lportTag).buildForRemoveFlow();
 636         elanService.removeArpResponderFlow(arpInput);
 637     }
 638
 639     @Override
 640     public void onSubnetAddedToVpn(Subnetmap subnetmap, boolean isBgpVpn, Long elanTag) {
 641         vpnSubnetRouteHandler.onSubnetAddedToVpn(subnetmap, isBgpVpn, elanTag);
 642     }
 643
 644     @Override
 645     public void onSubnetDeletedFromVpn(Subnetmap subnetmap, boolean isBgpVpn) {
 646         vpnSubnetRouteHandler.onSubnetDeletedFromVpn(subnetmap, isBgpVpn);
 647     }
 648
 649     @Override
 650     @Nullable
 651     public VpnInstance getVpnInstance(DataBroker broker, String vpnInstanceName) {
 652         return vpnUtil.getVpnInstance(vpnInstanceName);
 653     }
 654
 655     @Override
 656     public String getVpnRd(TypedReadTransaction<Configuration> confTx, String vpnName) {
 657         return VpnUtil.getVpnRd(confTx, vpnName);
 658     }
 659
 660     @Override
 661     public String getVpnRd(DataBroker broker, String vpnName) {
 662         return vpnUtil.getVpnRd(vpnName);
 663     }
 664
 665     @Override
 666     public VpnPortipToPort getNeutronPortFromVpnPortFixedIp(TypedReadTransaction<Configuration> confTx, String vpnName,
 667         String fixedIp) {
 668         return VpnUtil.getNeutronPortFromVpnPortFixedIp(confTx, vpnName, fixedIp);
 669     }
 670
 671     @Override
 672     @Nullable
 673     public VpnPortipToPort getNeutronPortFromVpnPortFixedIp(DataBroker broker, String vpnName, String fixedIp) {
 674         return vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, fixedIp);
 675     }
 676
 677     @Override
 678     public Set<VpnTarget> getRtListForVpn(String vpnName) {
 679         return vpnUtil.getRtListForVpn(vpnName);
 680     }
 681
 682     @Override
 683     // Allow deprecated TransactionRunner calls for now
 684     @SuppressWarnings("ForbidCertainMethod")
 685     public void updateRouteTargetsToSubnetAssociation(Set<VpnTarget> routeTargets, String cidr, String vpnName) {
 686         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
 687             for (VpnTarget rt : routeTargets) {
 688                 String rtValue = rt.getVrfRTValue();
 689                 switch (rt.getVrfRTType()) {
 690                     case ImportExtcommunity:
 691                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr,  vpnName,
 692                                 tx, false/*isAssociationRemoved*/);
 693                         break;
 694                     case ExportExtcommunity:
 695                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName,
 696                                 tx, false/*isAssociationRemoved*/);
 697                         break;
 698                     case Both:
 699                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr, vpnName,
 700                                 tx, false/*isAssociationRemoved*/);
 701                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName,
 702                                 tx, false/*isAssociationRemoved*/);
 703                         break;
 704                     default:
 705                         LOG.error("updateRouteTargetsToSubnetAssociation: Invalid rt-type {} for vpn {}"
 706                                 + " subnet-cidr {}", rt.getVrfRTType(), vpnName, cidr);
 707                         break;
 708                 }
 709             }
 710         }), LOG, "updateRouteTargetsToSubnetAssociation: Failed for vpn {} subnet-cidr {}", vpnName, cidr);
 711         LOG.info("updateRouteTargetsToSubnetAssociation: Updated RT to subnet association for vpn {} cidr {}",
 712                 vpnName, cidr);
 713     }
 714
 715     @Override
 716     // Allow deprecated TransactionRunner calls for now
 717     @SuppressWarnings("ForbidCertainMethod")
 718     public void removeRouteTargetsToSubnetAssociation(Set<VpnTarget> routeTargets, String cidr, String vpnName) {
 719         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
 720             for (VpnTarget rt : routeTargets) {
 721                 String rtValue = rt.getVrfRTValue();
 722                 switch (rt.getVrfRTType()) {
 723                     case ImportExtcommunity:
 724                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr, vpnName, tx,
 725                                 true/*isAssociationRemoved*/);
 726                         break;
 727                     case ExportExtcommunity:
 728                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName, tx,
 729                                 true/*isAssociationRemoved*/);
 730                         break;
 731                     case Both:
 732                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr, vpnName, tx,
 733                                 true/*isAssociationRemoved*/);
 734                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName, tx,
 735                                 true/*isAssociationRemoved*/);
 736                         break;
 737                     default:
 738                         LOG.error("removeRouteTargetsToSubnetAssociation: Invalid route-target type {} for vpn {}"
 739                                 + " subnet-cidr {}", rt.getVrfRTType(), vpnName, cidr);
 740                         break;
 741                 }
 742             }
 743         }), LOG, "removeRouteTargetsToSubnetAssociation: Failed for vpn {} subnet-cidr {}", vpnName, cidr);
 744         LOG.info("removeRouteTargetsToSubnetAssociation: vpn {} cidr {}", vpnName, cidr);
 745     }
 746
 747     private boolean doesExistingVpnsHaveConflictingSubnet(Set<VpnTarget> routeTargets, String subnetCidr) {
 748         Set<RouteTarget> routeTargetSet = vpnUtil.getRouteTargetSet(routeTargets);
 749         for (RouteTarget routerTarget : routeTargetSet) {
 750             if (routerTarget.getAssociatedSubnet() != null) {
 751                 for (int i = 0; i < routerTarget.getAssociatedSubnet().size(); i++) {
 752                     AssociatedSubnet associatedSubnet = routerTarget.getAssociatedSubnet().get(i);
 753                     if (VpnUtil.areSubnetsOverlapping(associatedSubnet.getCidr(), subnetCidr)) {
 754                         return true;
 755                     }
 756                     if (routerTarget.getRtType() == RouteTarget.RtType.ERT) {
 757                         /* Check if there are multiple exports for the input iRT value (iRT in routeTargets)
 758                          *  Example : (1) iRT=A eRT=B subnet-range=S1; OK
 759                          *            (2) iRT=A eRT=B subnet-range=S1; OK
 760                          *            (3) iRT=B eRT=A subnet-range=S2; NOK
 761                          * Check if (1) and (2) are importing the same subnet-range routes to (3) */
 762                         List<AssociatedVpn> multipleAssociatedVpn = associatedSubnet.getAssociatedVpn();
 763                         if (multipleAssociatedVpn != null && multipleAssociatedVpn.size() > 1) {
 764                             LOG.error("doesExistingVpnsHaveConflictingSubnet: There is an indirect complete  overlap"
 765                                     + " for subnet CIDR {} for rt {} rtType {}", subnetCidr, routerTarget.getRt(),
 766                                     routerTarget.getRtType());
 767                             return true;
 768                         }
 769                         for (int j = i + 1; j < routerTarget.getAssociatedSubnet().size(); j++) {
 770                             if (VpnUtil.areSubnetsOverlapping(associatedSubnet.getCidr(),
 771                                     routerTarget.getAssociatedSubnet().get(j).getCidr())) {
 772                                 LOG.error("doesExistingVpnsHaveConflictingSubnet: There is an indirect paartial"
 773                                                 + " overlap for subnet CIDR {} for rt {} rtType {}", subnetCidr,
 774                                         routerTarget.getRt(), routerTarget.getRtType());
 775                                 return true;
 776                             }
 777                         }
 778                     }
 779                 }
 780                 /* Check if there are indirect EXPORTS for the eRT value (eRT in routeTargets)
 781                  *  Example : (1) iRT=A eRT=B subnet-range=S1; OK
 782                  *            (2) iRT=B eRT=A subnet-range=S2; OK
 783                  *            (3) iRT=A eRT=B subnet-range=S1; NOK
 784                  * If associatedSubnet is non-null for a routeTarget in (2),
 785                  * it may have already imported routes from (1) */
 786                 if (routerTarget.getRtType() == RouteTarget.RtType.IRT) {
 787                     try {
 788                         Optional<RouteTarget> indirectRts = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 789                                 LogicalDatastoreType.OPERATIONAL, VpnUtil.getRouteTargetsIdentifier(
 790                                         routerTarget.getRt(), RouteTarget.RtType.ERT));
 791                         if (indirectRts.isPresent() && indirectRts.get().getAssociatedSubnet() != null
 792                                 && routerTarget.getAssociatedSubnet() != null) {
 793                             for (AssociatedSubnet associatedSubnet : indirectRts.get().getAssociatedSubnet()) {
 794                                 if (VpnUtil.areSubnetsOverlapping(associatedSubnet.getCidr(), subnetCidr)) {
 795                                     LOG.error("doesExistingVpnsHaveConflictingSubnet: There is an indirect overlap for"
 796                                             + " subnet CIDR {} for rt {} rtType {}", subnetCidr, routerTarget.getRt(),
 797                                             routerTarget.getRtType());
 798                                     return true;
 799                                 }
 800                             }
 801                         }
 802                     } catch (ReadFailedException e) {
 803                         LOG.error("doesExistingVpnsHaveConflictingSubnet: Failed to read route targets to subnet"
 804                                 + "association for rt {} type {} subnet-cidr {}", routerTarget.getRt(),
 805                                 RouteTarget.RtType.ERT, subnetCidr);
 806                         return true; //Fail subnet association to avoid further damage to the data-stores
 807                     }
 808                 }
 809             } else {
 810                 LOG.info("doesExistingVpnsHaveConflictingSubnet: No prior associated subnets for rtVal {} rtType {}",
 811                         routerTarget.getRt(), routerTarget.getRtType());
 812             }
 813         }
 814         return false;
 815     }
 816
 817     private void addSubnetAssociationOperationToTx(String rt, RouteTarget.RtType rtType, String cidr,
 818                                                    String vpnName, ReadWriteTransaction tx,
 819                                                    boolean isAssociationRemoved)
 820             throws InterruptedException, ExecutionException {
 821         if (isAssociationRemoved) {
 822             //Remove RT-Subnet-Vpn Association
 823             Optional<AssociatedSubnet> associatedSubnet = tx.read(LogicalDatastoreType.OPERATIONAL,
 824                     VpnUtil.getAssociatedSubnetIdentifier(rt, rtType, cidr)).get();
 825             boolean deleteParent = false;
 826             if (associatedSubnet.isPresent()) {
 827                 List<AssociatedVpn> associatedVpns = new ArrayList<>(associatedSubnet.get().nonnullAssociatedVpn());
 828                 if (associatedVpns == null || associatedVpns.isEmpty()) {
 829                     deleteParent = true;
 830                 } else {
 831                     for (Iterator<AssociatedVpn> iterator = associatedVpns.iterator(); iterator.hasNext();) {
 832                         AssociatedVpn associatedVpn = iterator.next();
 833                         if (Objects.equals(associatedVpn.getName(), vpnName)) {
 834                             iterator.remove();
 835                             break;
 836                         }
 837                     }
 838                     if (associatedVpns.isEmpty()) {
 839                         deleteParent = true;
 840                     }
 841                 }
 842             }
 843             if (deleteParent) {
 844                 deleteParentForSubnetToVpnAssocication(rt, rtType, cidr, tx);
 845             } else {
 846                 //Some other VPNs are also part of this rtVal, rtType and subnetCidr combination.
 847                 //Delete only this AssociatedVpn Object
 848                 tx.delete(LogicalDatastoreType.OPERATIONAL,
 849                         VpnUtil.getAssociatedSubnetAndVpnIdentifier(rt, rtType, cidr, vpnName));
 850                 LOG.debug("addSubnetAssocOperationToTx: Removed vpn {} from association rt {} rtType {} cidr {}",
 851                         vpnName, rt, rtType, cidr);
 852             }
 853         } else {
 854             //Add RT-Subnet-Vpn Association
 855             tx.put(LogicalDatastoreType.OPERATIONAL,
 856                     VpnUtil.getAssociatedSubnetAndVpnIdentifier(rt, rtType, cidr, vpnName),
 857                     VpnUtil.buildAssociatedSubnetAndVpn(vpnName), true);
 858         }
 859     }
 860
 861     private void deleteParentForSubnetToVpnAssocication(String rt, RouteTarget.RtType rtType,
 862                                                 String cidr, ReadWriteTransaction tx)
 863             throws InterruptedException, ExecutionException {
 864         //Check if you need to delete rtVal+rtType or just the subnetCidr
 865         InstanceIdentifier<RouteTarget> rtIdentifier = InstanceIdentifier
 866                 .builder(SubnetsAssociatedToRouteTargets.class).child(RouteTarget.class,
 867                         new RouteTargetKey(rt, rtType)).build();
 868         Optional<RouteTarget> rtToSubnetsAssociation = tx.read(LogicalDatastoreType.OPERATIONAL,
 869                 rtIdentifier).get();
 870         if (rtToSubnetsAssociation.isPresent()) {
 871             List<AssociatedSubnet> associatedSubnets = new ArrayList<>(rtToSubnetsAssociation.get()
 872                     .nonnullAssociatedSubnet());
 873             if (associatedSubnets != null && !associatedSubnets.isEmpty()) {
 874                 for (Iterator<AssociatedSubnet> iterator = associatedSubnets.iterator(); iterator.hasNext(); ) {
 875                     if (Objects.equals(iterator.next().getCidr(), cidr)) {
 876                         iterator.remove();
 877                         break;
 878                     }
 879                 }
 880                 if (associatedSubnets.isEmpty()) {
 881                     //The entire rt to subnet association is empty
 882                     //Delete the RouteTarget object
 883                     tx.delete(LogicalDatastoreType.OPERATIONAL, rtIdentifier);
 884                     LOG.debug("deleteParentForSubnetToVpnAssocication: Removed rt {} rtType {} from association,",
 885                             rt, rtType);
 886                 } else {
 887                     //Some other VPNs are also part of this rtVal, rtType combination
 888                     //Delete only this AssociatedSubnet
 889                     tx.delete(LogicalDatastoreType.OPERATIONAL, VpnUtil.getAssociatedSubnetIdentifier(rt, rtType,
 890                             cidr));
 891                     LOG.debug("deleteParentForSubnetToVpnAssocication: Removed cidr {} from association rt {}"
 892                             + " rtType {}", cidr, rt, rtType);
 893                 }
 894             }
 895         }
 896     }
 897
 898     @Override
 899     public boolean checkForOverlappingSubnets(Uuid network, List<Subnetmap> subnetmapList, Uuid vpn,
 900                                        Set<VpnTarget> routeTargets, List<String> failedNwList) {
 901         for (int i = 0; i < subnetmapList.size(); i++) {
 902             //Check if any other subnet that is already part of a different vpn with same rt, has overlapping CIDR
 903             if (checkExistingSubnetWithSameRoutTargets(routeTargets, vpn, subnetmapList.get(i), failedNwList)) {
 904                 return true;
 905             }
 906         }
 907         return false;
 908     }
 909
 910     private boolean checkExistingSubnetWithSameRoutTargets(Set<VpnTarget> routeTargets, Uuid vpn, Subnetmap subnetmap,
 911                                                    List<String> failedNwList) {
 912         String cidr = String.valueOf(subnetmap.getSubnetIp());
 913         boolean subnetExistsWithSameRt = doesExistingVpnsHaveConflictingSubnet(routeTargets, cidr);
 914         if (subnetExistsWithSameRt) {
 915             failedNwList.add(String.format("Another subnet with the same/overlapping cidr %s as subnet %s"
 916                     + " is already associated to a vpn with routeTargets %s. Ignoring subnet addition to vpn"
 917                     + " %s", cidr, subnetmap.getId().getValue(), routeTargets.toString(), vpn.getValue()));
 918         }
 919         return subnetExistsWithSameRt;
 920     }
 921 }