vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnManagerImpl.java

   1 /*
   2  * Copyright (c) 2015 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  11
  12 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  13 import java.util.ArrayList;
  14 import java.util.Collection;
  15 import java.util.Collections;
  16 import java.util.Iterator;
  17 import java.util.List;
  18 import java.util.Objects;
  19 import java.util.Optional;
  20 import java.util.Set;
  21 import java.util.concurrent.ExecutionException;
  22 import java.util.concurrent.Future;
  23 import java.util.concurrent.locks.ReentrantLock;
  24 import javax.annotation.PostConstruct;
  25 import javax.inject.Inject;
  26 import javax.inject.Singleton;
  27 import org.eclipse.jdt.annotation.NonNull;
  28 import org.eclipse.jdt.annotation.Nullable;
  29 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  30 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
  31 import org.opendaylight.genius.infra.Datastore.Configuration;
  32 import org.opendaylight.genius.infra.Datastore.Operational;
  33 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  34 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  35 import org.opendaylight.genius.infra.TypedReadTransaction;
  36 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  37 import org.opendaylight.genius.infra.TypedWriteTransaction;
  38 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  39 import org.opendaylight.genius.mdsalutil.FlowEntity;
  40 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  41 import org.opendaylight.genius.mdsalutil.NwConstants;
  42 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  43 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  44 import org.opendaylight.genius.utils.JvmGlobalLocks;
  45 import org.opendaylight.infrautils.utils.concurrent.LoggingFutures;
  46 import org.opendaylight.infrautils.utils.function.InterruptibleCheckedConsumer;
  47 import org.opendaylight.mdsal.binding.api.DataBroker;
  48 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  49 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  50 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput;
  51 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput.ArpReponderInputBuilder;
  52 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderUtil;
  53 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  54 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  55 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  56 import org.opendaylight.netvirt.neutronvpn.api.enums.IpVersionChoice;
  57 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  58 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  59 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  60 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  61 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  62 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  63 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  64 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  65 import org.opendaylight.serviceutils.upgrade.UpgradeState;
  66 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  67 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInput;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInputBuilder;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolOutput;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.SubnetsAssociatedToRouteTargets;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.RouteTarget;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.RouteTargetKey;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.route.target.AssociatedSubnet;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.route.target.associated.subnet.AssociatedVpn;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.VpnInstances;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.Adjacency;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.instances.VpnInstance;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.instances.vpn.instance.vpntargets.VpnTarget;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  89 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  90 import org.opendaylight.yangtools.yang.common.RpcError;
  91 import org.opendaylight.yangtools.yang.common.RpcResult;
  92 import org.opendaylight.yangtools.yang.common.Uint32;
  93 import org.opendaylight.yangtools.yang.common.Uint64;
  94 import org.slf4j.Logger;
  95 import org.slf4j.LoggerFactory;
  96
  97 @Singleton
  98 public class VpnManagerImpl implements IVpnManager {
  99
 100     private static final Logger LOG = LoggerFactory.getLogger(VpnManagerImpl.class);
 101     private final DataBroker dataBroker;
 102     private final ManagedNewTransactionRunner txRunner;
 103     private final IdManagerService idManager;
 104     private final IMdsalApiManager mdsalManager;
 105     private final IElanService elanService;
 106     private final IInterfaceManager interfaceManager;
 107     private final VpnSubnetRouteHandler vpnSubnetRouteHandler;
 108     private final OdlInterfaceRpcService ifaceMgrRpcService;
 109     private final IVpnLinkService ivpnLinkService;
 110     private final IFibManager fibManager;
 111     private final IBgpManager bgpManager;
 112     private final InterVpnLinkCache interVpnLinkCache;
 113     private final DataTreeEventCallbackRegistrar eventCallbacks;
 114     private final UpgradeState upgradeState;
 115     private final ItmRpcService itmRpcService;
 116     private final VpnUtil vpnUtil;
 117
 118     @Inject
 119     public VpnManagerImpl(final DataBroker dataBroker,
 120                           final IdManagerService idManagerService,
 121                           final IMdsalApiManager mdsalManager,
 122                           final IElanService elanService,
 123                           final IInterfaceManager interfaceManager,
 124                           final VpnSubnetRouteHandler vpnSubnetRouteHandler,
 125                           final OdlInterfaceRpcService ifaceMgrRpcService,
 126                           final IVpnLinkService ivpnLinkService,
 127                           final IFibManager fibManager,
 128                           final IBgpManager bgpManager,
 129                           final InterVpnLinkCache interVpnLinkCache,
 130                           final DataTreeEventCallbackRegistrar dataTreeEventCallbackRegistrar,
 131                           final UpgradeState upgradeState,
 132                           final ItmRpcService itmRpcService,
 133                           final VpnUtil vpnUtil) {
 134         this.dataBroker = dataBroker;
 135         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 136         this.idManager = idManagerService;
 137         this.mdsalManager = mdsalManager;
 138         this.elanService = elanService;
 139         this.interfaceManager = interfaceManager;
 140         this.vpnSubnetRouteHandler = vpnSubnetRouteHandler;
 141         this.ifaceMgrRpcService = ifaceMgrRpcService;
 142         this.ivpnLinkService = ivpnLinkService;
 143         this.fibManager = fibManager;
 144         this.bgpManager = bgpManager;
 145         this.interVpnLinkCache = interVpnLinkCache;
 146         this.eventCallbacks = dataTreeEventCallbackRegistrar;
 147         this.upgradeState = upgradeState;
 148         this.itmRpcService = itmRpcService;
 149         this.vpnUtil = vpnUtil;
 150     }
 151
 152     @PostConstruct
 153     public void start() {
 154         LOG.info("{} start", getClass().getSimpleName());
 155         createIdPool();
 156     }
 157
 158     private void createIdPool() {
 159         CreateIdPoolInput createPool = new CreateIdPoolInputBuilder()
 160             .setPoolName(VpnConstants.VPN_IDPOOL_NAME)
 161             .setLow(VpnConstants.VPN_IDPOOL_LOW)
 162             .setHigh(VpnConstants.VPN_IDPOOL_HIGH)
 163             .build();
 164         try {
 165             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPool);
 166             if (result != null && result.get().isSuccessful()) {
 167                 LOG.info("Created IdPool for VPN Service");
 168             } else {
 169                 LOG.error("createIdPool: Unable to create ID pool for VPNService");
 170             }
 171         } catch (InterruptedException | ExecutionException e) {
 172             LOG.error("Failed to create idPool for VPN Service", e);
 173         }
 174
 175         // Now an IdPool for InterVpnLink endpoint's pseudo ports
 176         CreateIdPoolInput createPseudoLporTagPool =
 177             new CreateIdPoolInputBuilder().setPoolName(VpnConstants.PSEUDO_LPORT_TAG_ID_POOL_NAME)
 178                 .setLow(VpnConstants.LOWER_PSEUDO_LPORT_TAG)
 179                 .setHigh(VpnConstants.UPPER_PSEUDO_LPORT_TAG)
 180                 .build();
 181         try {
 182             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPseudoLporTagPool);
 183             if (result != null && result.get().isSuccessful()) {
 184                 LOG.debug("Created IdPool for Pseudo Port tags");
 185             } else {
 186                 StringBuilder errMsg = new StringBuilder();
 187                 if (result != null && result.get() != null) {
 188                     Collection<RpcError> errors = result.get().getErrors();
 189                     for (RpcError err : errors) {
 190                         errMsg.append(err.getMessage()).append("\n");
 191                     }
 192                 }
 193                 LOG.error("IdPool creation for PseudoPort tags failed. Reasons: {}", errMsg);
 194             }
 195         } catch (InterruptedException | ExecutionException e) {
 196             LOG.error("Failed to create idPool for Pseudo Port tags", e);
 197         }
 198     }
 199
 200     @Override
 201     public void addExtraRoute(String vpnName, String destination, String nextHop, String rd, @Nullable String routerID,
 202         Uint32 l3vni, RouteOrigin origin, @Nullable String intfName, @Nullable Adjacency operationalAdj,
 203         VrfEntry.EncapType encapType, Set<String> prefixListForRefreshFib,
 204         @NonNull TypedWriteTransaction<Configuration> confTx) {
 205         //add extra route to vpn mapping; advertise with nexthop as tunnel ip
 206         vpnUtil.syncUpdate(LogicalDatastoreType.OPERATIONAL,
 207                 VpnExtraRouteHelper.getVpnToExtrarouteVrfIdIdentifier(vpnName, rd != null ? rd : routerID,
 208                         destination),
 209                 VpnUtil.getVpnToExtraroute(destination, Collections.singletonList(nextHop)));
 210
 211         Uint64 dpnId = null;
 212         if (intfName != null && !intfName.isEmpty()) {
 213             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
 214             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 215             if (nextHopIp == null || nextHopIp.isEmpty()) {
 216                 LOG.error("addExtraRoute: NextHop for interface {} is null / empty."
 217                         + " Failed advertising extra route for rd {} prefix {} dpn {}", intfName, rd, destination,
 218                         dpnId);
 219                 return;
 220             }
 221             nextHop = nextHopIp;
 222         }
 223
 224         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 225
 226         // TODO: This is a limitation to be stated in docs. When configuring static route to go to
 227         // another VPN, there can only be one nexthop or, at least, the nexthop to the interVpnLink should be in
 228         // first place.
 229         Optional<InterVpnLinkDataComposite> optVpnLink = interVpnLinkCache.getInterVpnLinkByEndpoint(nextHop);
 230         if (optVpnLink.isPresent() && optVpnLink.get().isActive()) {
 231             InterVpnLinkDataComposite interVpnLink = optVpnLink.get();
 232             // If the nexthop is the endpoint of Vpn2, then prefix must be advertised to Vpn1 in DC-GW, with nexthops
 233             // pointing to the DPNs where Vpn1 is instantiated. LFIB in these DPNS must have a flow entry, with lower
 234             // priority, where if Label matches then sets the lportTag of the Vpn2 endpoint and goes to LportDispatcher
 235             // This is like leaking one of the Vpn2 routes towards Vpn1
 236             String srcVpnUuid = interVpnLink.getVpnNameByIpAddress(nextHop);
 237             String dstVpnUuid = interVpnLink.getOtherVpnNameByIpAddress(nextHop);
 238             String dstVpnRd = vpnUtil.getVpnRd(dstVpnUuid);
 239             Uint32 newLabel = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
 240                     VpnUtil.getNextHopLabelKey(dstVpnRd, destination));
 241             if (newLabel.longValue() == VpnConstants.INVALID_LABEL) {
 242                 LOG.error("addExtraRoute: Unable to fetch label from Id Manager. Bailing out of adding intervpnlink"
 243                         + " route for destination {}", destination);
 244                 return;
 245             }
 246             ivpnLinkService.leakRoute(interVpnLink, srcVpnUuid, dstVpnUuid, destination, newLabel, RouteOrigin.STATIC);
 247         } else {
 248             Optional<Routes> optVpnExtraRoutes = VpnExtraRouteHelper
 249                     .getVpnExtraroutes(dataBroker, vpnName, rd != null ? rd : routerID, destination);
 250             if (optVpnExtraRoutes.isPresent()) {
 251                 List<String> nhList = optVpnExtraRoutes.get().getNexthopIpList();
 252                 if (nhList != null && nhList.size() > 1) {
 253                     // If nhList is greater than one for vpnextraroute, a call to populatefib doesn't update vrfentry.
 254                     prefixListForRefreshFib.add(destination);
 255                 } else {
 256                     L3vpnInput input = new L3vpnInput().setNextHop(operationalAdj).setNextHopIp(nextHop)
 257                             .setL3vni(l3vni.longValue())
 258                             .setPrimaryRd(primaryRd).setVpnName(vpnName).setDpnId(dpnId)
 259                             .setEncapType(encapType).setRd(rd).setRouteOrigin(origin);
 260                     L3vpnRegistry.getRegisteredPopulator(encapType).populateFib(input, confTx);
 261                 }
 262             }
 263         }
 264     }
 265
 266     @Override
 267     public void delExtraRoute(String vpnName, String destination, String nextHop, String rd, @Nullable String routerID,
 268                               @Nullable String intfName, @NonNull TypedWriteTransaction<Configuration> confTx,
 269                               @NonNull TypedWriteTransaction<Operational> operTx) {
 270         Uint64 dpnId = null;
 271         String tunnelIp = nextHop;
 272         if (intfName != null && !intfName.isEmpty()) {
 273             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
 274             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 275             if (nextHopIp == null || nextHopIp.isEmpty()) {
 276                 LOG.error("delExtraRoute: NextHop for interface {} is null / empty."
 277                         + " Failed advertising extra route for rd {} prefix {} dpn {}", intfName, rd, destination,
 278                         dpnId);
 279             }
 280             tunnelIp = nextHopIp;
 281         }
 282         if (rd != null) {
 283             String primaryRd = vpnUtil.getVpnRd(vpnName);
 284             removePrefixFromBGP(vpnName, primaryRd, rd, intfName, destination,
 285                                 nextHop, tunnelIp, dpnId, confTx, operTx);
 286             LOG.info("delExtraRoute: Removed extra route {} from interface {} for rd {}", destination, intfName, rd);
 287         } else {
 288             // add FIB route directly
 289             fibManager.removeOrUpdateFibEntry(routerID, destination, tunnelIp, confTx);
 290             LOG.info("delExtraRoute: Removed extra route {} from interface {} for rd {}", destination, intfName,
 291                     routerID);
 292         }
 293     }
 294
 295     @Override
 296     @SuppressWarnings("checkstyle:IllegalCatch")
 297     public void removePrefixFromBGP(String vpnName, String primaryRd, String extraRouteRd, String vpnInterfaceName,
 298                                     String prefix, String nextHop, String nextHopTunnelIp, Uint64 dpnId,
 299                                     TypedWriteTransaction<Configuration> confTx,
 300                                     TypedWriteTransaction<Operational> operTx) {
 301         String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 302         // FIXME: separate out to somehow?
 303         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnNamePrefixKey);
 304         LOG.info("removing prefix {} for nexthop {} in VPN {} rd {}", prefix, nextHop, vpnName, extraRouteRd);
 305         try {
 306             lock.lock();
 307             if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, extraRouteRd, vpnInterfaceName, prefix,
 308                     nextHop, nextHopTunnelIp, operTx)) {
 309                 return;
 310             }
 311             fibManager.removeOrUpdateFibEntry(primaryRd, prefix, nextHopTunnelIp, confTx);
 312             if (VpnUtil.isEligibleForBgp(extraRouteRd, vpnName, dpnId, null /*networkName*/)) {
 313                 // TODO: Might be needed to include nextHop here
 314                 bgpManager.withdrawPrefix(extraRouteRd, prefix);
 315             }
 316             LOG.info("removePrefixFromBGP: VPN WITHDRAW: Removed Fib Entry rd {} prefix {} nexthop {}", extraRouteRd,
 317                     prefix, nextHop);
 318         } catch (RuntimeException e) {
 319             LOG.error("removePrefixFromBGP: Delete prefix {} rd {} nextHop {} failed", prefix, extraRouteRd, nextHop);
 320         } finally {
 321             lock.unlock();
 322         }
 323     }
 324
 325     @Override
 326     public boolean isVPNConfigured() {
 327         InstanceIdentifier<VpnInstances> vpnsIdentifier = InstanceIdentifier.builder(VpnInstances.class).build();
 328         try {
 329             Optional<VpnInstances> optionalVpns =
 330                     SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
 331                             vpnsIdentifier);
 332             if (!optionalVpns.isPresent()
 333                     || optionalVpns.get().getVpnInstance() == null
 334                     || optionalVpns.get().getVpnInstance().isEmpty()) {
 335                 LOG.trace("isVPNConfigured: No VPNs configured.");
 336                 return false;
 337             }
 338         } catch (InterruptedException | ExecutionException e) {
 339             throw new RuntimeException("Error reading VPN " + vpnsIdentifier, e);
 340         }
 341         LOG.trace("isVPNConfigured: VPNs are configured on the system.");
 342         return true;
 343     }
 344
 345     @Override
 346     public void addSubnetMacIntoVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 347             Uint64 dpnId, TypedWriteTransaction<Configuration> confTx)
 348             throws ExecutionException, InterruptedException {
 349         setupSubnetMacInVpnInstance(vpnName, subnetVpnName, srcMacAddress, dpnId,
 350             (vpnId, dpId, subnetVpnId) -> addGwMac(srcMacAddress, confTx, vpnId, dpId, subnetVpnId));
 351     }
 352
 353     @Override
 354     public void removeSubnetMacFromVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 355             Uint64 dpnId, TypedReadWriteTransaction<Configuration> confTx)
 356             throws ExecutionException, InterruptedException {
 357         setupSubnetMacInVpnInstance(vpnName, subnetVpnName, srcMacAddress, dpnId,
 358             (vpnId, dpId, subnetVpnId) -> removeGwMac(srcMacAddress, confTx, vpnId, dpId, subnetVpnId));
 359     }
 360
 361     @FunctionalInterface
 362     private interface VpnInstanceSubnetMacSetupMethod {
 363         void process(Uint32 vpnId, Uint64 dpId, Uint32 subnetVpnId) throws InterruptedException, ExecutionException;
 364     }
 365
 366     private void setupSubnetMacInVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 367             Uint64 dpnId, VpnInstanceSubnetMacSetupMethod consumer)
 368             throws ExecutionException, InterruptedException {
 369         if (vpnName == null) {
 370             LOG.warn("Cannot setup subnet MAC {} on DPN {}, null vpnName", srcMacAddress, dpnId);
 371             return;
 372         }
 373
 374         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 375         Uint32 subnetVpnId = vpnUtil.getVpnId(subnetVpnName);
 376         if (dpnId.equals(Uint64.ZERO)) {
 377             /* Apply the MAC on all DPNs in a VPN */
 378             for (Uint64 dpId : vpnUtil.getDpnsOnVpn(vpnName)) {
 379                 consumer.process(vpnId, dpId, subnetVpnId);
 380             }
 381         } else {
 382             consumer.process(vpnId, dpnId, subnetVpnId);
 383         }
 384     }
 385
 386     private void addGwMac(String srcMacAddress, TypedWriteTransaction<Configuration> tx, Uint32 vpnId, Uint64 dpId,
 387         Uint32 subnetVpnId) {
 388         FlowEntity flowEntity = vpnUtil.buildL3vpnGatewayFlow(dpId, srcMacAddress, vpnId, subnetVpnId);
 389         mdsalManager.addFlow(tx, flowEntity);
 390     }
 391
 392     // TODO skitt Fix the exception handling here
 393     @SuppressWarnings("checkstyle:IllegalCatch")
 394     @SuppressFBWarnings("REC_CATCH_EXCEPTION")
 395     private void removeGwMac(String srcMacAddress, TypedReadWriteTransaction<Configuration> tx, Uint32 vpnId,
 396             Uint64 dpId, Uint32 subnetVpnId) throws ExecutionException, InterruptedException {
 397         mdsalManager.removeFlow(tx, dpId,
 398             VpnUtil.getL3VpnGatewayFlowRef(NwConstants.L3_GW_MAC_TABLE, dpId, vpnId, srcMacAddress, subnetVpnId),
 399             NwConstants.L3_GW_MAC_TABLE);
 400     }
 401
 402     @Override
 403     public void addRouterGwMacFlow(String routerName, String routerGwMac, Uint64 dpnId, Uuid extNetworkId,
 404             String subnetVpnName, TypedWriteTransaction<Configuration> confTx)
 405             throws ExecutionException, InterruptedException {
 406         setupRouterGwMacFlow(routerName, routerGwMac, dpnId, extNetworkId,
 407             vpnId -> addSubnetMacIntoVpnInstance(vpnId, subnetVpnName, routerGwMac, dpnId, confTx), "Installing");
 408     }
 409
 410     @Override
 411     public void removeRouterGwMacFlow(String routerName, String routerGwMac, Uint64 dpnId, Uuid extNetworkId,
 412             String subnetVpnName, TypedReadWriteTransaction<Configuration> confTx)
 413             throws ExecutionException, InterruptedException {
 414         setupRouterGwMacFlow(routerName, routerGwMac, dpnId, extNetworkId,
 415             vpnId -> removeSubnetMacFromVpnInstance(vpnId, subnetVpnName, routerGwMac, dpnId, confTx), "Removing");
 416     }
 417
 418     private void setupRouterGwMacFlow(String routerName, String routerGwMac, Uint64 dpnId, Uuid extNetworkId,
 419             InterruptibleCheckedConsumer<String, ExecutionException> consumer, String operation)
 420             throws ExecutionException, InterruptedException {
 421         if (routerGwMac == null) {
 422             LOG.warn("Failed to handle router GW flow in GW-MAC table. MAC address is missing for router-id {}",
 423                 routerName);
 424             return;
 425         }
 426
 427         if (dpnId == null || Uint64.ZERO.equals(dpnId)) {
 428             LOG.info("setupRouterGwMacFlow: DPN id is missing for router-id {}",
 429                 routerName);
 430             return;
 431         }
 432
 433         Uuid vpnId = vpnUtil.getExternalNetworkVpnId(extNetworkId);
 434         if (vpnId == null) {
 435             LOG.warn("Network {} is not associated with VPN", extNetworkId.getValue());
 436             return;
 437         }
 438
 439         LOG.info("{} router GW MAC flow for router-id {} on switch {}", operation, routerName, dpnId);
 440         consumer.accept(vpnId.getValue());
 441     }
 442
 443     @Override
 444     public void addArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 445             Uint64 dpnId, Uuid extNetworkId) {
 446
 447         if (dpnId == null || Uint64.ZERO.equals(dpnId)) {
 448             LOG.warn("Failed to install arp responder flows for router {}. DPN id is missing.", id);
 449             return;
 450         }
 451
 452         String extInterfaceName = elanService.getExternalElanInterface(extNetworkId.getValue(), dpnId);
 453         if (extInterfaceName != null) {
 454             doAddArpResponderFlowsToExternalNetworkIps(
 455                     id, fixedIps, macAddress, dpnId, extInterfaceName);
 456             return;
 457         }
 458
 459         LOG.warn("Failed to install responder flows for {}. No external interface found for DPN id {}", id, dpnId);
 460
 461         if (!upgradeState.isUpgradeInProgress()) {
 462             return;
 463         }
 464
 465         // The following through the end of the function deals with an upgrade scenario where the neutron configuration
 466         // is restored before the OVS switches reconnect. In such a case, the elan-dpn-interfaces entries will be
 467         // missing from the operational data store. In order to mitigate this we use DataTreeEventCallbackRegistrar
 468         // to wait for the exact operational md-sal object we need to contain the external interface we need.
 469
 470         LOG.info("Upgrade in process, waiting for an external interface to appear on dpn {} for elan {}",
 471                 dpnId, extNetworkId.getValue());
 472
 473         InstanceIdentifier<DpnInterfaces> dpnInterfacesIid =
 474                             elanService.getElanDpnInterfaceOperationalDataPath(extNetworkId.getValue(), dpnId);
 475
 476         eventCallbacks.onAddOrUpdate(LogicalDatastoreType.OPERATIONAL, dpnInterfacesIid, (unused, alsoUnused) -> {
 477             LOG.info("Reattempting write of arp responder for external interfaces for external network {}",
 478                     extNetworkId);
 479             DpnInterfaces dpnInterfaces = elanService.getElanInterfaceInfoByElanDpn(extNetworkId.getValue(), dpnId);
 480             if (dpnInterfaces == null) {
 481                 LOG.error("Could not retrieve DpnInterfaces for {}, {}", extNetworkId.getValue(), dpnId);
 482                 return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 483             }
 484
 485             String extIfc = null;
 486             @Nullable List<String> interfaces = dpnInterfaces.getInterfaces();
 487             if (interfaces != null) {
 488                 for (String dpnInterface : interfaces) {
 489                     if (interfaceManager.isExternalInterface(dpnInterface)) {
 490                         extIfc = dpnInterface;
 491                         break;
 492                     }
 493                 }
 494             }
 495
 496             if (extIfc == null) {
 497                 if (upgradeState.isUpgradeInProgress()) {
 498                     LOG.info("External interface not found yet in elan {} on dpn {}, keep waiting",
 499                             extNetworkId.getValue(), dpnInterfaces);
 500                     return DataTreeEventCallbackRegistrar.NextAction.CALL_AGAIN;
 501                 } else {
 502                     return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 503                 }
 504             }
 505
 506             final String extIfcFinal = extIfc;
 507             doAddArpResponderFlowsToExternalNetworkIps(id, fixedIps, macAddress, dpnId, extIfcFinal);
 508
 509             return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 510         });
 511
 512     }
 513
 514     @Override
 515     public void addArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 516             Uint64 dpnId, String extInterfaceName, int lportTag) {
 517         if (fixedIps == null || fixedIps.isEmpty()) {
 518             LOG.debug("No external IPs defined for {}", id);
 519             return;
 520         }
 521
 522         LOG.info("Installing ARP responder flows for {} fixed-ips {} on switch {}", id, fixedIps, dpnId);
 523
 524         for (String fixedIp : fixedIps) {
 525             IpVersionChoice ipVersionChoice = VpnUtil.getIpVersionFromString(fixedIp);
 526             if (ipVersionChoice == IpVersionChoice.IPV6) {
 527                 continue;
 528             }
 529             installArpResponderFlowsToExternalNetworkIp(macAddress, dpnId, extInterfaceName, lportTag,
 530                     fixedIp);
 531         }
 532     }
 533
 534     private void doAddArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 535             Uint64 dpnId, String extInterfaceName) {
 536         Interface extInterfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, extInterfaceName);
 537         if (extInterfaceState == null) {
 538             LOG.debug("No interface state found for interface {}. Delaying responder flows for {}", extInterfaceName,
 539                     id);
 540             return;
 541         }
 542
 543         Integer lportTag = extInterfaceState.getIfIndex();
 544         if (lportTag == null) {
 545             LOG.debug("No Lport tag found for interface {}. Delaying flows for router-id {}", extInterfaceName, id);
 546             return;
 547         }
 548
 549         if (macAddress == null) {
 550
 551             LOG.debug("Failed to install arp responder flows for router-gateway-ip {} for router {}."
 552                     + "External Gw MacAddress is missing.", fixedIps,  id);
 553             return;
 554         }
 555
 556         addArpResponderFlowsToExternalNetworkIps(id, fixedIps, macAddress, dpnId, extInterfaceName, lportTag);
 557     }
 558
 559     @Override
 560     public void removeArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 561             Uint64 dpnId, Uuid extNetworkId) {
 562
 563         if (dpnId == null || Uint64.ZERO.equals(dpnId)) {
 564             LOG.warn("Failed to remove arp responder flows for router {}. DPN id is missing.", id);
 565             return;
 566         }
 567
 568         String extInterfaceName = elanService.getExternalElanInterface(extNetworkId.getValue(), dpnId);
 569         if (extInterfaceName == null) {
 570             LOG.warn("Failed to remove responder flows for {}. No external interface found for DPN id {}", id, dpnId);
 571             return;
 572         }
 573
 574         Interface extInterfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, extInterfaceName);
 575         if (extInterfaceState == null) {
 576             LOG.debug("No interface state found for interface {}. Delaying responder flows for {}", extInterfaceName,
 577                     id);
 578             return;
 579         }
 580
 581         Integer lportTag = extInterfaceState.getIfIndex();
 582         if (lportTag == null) {
 583             LOG.debug("No Lport tag found for interface {}. Delaying flows for router-id {}", extInterfaceName, id);
 584             return;
 585         }
 586
 587         if (macAddress == null) {
 588
 589             LOG.debug("Failed to remove arp responder flows for router-gateway-ip {} for router {}."
 590                     + "External Gw MacAddress is missing.", fixedIps,  id);
 591             return;
 592         }
 593
 594         removeArpResponderFlowsToExternalNetworkIps(id, fixedIps, dpnId,
 595                 extInterfaceName, lportTag);
 596     }
 597
 598     @Override
 599     public void removeArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps,
 600             Uint64 dpnId, String extInterfaceName, int lportTag) {
 601         if (fixedIps == null || fixedIps.isEmpty()) {
 602             LOG.debug("No external IPs defined for {}", id);
 603             return;
 604         }
 605
 606         LOG.info("Removing ARP responder flows for {} fixed-ips {} on switch {}", id, fixedIps, dpnId);
 607
 608         for (String fixedIp : fixedIps) {
 609             removeArpResponderFlowsToExternalNetworkIp(dpnId, lportTag, fixedIp, extInterfaceName);
 610         }
 611     }
 612
 613     @Override
 614     public String getPrimaryRdFromVpnInstance(VpnInstance vpnInstance) {
 615         return VpnUtil.getPrimaryRd(vpnInstance);
 616     }
 617
 618     private void installArpResponderFlowsToExternalNetworkIp(String macAddress, Uint64 dpnId,
 619             String extInterfaceName, int lportTag, String fixedIp) {
 620         // reset the split-horizon bit to allow traffic to be sent back to the
 621         // provider port
 622         List<Instruction> instructions = new ArrayList<>();
 623         instructions.add(
 624                 new InstructionWriteMetadata(Uint64.ZERO,
 625                         MetaDataUtil.METADATA_MASK_SH_FLAG).buildInstruction(1));
 626         instructions.addAll(
 627                 ArpResponderUtil.getExtInterfaceInstructions(interfaceManager, itmRpcService, extInterfaceName,
 628                         fixedIp, macAddress));
 629         ArpReponderInputBuilder builder = new ArpReponderInputBuilder().setDpId(dpnId.toJava())
 630                 .setInterfaceName(extInterfaceName).setSpa(fixedIp).setSha(macAddress).setLportTag(lportTag);
 631         builder.setInstructions(instructions);
 632         elanService.addArpResponderFlow(builder.buildForInstallFlow());
 633     }
 634
 635     private void removeArpResponderFlowsToExternalNetworkIp(Uint64 dpnId, Integer lportTag, String fixedIp,
 636             String extInterfaceName) {
 637         ArpResponderInput arpInput = new ArpReponderInputBuilder()
 638                 .setDpId(dpnId.toJava()).setInterfaceName(extInterfaceName)
 639                 .setSpa(fixedIp).setLportTag(lportTag).buildForRemoveFlow();
 640         elanService.removeArpResponderFlow(arpInput);
 641     }
 642
 643     @Override
 644     public void onSubnetAddedToVpn(Subnetmap subnetmap, boolean isBgpVpn, Long elanTag) {
 645         vpnSubnetRouteHandler.onSubnetAddedToVpn(subnetmap, isBgpVpn, elanTag);
 646     }
 647
 648     @Override
 649     public void onSubnetDeletedFromVpn(Subnetmap subnetmap, boolean isBgpVpn) {
 650         vpnSubnetRouteHandler.onSubnetDeletedFromVpn(subnetmap, isBgpVpn);
 651     }
 652
 653     @Override
 654     @Nullable
 655     public VpnInstance getVpnInstance(DataBroker broker, String vpnInstanceName) {
 656         return vpnUtil.getVpnInstance(vpnInstanceName);
 657     }
 658
 659     @Override
 660     public String getVpnRd(TypedReadTransaction<Configuration> confTx, String vpnName) {
 661         return VpnUtil.getVpnRd(confTx, vpnName);
 662     }
 663
 664     @Override
 665     public String getVpnRd(DataBroker broker, String vpnName) {
 666         return vpnUtil.getVpnRd(vpnName);
 667     }
 668
 669     @Override
 670     public VpnPortipToPort getNeutronPortFromVpnPortFixedIp(TypedReadTransaction<Configuration> confTx, String vpnName,
 671         String fixedIp) {
 672         return VpnUtil.getNeutronPortFromVpnPortFixedIp(confTx, vpnName, fixedIp);
 673     }
 674
 675     @Override
 676     @Nullable
 677     public VpnPortipToPort getNeutronPortFromVpnPortFixedIp(DataBroker broker, String vpnName, String fixedIp) {
 678         return vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, fixedIp);
 679     }
 680
 681     @Override
 682     public Set<VpnTarget> getRtListForVpn(String vpnName) {
 683         return vpnUtil.getRtListForVpn(dataBroker, vpnName);
 684     }
 685
 686     @Override
 687     public void updateRouteTargetsToSubnetAssociation(Set<VpnTarget> routeTargets, String cidr, String vpnName) {
 688         LoggingFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 689             for (VpnTarget rt : routeTargets) {
 690                 String rtValue = rt.getVrfRTValue();
 691                 switch (rt.getVrfRTType()) {
 692                     case ImportExtcommunity:
 693                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr,  vpnName,
 694                                 tx, false/*isAssociationRemoved*/);
 695                         break;
 696                     case ExportExtcommunity:
 697                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName,
 698                                 tx, false/*isAssociationRemoved*/);
 699                         break;
 700                     case Both:
 701                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr, vpnName,
 702                                 tx, false/*isAssociationRemoved*/);
 703                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName,
 704                                 tx, false/*isAssociationRemoved*/);
 705                         break;
 706                     default:
 707                         LOG.error("updateRouteTargetsToSubnetAssociation: Invalid rt-type {} for vpn {}"
 708                                 + " subnet-cidr {}", rt.getVrfRTType(), vpnName, cidr);
 709                         break;
 710                 }
 711             }
 712         }), LOG, "updateRouteTargetsToSubnetAssociation: Failed for vpn {} subnet-cidr {}", vpnName, cidr);
 713         LOG.info("updateRouteTargetsToSubnetAssociation: Updated RT to subnet association for vpn {} cidr {}",
 714                 vpnName, cidr);
 715     }
 716
 717     @Override
 718     public void removeRouteTargetsToSubnetAssociation(Set<VpnTarget> routeTargets, String cidr, String vpnName) {
 719         LoggingFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 720             for (VpnTarget rt : routeTargets) {
 721                 String rtValue = rt.getVrfRTValue();
 722                 switch (rt.getVrfRTType()) {
 723                     case ImportExtcommunity:
 724                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr, vpnName, tx,
 725                                 true/*isAssociationRemoved*/);
 726                         break;
 727                     case ExportExtcommunity:
 728                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName, tx,
 729                                 true/*isAssociationRemoved*/);
 730                         break;
 731                     case Both:
 732                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr, vpnName, tx,
 733                                 true/*isAssociationRemoved*/);
 734                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName, tx,
 735                                 true/*isAssociationRemoved*/);
 736                         break;
 737                     default:
 738                         LOG.error("removeRouteTargetsToSubnetAssociation: Invalid route-target type {} for vpn {}"
 739                                 + " subnet-cidr {}", rt.getVrfRTType(), vpnName, cidr);
 740                         break;
 741                 }
 742             }
 743         }), LOG, "removeRouteTargetsToSubnetAssociation: Failed for vpn {} subnet-cidr {}", vpnName, cidr);
 744         LOG.info("removeRouteTargetsToSubnetAssociation: vpn {} cidr {}", vpnName, cidr);
 745     }
 746
 747     private boolean doesExistingVpnsHaveConflictingSubnet(Set<VpnTarget> routeTargets, String subnetCidr) {
 748         Set<RouteTarget> routeTargetSet = vpnUtil.getRouteTargetSet(routeTargets);
 749         for (RouteTarget routerTarget : routeTargetSet) {
 750             if (routerTarget.getAssociatedSubnet() != null) {
 751                 for (int i = 0; i < routerTarget.getAssociatedSubnet().size(); i++) {
 752                     AssociatedSubnet associatedSubnet =
 753                             new ArrayList<AssociatedSubnet>(routerTarget.nonnullAssociatedSubnet().values()).get(i);
 754                     if (VpnUtil.areSubnetsOverlapping(associatedSubnet.getCidr(), subnetCidr)) {
 755                         return true;
 756                     }
 757                     if (routerTarget.getRtType() == RouteTarget.RtType.ERT) {
 758                         /* Check if there are multiple exports for the input iRT value (iRT in routeTargets)
 759                          *  Example : (1) iRT=A eRT=B subnet-range=S1; OK
 760                          *            (2) iRT=A eRT=B subnet-range=S1; OK
 761                          *            (3) iRT=B eRT=A subnet-range=S2; NOK
 762                          * Check if (1) and (2) are importing the same subnet-range routes to (3) */
 763                         List<AssociatedVpn> multipleAssociatedVpn
 764                                 = new ArrayList<AssociatedVpn>(associatedSubnet.nonnullAssociatedVpn().values());
 765                         if (multipleAssociatedVpn != null && multipleAssociatedVpn.size() > 1) {
 766                             LOG.error("doesExistingVpnsHaveConflictingSubnet: There is an indirect complete  overlap"
 767                                     + " for subnet CIDR {} for rt {} rtType {}", subnetCidr, routerTarget.getRt(),
 768                                     routerTarget.getRtType());
 769                             return true;
 770                         }
 771                         for (int j = i + 1; j < routerTarget.getAssociatedSubnet().size(); j++) {
 772                             if (VpnUtil.areSubnetsOverlapping(associatedSubnet.getCidr(),
 773                                     new ArrayList<AssociatedSubnet>(routerTarget.nonnullAssociatedSubnet()
 774                                             .values()).get(j).getCidr())) {
 775                                 LOG.error("doesExistingVpnsHaveConflictingSubnet: There is an indirect paartial"
 776                                                 + " overlap for subnet CIDR {} for rt {} rtType {}", subnetCidr,
 777                                         routerTarget.getRt(), routerTarget.getRtType());
 778                                 return true;
 779                             }
 780                         }
 781                     }
 782                 }
 783                 /* Check if there are indirect EXPORTS for the eRT value (eRT in routeTargets)
 784                  *  Example : (1) iRT=A eRT=B subnet-range=S1; OK
 785                  *            (2) iRT=B eRT=A subnet-range=S2; OK
 786                  *            (3) iRT=A eRT=B subnet-range=S1; NOK
 787                  * If associatedSubnet is non-null for a routeTarget in (2),
 788                  * it may have already imported routes from (1) */
 789                 if (routerTarget.getRtType() == RouteTarget.RtType.IRT) {
 790                     try {
 791                         Optional<RouteTarget> indirectRts = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 792                                 LogicalDatastoreType.OPERATIONAL, VpnUtil.getRouteTargetsIdentifier(
 793                                         routerTarget.getRt(), RouteTarget.RtType.ERT));
 794                         if (indirectRts.isPresent() && indirectRts.get().getAssociatedSubnet() != null
 795                                 && routerTarget.getAssociatedSubnet() != null) {
 796                             for (AssociatedSubnet associatedSubnet : indirectRts.get().getAssociatedSubnet().values()) {
 797                                 if (VpnUtil.areSubnetsOverlapping(associatedSubnet.getCidr(), subnetCidr)) {
 798                                     LOG.error("doesExistingVpnsHaveConflictingSubnet: There is an indirect overlap for"
 799                                             + " subnet CIDR {} for rt {} rtType {}", subnetCidr, routerTarget.getRt(),
 800                                             routerTarget.getRtType());
 801                                     return true;
 802                                 }
 803                             }
 804                         }
 805                     } catch (InterruptedException | ExecutionException e) {
 806                         LOG.error("doesExistingVpnsHaveConflictingSubnet: Failed to read route targets to subnet"
 807                                 + "association for rt {} type {} subnet-cidr {}", routerTarget.getRt(),
 808                                 RouteTarget.RtType.ERT, subnetCidr);
 809                         return true; //Fail subnet association to avoid further damage to the data-stores
 810                     }
 811                 }
 812             } else {
 813                 LOG.info("doesExistingVpnsHaveConflictingSubnet: No prior associated subnets for rtVal {} rtType {}",
 814                         routerTarget.getRt(), routerTarget.getRtType());
 815             }
 816         }
 817         return false;
 818     }
 819
 820     private void addSubnetAssociationOperationToTx(String rt, RouteTarget.RtType rtType, String cidr,
 821                                                    String vpnName, TypedReadWriteTransaction<Operational> tx,
 822                                                    boolean isAssociationRemoved)
 823             throws InterruptedException, ExecutionException {
 824         if (isAssociationRemoved) {
 825             //Remove RT-Subnet-Vpn Association
 826             Optional<AssociatedSubnet> associatedSubnet =
 827                 tx.read(VpnUtil.getAssociatedSubnetIdentifier(rt, rtType, cidr)).get();
 828             boolean deleteParent = false;
 829             if (associatedSubnet.isPresent()) {
 830                 List<AssociatedVpn> associatedVpns
 831                         = new ArrayList<>(associatedSubnet.get().nonnullAssociatedVpn().values());
 832                 if (associatedVpns == null || associatedVpns.isEmpty()) {
 833                     deleteParent = true;
 834                 } else {
 835                     for (Iterator<AssociatedVpn> iterator = associatedVpns.iterator(); iterator.hasNext();) {
 836                         AssociatedVpn associatedVpn = iterator.next();
 837                         if (Objects.equals(associatedVpn.getName(), vpnName)) {
 838                             iterator.remove();
 839                             break;
 840                         }
 841                     }
 842                     if (associatedVpns.isEmpty()) {
 843                         deleteParent = true;
 844                     }
 845                 }
 846             }
 847             if (deleteParent) {
 848                 deleteParentForSubnetToVpnAssociation(rt, rtType, cidr, tx);
 849             } else {
 850                 //Some other VPNs are also part of this rtVal, rtType and subnetCidr combination.
 851                 //Delete only this AssociatedVpn Object
 852                 tx.delete(VpnUtil.getAssociatedSubnetAndVpnIdentifier(rt, rtType, cidr, vpnName));
 853                 LOG.debug("addSubnetAssocOperationToTx: Removed vpn {} from association rt {} rtType {} cidr {}",
 854                         vpnName, rt, rtType, cidr);
 855             }
 856         } else {
 857             //Add RT-Subnet-Vpn Association
 858             tx.mergeParentStructurePut(VpnUtil.getAssociatedSubnetAndVpnIdentifier(rt, rtType, cidr, vpnName),
 859                     VpnUtil.buildAssociatedSubnetAndVpn(vpnName));
 860         }
 861     }
 862
 863     private void deleteParentForSubnetToVpnAssociation(String rt, RouteTarget.RtType rtType,
 864                                                 String cidr, TypedReadWriteTransaction<Operational> tx)
 865             throws InterruptedException, ExecutionException {
 866         //Check if you need to delete rtVal+rtType or just the subnetCidr
 867         InstanceIdentifier<RouteTarget> rtIdentifier = InstanceIdentifier
 868                 .builder(SubnetsAssociatedToRouteTargets.class).child(RouteTarget.class,
 869                         new RouteTargetKey(rt, rtType)).build();
 870         Optional<RouteTarget> rtToSubnetsAssociation = tx.read(rtIdentifier).get();
 871         if (rtToSubnetsAssociation.isPresent()) {
 872             List<AssociatedSubnet> associatedSubnets = new ArrayList<>(rtToSubnetsAssociation.get()
 873                     .nonnullAssociatedSubnet().values());
 874             if (associatedSubnets != null && !associatedSubnets.isEmpty()) {
 875                 for (Iterator<AssociatedSubnet> iterator = associatedSubnets.iterator(); iterator.hasNext(); ) {
 876                     if (Objects.equals(iterator.next().getCidr(), cidr)) {
 877                         iterator.remove();
 878                         break;
 879                     }
 880                 }
 881                 if (associatedSubnets.isEmpty()) {
 882                     //The entire rt to subnet association is empty
 883                     //Delete the RouteTarget object
 884                     tx.delete(rtIdentifier);
 885                     LOG.debug("deleteParentForSubnetToVpnAssociation: Removed rt {} rtType {} from association,",
 886                             rt, rtType);
 887                 } else {
 888                     //Some other VPNs are also part of this rtVal, rtType combination
 889                     //Delete only this AssociatedSubnet
 890                     tx.delete(VpnUtil.getAssociatedSubnetIdentifier(rt, rtType, cidr));
 891                     LOG.debug("deleteParentForSubnetToVpnAssociation: Removed cidr {} from association rt {}"
 892                             + " rtType {}", cidr, rt, rtType);
 893                 }
 894             }
 895         }
 896     }
 897
 898     @Override
 899     public boolean checkForOverlappingSubnets(Uuid network, List<Subnetmap> subnetmapList, Uuid vpn,
 900                                        Set<VpnTarget> routeTargets, List<String> failedNwList) {
 901         for (Subnetmap subnetmap : subnetmapList) {
 902             //Check if any other subnet that is already part of a different vpn with same rt, has overlapping CIDR
 903             if (checkExistingSubnetWithSameRoutTargets(routeTargets, vpn, subnetmap, failedNwList)) {
 904                 return true;
 905             }
 906         }
 907         return false;
 908     }
 909
 910     private boolean checkExistingSubnetWithSameRoutTargets(Set<VpnTarget> routeTargets, Uuid vpn, Subnetmap subnetmap,
 911                                                    List<String> failedNwList) {
 912         String cidr = String.valueOf(subnetmap.getSubnetIp());
 913         boolean subnetExistsWithSameRt = doesExistingVpnsHaveConflictingSubnet(routeTargets, cidr);
 914         if (subnetExistsWithSameRt) {
 915             failedNwList.add(String.format("Another subnet with the same/overlapping cidr %s as subnet %s"
 916                     + " is already associated to a vpn with routeTargets %s. Ignoring subnet addition to vpn"
 917                     + " %s", cidr, subnetmap.getId().getValue(), routeTargets.toString(), vpn.getValue()));
 918         }
 919         return subnetExistsWithSameRt;
 920     }
 921 }