vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnManagerImpl.java

   1 /*
   2  * Copyright (c) 2015 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  11 import static org.opendaylight.mdsal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12
  13 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  14 import java.util.ArrayList;
  15 import java.util.Collection;
  16 import java.util.Collections;
  17 import java.util.Iterator;
  18 import java.util.List;
  19 import java.util.Objects;
  20 import java.util.Optional;
  21 import java.util.Set;
  22 import java.util.concurrent.ExecutionException;
  23 import java.util.concurrent.Future;
  24 import java.util.concurrent.locks.ReentrantLock;
  25 import javax.annotation.PostConstruct;
  26 import javax.inject.Inject;
  27 import javax.inject.Singleton;
  28 import org.eclipse.jdt.annotation.NonNull;
  29 import org.eclipse.jdt.annotation.Nullable;
  30 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  31 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
  32 import org.opendaylight.genius.infra.Datastore.Configuration;
  33 import org.opendaylight.genius.infra.Datastore.Operational;
  34 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  35 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  36 import org.opendaylight.genius.infra.TypedReadTransaction;
  37 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  38 import org.opendaylight.genius.infra.TypedWriteTransaction;
  39 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  40 import org.opendaylight.genius.mdsalutil.FlowEntity;
  41 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  42 import org.opendaylight.genius.mdsalutil.NwConstants;
  43 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  44 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  45 import org.opendaylight.genius.utils.JvmGlobalLocks;
  46 import org.opendaylight.infrautils.utils.concurrent.LoggingFutures;
  47 import org.opendaylight.infrautils.utils.function.InterruptibleCheckedConsumer;
  48 import org.opendaylight.mdsal.binding.api.DataBroker;
  49 import org.opendaylight.mdsal.common.api.LogicalDatastoreType;
  50 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  51 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput;
  52 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput.ArpReponderInputBuilder;
  53 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderUtil;
  54 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  55 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  56 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  57 import org.opendaylight.netvirt.neutronvpn.api.enums.IpVersionChoice;
  58 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  59 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  60 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  61 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  62 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  63 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  64 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  65 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  66 import org.opendaylight.serviceutils.upgrade.UpgradeState;
  67 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  68 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInput;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInputBuilder;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolOutput;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.SubnetsAssociatedToRouteTargets;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.RouteTarget;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.RouteTargetKey;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.route.target.AssociatedSubnet;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.route.target.associated.subnet.AssociatedVpn;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.VpnInstances;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.Adjacency;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.instances.VpnInstance;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.instances.vpn.instance.vpntargets.VpnTarget;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  90 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  91 import org.opendaylight.yangtools.yang.common.RpcError;
  92 import org.opendaylight.yangtools.yang.common.RpcResult;
  93 import org.opendaylight.yangtools.yang.common.Uint32;
  94 import org.opendaylight.yangtools.yang.common.Uint64;
  95 import org.slf4j.Logger;
  96 import org.slf4j.LoggerFactory;
  97
  98 @Singleton
  99 public class VpnManagerImpl implements IVpnManager {
 100
 101     private static final Logger LOG = LoggerFactory.getLogger(VpnManagerImpl.class);
 102     private final DataBroker dataBroker;
 103     private final ManagedNewTransactionRunner txRunner;
 104     private final IdManagerService idManager;
 105     private final IMdsalApiManager mdsalManager;
 106     private final IElanService elanService;
 107     private final IInterfaceManager interfaceManager;
 108     private final VpnSubnetRouteHandler vpnSubnetRouteHandler;
 109     private final OdlInterfaceRpcService ifaceMgrRpcService;
 110     private final IVpnLinkService ivpnLinkService;
 111     private final IFibManager fibManager;
 112     private final IBgpManager bgpManager;
 113     private final InterVpnLinkCache interVpnLinkCache;
 114     private final DataTreeEventCallbackRegistrar eventCallbacks;
 115     private final UpgradeState upgradeState;
 116     private final ItmRpcService itmRpcService;
 117     private final VpnUtil vpnUtil;
 118
 119     @Inject
 120     public VpnManagerImpl(final DataBroker dataBroker,
 121                           final IdManagerService idManagerService,
 122                           final IMdsalApiManager mdsalManager,
 123                           final IElanService elanService,
 124                           final IInterfaceManager interfaceManager,
 125                           final VpnSubnetRouteHandler vpnSubnetRouteHandler,
 126                           final OdlInterfaceRpcService ifaceMgrRpcService,
 127                           final IVpnLinkService ivpnLinkService,
 128                           final IFibManager fibManager,
 129                           final IBgpManager bgpManager,
 130                           final InterVpnLinkCache interVpnLinkCache,
 131                           final DataTreeEventCallbackRegistrar dataTreeEventCallbackRegistrar,
 132                           final UpgradeState upgradeState,
 133                           final ItmRpcService itmRpcService,
 134                           final VpnUtil vpnUtil) {
 135         this.dataBroker = dataBroker;
 136         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 137         this.idManager = idManagerService;
 138         this.mdsalManager = mdsalManager;
 139         this.elanService = elanService;
 140         this.interfaceManager = interfaceManager;
 141         this.vpnSubnetRouteHandler = vpnSubnetRouteHandler;
 142         this.ifaceMgrRpcService = ifaceMgrRpcService;
 143         this.ivpnLinkService = ivpnLinkService;
 144         this.fibManager = fibManager;
 145         this.bgpManager = bgpManager;
 146         this.interVpnLinkCache = interVpnLinkCache;
 147         this.eventCallbacks = dataTreeEventCallbackRegistrar;
 148         this.upgradeState = upgradeState;
 149         this.itmRpcService = itmRpcService;
 150         this.vpnUtil = vpnUtil;
 151     }
 152
 153     @PostConstruct
 154     public void start() {
 155         LOG.info("{} start", getClass().getSimpleName());
 156         createIdPool();
 157     }
 158
 159     private void createIdPool() {
 160         CreateIdPoolInput createPool = new CreateIdPoolInputBuilder()
 161             .setPoolName(VpnConstants.VPN_IDPOOL_NAME)
 162             .setLow(VpnConstants.VPN_IDPOOL_LOW)
 163             .setHigh(VpnConstants.VPN_IDPOOL_HIGH)
 164             .build();
 165         try {
 166             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPool);
 167             if (result != null && result.get().isSuccessful()) {
 168                 LOG.info("Created IdPool for VPN Service");
 169             } else {
 170                 LOG.error("createIdPool: Unable to create ID pool for VPNService");
 171             }
 172         } catch (InterruptedException | ExecutionException e) {
 173             LOG.error("Failed to create idPool for VPN Service", e);
 174         }
 175
 176         // Now an IdPool for InterVpnLink endpoint's pseudo ports
 177         CreateIdPoolInput createPseudoLporTagPool =
 178             new CreateIdPoolInputBuilder().setPoolName(VpnConstants.PSEUDO_LPORT_TAG_ID_POOL_NAME)
 179                 .setLow(VpnConstants.LOWER_PSEUDO_LPORT_TAG)
 180                 .setHigh(VpnConstants.UPPER_PSEUDO_LPORT_TAG)
 181                 .build();
 182         try {
 183             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPseudoLporTagPool);
 184             if (result != null && result.get().isSuccessful()) {
 185                 LOG.debug("Created IdPool for Pseudo Port tags");
 186             } else {
 187                 StringBuilder errMsg = new StringBuilder();
 188                 if (result != null && result.get() != null) {
 189                     Collection<RpcError> errors = result.get().getErrors();
 190                     for (RpcError err : errors) {
 191                         errMsg.append(err.getMessage()).append("\n");
 192                     }
 193                 }
 194                 LOG.error("IdPool creation for PseudoPort tags failed. Reasons: {}", errMsg);
 195             }
 196         } catch (InterruptedException | ExecutionException e) {
 197             LOG.error("Failed to create idPool for Pseudo Port tags", e);
 198         }
 199     }
 200
 201     @Override
 202     public void addExtraRoute(String vpnName, String destination, String nextHop, String rd, @Nullable String routerID,
 203         Uint32 l3vni, RouteOrigin origin, @Nullable String intfName, @Nullable Adjacency operationalAdj,
 204         VrfEntry.EncapType encapType, Set<String> prefixListForRefreshFib,
 205         @NonNull TypedWriteTransaction<Configuration> confTx) {
 206         //add extra route to vpn mapping; advertise with nexthop as tunnel ip
 207         vpnUtil.syncUpdate(LogicalDatastoreType.OPERATIONAL,
 208                 VpnExtraRouteHelper.getVpnToExtrarouteVrfIdIdentifier(vpnName, rd != null ? rd : routerID,
 209                         destination),
 210                 VpnUtil.getVpnToExtraroute(destination, Collections.singletonList(nextHop)));
 211
 212         Uint64 dpnId = null;
 213         if (intfName != null && !intfName.isEmpty()) {
 214             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
 215             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 216             if (nextHopIp == null || nextHopIp.isEmpty()) {
 217                 LOG.error("addExtraRoute: NextHop for interface {} is null / empty."
 218                         + " Failed advertising extra route for rd {} prefix {} dpn {}", intfName, rd, destination,
 219                         dpnId);
 220                 return;
 221             }
 222             nextHop = nextHopIp;
 223         }
 224
 225         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 226
 227         // TODO: This is a limitation to be stated in docs. When configuring static route to go to
 228         // another VPN, there can only be one nexthop or, at least, the nexthop to the interVpnLink should be in
 229         // first place.
 230         Optional<InterVpnLinkDataComposite> optVpnLink = interVpnLinkCache.getInterVpnLinkByEndpoint(nextHop);
 231         if (optVpnLink.isPresent() && optVpnLink.get().isActive()) {
 232             InterVpnLinkDataComposite interVpnLink = optVpnLink.get();
 233             // If the nexthop is the endpoint of Vpn2, then prefix must be advertised to Vpn1 in DC-GW, with nexthops
 234             // pointing to the DPNs where Vpn1 is instantiated. LFIB in these DPNS must have a flow entry, with lower
 235             // priority, where if Label matches then sets the lportTag of the Vpn2 endpoint and goes to LportDispatcher
 236             // This is like leaking one of the Vpn2 routes towards Vpn1
 237             String srcVpnUuid = interVpnLink.getVpnNameByIpAddress(nextHop);
 238             String dstVpnUuid = interVpnLink.getOtherVpnNameByIpAddress(nextHop);
 239             String dstVpnRd = vpnUtil.getVpnRd(dstVpnUuid);
 240             Uint32 newLabel = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
 241                     VpnUtil.getNextHopLabelKey(dstVpnRd, destination));
 242             if (newLabel.longValue() == VpnConstants.INVALID_LABEL) {
 243                 LOG.error("addExtraRoute: Unable to fetch label from Id Manager. Bailing out of adding intervpnlink"
 244                         + " route for destination {}", destination);
 245                 return;
 246             }
 247             ivpnLinkService.leakRoute(interVpnLink, srcVpnUuid, dstVpnUuid, destination, newLabel, RouteOrigin.STATIC);
 248         } else {
 249             Optional<Routes> optVpnExtraRoutes = VpnExtraRouteHelper
 250                     .getVpnExtraroutes(dataBroker, vpnName, rd != null ? rd : routerID, destination);
 251             if (optVpnExtraRoutes.isPresent()) {
 252                 List<String> nhList = optVpnExtraRoutes.get().getNexthopIpList();
 253                 if (nhList != null && nhList.size() > 1) {
 254                     // If nhList is greater than one for vpnextraroute, a call to populatefib doesn't update vrfentry.
 255                     prefixListForRefreshFib.add(destination);
 256                 } else {
 257                     L3vpnInput input = new L3vpnInput().setNextHop(operationalAdj).setNextHopIp(nextHop)
 258                             .setL3vni(l3vni.longValue())
 259                             .setPrimaryRd(primaryRd).setVpnName(vpnName).setDpnId(dpnId)
 260                             .setEncapType(encapType).setRd(rd).setRouteOrigin(origin);
 261                     L3vpnRegistry.getRegisteredPopulator(encapType).populateFib(input, confTx);
 262                 }
 263             }
 264         }
 265     }
 266
 267     @Override
 268     public void delExtraRoute(String vpnName, String destination, String nextHop, String rd, @Nullable String routerID,
 269                               @Nullable String intfName, @NonNull TypedWriteTransaction<Configuration> confTx,
 270                               @NonNull TypedWriteTransaction<Operational> operTx) {
 271         Uint64 dpnId = null;
 272         String tunnelIp = nextHop;
 273         if (intfName != null && !intfName.isEmpty()) {
 274             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
 275             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 276             if (nextHopIp == null || nextHopIp.isEmpty()) {
 277                 LOG.error("delExtraRoute: NextHop for interface {} is null / empty."
 278                         + " Failed advertising extra route for rd {} prefix {} dpn {}", intfName, rd, destination,
 279                         dpnId);
 280             }
 281             tunnelIp = nextHopIp;
 282         }
 283         if (rd != null) {
 284             String primaryRd = vpnUtil.getVpnRd(vpnName);
 285             removePrefixFromBGP(vpnName, primaryRd, rd, intfName, destination,
 286                                 nextHop, tunnelIp, dpnId, confTx, operTx);
 287             LOG.info("delExtraRoute: Removed extra route {} from interface {} for rd {}", destination, intfName, rd);
 288         } else {
 289             // add FIB route directly
 290             fibManager.removeOrUpdateFibEntry(routerID, destination, tunnelIp, confTx);
 291             LOG.info("delExtraRoute: Removed extra route {} from interface {} for rd {}", destination, intfName,
 292                     routerID);
 293         }
 294     }
 295
 296     @Override
 297     @SuppressWarnings("checkstyle:IllegalCatch")
 298     public void removePrefixFromBGP(String vpnName, String primaryRd, String extraRouteRd, String vpnInterfaceName,
 299                                     String prefix, String nextHop, String nextHopTunnelIp, Uint64 dpnId,
 300                                     TypedWriteTransaction<Configuration> confTx,
 301                                     TypedWriteTransaction<Operational> operTx) {
 302         String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 303         // FIXME: separate out to somehow?
 304         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnNamePrefixKey);
 305         LOG.info("removing prefix {} for nexthop {} in VPN {} rd {}", prefix, nextHop, vpnName, extraRouteRd);
 306         try {
 307             lock.lock();
 308             if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, extraRouteRd, vpnInterfaceName, prefix,
 309                     nextHop, nextHopTunnelIp, operTx)) {
 310                 return;
 311             }
 312             fibManager.removeOrUpdateFibEntry(primaryRd, prefix, nextHopTunnelIp, confTx);
 313             if (VpnUtil.isEligibleForBgp(extraRouteRd, vpnName, dpnId, null /*networkName*/)) {
 314                 // TODO: Might be needed to include nextHop here
 315                 bgpManager.withdrawPrefix(extraRouteRd, prefix);
 316             }
 317             LOG.info("removePrefixFromBGP: VPN WITHDRAW: Removed Fib Entry rd {} prefix {} nexthop {}", extraRouteRd,
 318                     prefix, nextHop);
 319         } catch (RuntimeException e) {
 320             LOG.error("removePrefixFromBGP: Delete prefix {} rd {} nextHop {} failed", prefix, extraRouteRd, nextHop);
 321         } finally {
 322             lock.unlock();
 323         }
 324     }
 325
 326     @Override
 327     public boolean isVPNConfigured() {
 328         InstanceIdentifier<VpnInstances> vpnsIdentifier = InstanceIdentifier.builder(VpnInstances.class).build();
 329         try {
 330             Optional<VpnInstances> optionalVpns =
 331                     SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
 332                             vpnsIdentifier);
 333             if (!optionalVpns.isPresent()
 334                     || optionalVpns.get().getVpnInstance() == null
 335                     || optionalVpns.get().getVpnInstance().isEmpty()) {
 336                 LOG.trace("isVPNConfigured: No VPNs configured.");
 337                 return false;
 338             }
 339         } catch (InterruptedException | ExecutionException e) {
 340             throw new RuntimeException("Error reading VPN " + vpnsIdentifier, e);
 341         }
 342         LOG.trace("isVPNConfigured: VPNs are configured on the system.");
 343         return true;
 344     }
 345
 346     @Override
 347     public void addSubnetMacIntoVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 348             Uint64 dpnId, TypedWriteTransaction<Configuration> confTx)
 349             throws ExecutionException, InterruptedException {
 350         setupSubnetMacInVpnInstance(vpnName, subnetVpnName, srcMacAddress, dpnId,
 351             (vpnId, dpId, subnetVpnId) -> addGwMac(srcMacAddress, confTx, vpnId, dpId, subnetVpnId));
 352     }
 353
 354     @Override
 355     public void removeSubnetMacFromVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 356             Uint64 dpnId, TypedReadWriteTransaction<Configuration> confTx)
 357             throws ExecutionException, InterruptedException {
 358         setupSubnetMacInVpnInstance(vpnName, subnetVpnName, srcMacAddress, dpnId,
 359             (vpnId, dpId, subnetVpnId) -> removeGwMac(srcMacAddress, confTx, vpnId, dpId, subnetVpnId));
 360     }
 361
 362     @FunctionalInterface
 363     private interface VpnInstanceSubnetMacSetupMethod {
 364         void process(Uint32 vpnId, Uint64 dpId, Uint32 subnetVpnId) throws InterruptedException, ExecutionException;
 365     }
 366
 367     private void setupSubnetMacInVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 368             Uint64 dpnId, VpnInstanceSubnetMacSetupMethod consumer)
 369             throws ExecutionException, InterruptedException {
 370         if (vpnName == null) {
 371             LOG.warn("Cannot setup subnet MAC {} on DPN {}, null vpnName", srcMacAddress, dpnId);
 372             return;
 373         }
 374
 375         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 376         Uint32 subnetVpnId = vpnUtil.getVpnId(subnetVpnName);
 377         if (dpnId.equals(Uint64.ZERO)) {
 378             /* Apply the MAC on all DPNs in a VPN */
 379             for (Uint64 dpId : vpnUtil.getDpnsOnVpn(vpnName)) {
 380                 consumer.process(vpnId, dpId, subnetVpnId);
 381             }
 382         } else {
 383             consumer.process(vpnId, dpnId, subnetVpnId);
 384         }
 385     }
 386
 387     private void addGwMac(String srcMacAddress, TypedWriteTransaction<Configuration> tx, Uint32 vpnId, Uint64 dpId,
 388         Uint32 subnetVpnId) {
 389         FlowEntity flowEntity = vpnUtil.buildL3vpnGatewayFlow(dpId, srcMacAddress, vpnId, subnetVpnId);
 390         mdsalManager.addFlow(tx, flowEntity);
 391     }
 392
 393     // TODO skitt Fix the exception handling here
 394     @SuppressWarnings("checkstyle:IllegalCatch")
 395     @SuppressFBWarnings("REC_CATCH_EXCEPTION")
 396     private void removeGwMac(String srcMacAddress, TypedReadWriteTransaction<Configuration> tx, Uint32 vpnId,
 397             Uint64 dpId, Uint32 subnetVpnId) throws ExecutionException, InterruptedException {
 398         mdsalManager.removeFlow(tx, dpId,
 399             VpnUtil.getL3VpnGatewayFlowRef(NwConstants.L3_GW_MAC_TABLE, dpId, vpnId, srcMacAddress, subnetVpnId),
 400             NwConstants.L3_GW_MAC_TABLE);
 401     }
 402
 403     @Override
 404     public void addRouterGwMacFlow(String routerName, String routerGwMac, Uint64 dpnId, Uuid extNetworkId,
 405             String subnetVpnName, TypedWriteTransaction<Configuration> confTx)
 406             throws ExecutionException, InterruptedException {
 407         setupRouterGwMacFlow(routerName, routerGwMac, dpnId, extNetworkId,
 408             vpnId -> addSubnetMacIntoVpnInstance(vpnId, subnetVpnName, routerGwMac, dpnId, confTx), "Installing");
 409     }
 410
 411     @Override
 412     public void removeRouterGwMacFlow(String routerName, String routerGwMac, Uint64 dpnId, Uuid extNetworkId,
 413             String subnetVpnName, TypedReadWriteTransaction<Configuration> confTx)
 414             throws ExecutionException, InterruptedException {
 415         setupRouterGwMacFlow(routerName, routerGwMac, dpnId, extNetworkId,
 416             vpnId -> removeSubnetMacFromVpnInstance(vpnId, subnetVpnName, routerGwMac, dpnId, confTx), "Removing");
 417     }
 418
 419     private void setupRouterGwMacFlow(String routerName, String routerGwMac, Uint64 dpnId, Uuid extNetworkId,
 420             InterruptibleCheckedConsumer<String, ExecutionException> consumer, String operation)
 421             throws ExecutionException, InterruptedException {
 422         if (routerGwMac == null) {
 423             LOG.warn("Failed to handle router GW flow in GW-MAC table. MAC address is missing for router-id {}",
 424                 routerName);
 425             return;
 426         }
 427
 428         if (dpnId == null || Uint64.ZERO.equals(dpnId)) {
 429             LOG.info("setupRouterGwMacFlow: DPN id is missing for router-id {}",
 430                 routerName);
 431             return;
 432         }
 433
 434         Uuid vpnId = vpnUtil.getExternalNetworkVpnId(extNetworkId);
 435         if (vpnId == null) {
 436             LOG.warn("Network {} is not associated with VPN", extNetworkId.getValue());
 437             return;
 438         }
 439
 440         LOG.info("{} router GW MAC flow for router-id {} on switch {}", operation, routerName, dpnId);
 441         consumer.accept(vpnId.getValue());
 442     }
 443
 444     @Override
 445     public void addArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 446             Uint64 dpnId, Uuid extNetworkId) {
 447
 448         if (dpnId == null || Uint64.ZERO.equals(dpnId)) {
 449             LOG.warn("Failed to install arp responder flows for router {}. DPN id is missing.", id);
 450             return;
 451         }
 452
 453         String extInterfaceName = elanService.getExternalElanInterface(extNetworkId.getValue(), dpnId);
 454         if (extInterfaceName != null) {
 455             doAddArpResponderFlowsToExternalNetworkIps(
 456                     id, fixedIps, macAddress, dpnId, extInterfaceName);
 457             return;
 458         }
 459
 460         LOG.warn("Failed to install responder flows for {}. No external interface found for DPN id {}", id, dpnId);
 461
 462         if (!upgradeState.isUpgradeInProgress()) {
 463             return;
 464         }
 465
 466         // The following through the end of the function deals with an upgrade scenario where the neutron configuration
 467         // is restored before the OVS switches reconnect. In such a case, the elan-dpn-interfaces entries will be
 468         // missing from the operational data store. In order to mitigate this we use DataTreeEventCallbackRegistrar
 469         // to wait for the exact operational md-sal object we need to contain the external interface we need.
 470
 471         LOG.info("Upgrade in process, waiting for an external interface to appear on dpn {} for elan {}",
 472                 dpnId, extNetworkId.getValue());
 473
 474         InstanceIdentifier<DpnInterfaces> dpnInterfacesIid =
 475                             elanService.getElanDpnInterfaceOperationalDataPath(extNetworkId.getValue(), dpnId);
 476
 477         eventCallbacks.onAddOrUpdate(LogicalDatastoreType.OPERATIONAL, dpnInterfacesIid, (unused, alsoUnused) -> {
 478             LOG.info("Reattempting write of arp responder for external interfaces for external network {}",
 479                     extNetworkId);
 480             DpnInterfaces dpnInterfaces = elanService.getElanInterfaceInfoByElanDpn(extNetworkId.getValue(), dpnId);
 481             if (dpnInterfaces == null) {
 482                 LOG.error("Could not retrieve DpnInterfaces for {}, {}", extNetworkId.getValue(), dpnId);
 483                 return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 484             }
 485
 486             String extIfc = null;
 487             @Nullable List<String> interfaces = dpnInterfaces.getInterfaces();
 488             if (interfaces != null) {
 489                 for (String dpnInterface : interfaces) {
 490                     if (interfaceManager.isExternalInterface(dpnInterface)) {
 491                         extIfc = dpnInterface;
 492                         break;
 493                     }
 494                 }
 495             }
 496
 497             if (extIfc == null) {
 498                 if (upgradeState.isUpgradeInProgress()) {
 499                     LOG.info("External interface not found yet in elan {} on dpn {}, keep waiting",
 500                             extNetworkId.getValue(), dpnInterfaces);
 501                     return DataTreeEventCallbackRegistrar.NextAction.CALL_AGAIN;
 502                 } else {
 503                     return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 504                 }
 505             }
 506
 507             final String extIfcFinal = extIfc;
 508             doAddArpResponderFlowsToExternalNetworkIps(id, fixedIps, macAddress, dpnId, extIfcFinal);
 509
 510             return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 511         });
 512
 513     }
 514
 515     @Override
 516     public void addArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 517             Uint64 dpnId, String extInterfaceName, int lportTag) {
 518         if (fixedIps == null || fixedIps.isEmpty()) {
 519             LOG.debug("No external IPs defined for {}", id);
 520             return;
 521         }
 522
 523         LOG.info("Installing ARP responder flows for {} fixed-ips {} on switch {}", id, fixedIps, dpnId);
 524
 525         for (String fixedIp : fixedIps) {
 526             IpVersionChoice ipVersionChoice = VpnUtil.getIpVersionFromString(fixedIp);
 527             if (ipVersionChoice == IpVersionChoice.IPV6) {
 528                 continue;
 529             }
 530             installArpResponderFlowsToExternalNetworkIp(macAddress, dpnId, extInterfaceName, lportTag,
 531                     fixedIp);
 532         }
 533     }
 534
 535     private void doAddArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 536             Uint64 dpnId, String extInterfaceName) {
 537         Interface extInterfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, extInterfaceName);
 538         if (extInterfaceState == null) {
 539             LOG.debug("No interface state found for interface {}. Delaying responder flows for {}", extInterfaceName,
 540                     id);
 541             return;
 542         }
 543
 544         Integer lportTag = extInterfaceState.getIfIndex();
 545         if (lportTag == null) {
 546             LOG.debug("No Lport tag found for interface {}. Delaying flows for router-id {}", extInterfaceName, id);
 547             return;
 548         }
 549
 550         if (macAddress == null) {
 551
 552             LOG.debug("Failed to install arp responder flows for router-gateway-ip {} for router {}."
 553                     + "External Gw MacAddress is missing.", fixedIps,  id);
 554             return;
 555         }
 556
 557         addArpResponderFlowsToExternalNetworkIps(id, fixedIps, macAddress, dpnId, extInterfaceName, lportTag);
 558     }
 559
 560     @Override
 561     public void removeArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 562             Uint64 dpnId, Uuid extNetworkId) {
 563
 564         if (dpnId == null || Uint64.ZERO.equals(dpnId)) {
 565             LOG.warn("Failed to remove arp responder flows for router {}. DPN id is missing.", id);
 566             return;
 567         }
 568
 569         String extInterfaceName = elanService.getExternalElanInterface(extNetworkId.getValue(), dpnId);
 570         if (extInterfaceName == null) {
 571             LOG.warn("Failed to remove responder flows for {}. No external interface found for DPN id {}", id, dpnId);
 572             return;
 573         }
 574
 575         Interface extInterfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, extInterfaceName);
 576         if (extInterfaceState == null) {
 577             LOG.debug("No interface state found for interface {}. Delaying responder flows for {}", extInterfaceName,
 578                     id);
 579             return;
 580         }
 581
 582         Integer lportTag = extInterfaceState.getIfIndex();
 583         if (lportTag == null) {
 584             LOG.debug("No Lport tag found for interface {}. Delaying flows for router-id {}", extInterfaceName, id);
 585             return;
 586         }
 587
 588         if (macAddress == null) {
 589
 590             LOG.debug("Failed to remove arp responder flows for router-gateway-ip {} for router {}."
 591                     + "External Gw MacAddress is missing.", fixedIps,  id);
 592             return;
 593         }
 594
 595         removeArpResponderFlowsToExternalNetworkIps(id, fixedIps, dpnId,
 596                 extInterfaceName, lportTag);
 597     }
 598
 599     @Override
 600     public void removeArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps,
 601             Uint64 dpnId, String extInterfaceName, int lportTag) {
 602         if (fixedIps == null || fixedIps.isEmpty()) {
 603             LOG.debug("No external IPs defined for {}", id);
 604             return;
 605         }
 606
 607         LOG.info("Removing ARP responder flows for {} fixed-ips {} on switch {}", id, fixedIps, dpnId);
 608
 609         for (String fixedIp : fixedIps) {
 610             removeArpResponderFlowsToExternalNetworkIp(dpnId, lportTag, fixedIp, extInterfaceName);
 611         }
 612     }
 613
 614     @Override
 615     public String getPrimaryRdFromVpnInstance(VpnInstance vpnInstance) {
 616         return VpnUtil.getPrimaryRd(vpnInstance);
 617     }
 618
 619     private void installArpResponderFlowsToExternalNetworkIp(String macAddress, Uint64 dpnId,
 620             String extInterfaceName, int lportTag, String fixedIp) {
 621         // reset the split-horizon bit to allow traffic to be sent back to the
 622         // provider port
 623         List<Instruction> instructions = new ArrayList<>();
 624         instructions.add(
 625                 new InstructionWriteMetadata(Uint64.ZERO,
 626                         MetaDataUtil.METADATA_MASK_SH_FLAG).buildInstruction(1));
 627         instructions.addAll(
 628                 ArpResponderUtil.getExtInterfaceInstructions(interfaceManager, itmRpcService, extInterfaceName,
 629                         fixedIp, macAddress));
 630         ArpReponderInputBuilder builder = new ArpReponderInputBuilder().setDpId(dpnId.toJava())
 631                 .setInterfaceName(extInterfaceName).setSpa(fixedIp).setSha(macAddress).setLportTag(lportTag);
 632         builder.setInstructions(instructions);
 633         elanService.addArpResponderFlow(builder.buildForInstallFlow());
 634     }
 635
 636     private void removeArpResponderFlowsToExternalNetworkIp(Uint64 dpnId, Integer lportTag, String fixedIp,
 637             String extInterfaceName) {
 638         ArpResponderInput arpInput = new ArpReponderInputBuilder()
 639                 .setDpId(dpnId.toJava()).setInterfaceName(extInterfaceName)
 640                 .setSpa(fixedIp).setLportTag(lportTag).buildForRemoveFlow();
 641         elanService.removeArpResponderFlow(arpInput);
 642     }
 643
 644     @Override
 645     public void onSubnetAddedToVpn(Subnetmap subnetmap, boolean isBgpVpn, Long elanTag) {
 646         vpnSubnetRouteHandler.onSubnetAddedToVpn(subnetmap, isBgpVpn, elanTag);
 647     }
 648
 649     @Override
 650     public void onSubnetDeletedFromVpn(Subnetmap subnetmap, boolean isBgpVpn) {
 651         vpnSubnetRouteHandler.onSubnetDeletedFromVpn(subnetmap, isBgpVpn);
 652     }
 653
 654     @Override
 655     @Nullable
 656     public VpnInstance getVpnInstance(DataBroker broker, String vpnInstanceName) {
 657         return vpnUtil.getVpnInstance(vpnInstanceName);
 658     }
 659
 660     @Override
 661     public String getVpnRd(TypedReadTransaction<Configuration> confTx, String vpnName) {
 662         return VpnUtil.getVpnRd(confTx, vpnName);
 663     }
 664
 665     @Override
 666     public String getVpnRd(DataBroker broker, String vpnName) {
 667         return vpnUtil.getVpnRd(vpnName);
 668     }
 669
 670     @Override
 671     public VpnPortipToPort getNeutronPortFromVpnPortFixedIp(TypedReadTransaction<Configuration> confTx, String vpnName,
 672         String fixedIp) {
 673         return VpnUtil.getNeutronPortFromVpnPortFixedIp(confTx, vpnName, fixedIp);
 674     }
 675
 676     @Override
 677     @Nullable
 678     public VpnPortipToPort getNeutronPortFromVpnPortFixedIp(DataBroker broker, String vpnName, String fixedIp) {
 679         return vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, fixedIp);
 680     }
 681
 682     @Override
 683     public Set<VpnTarget> getRtListForVpn(String vpnName) {
 684         return vpnUtil.getRtListForVpn(dataBroker, vpnName);
 685     }
 686
 687     @Override
 688     public void updateRouteTargetsToSubnetAssociation(Set<VpnTarget> routeTargets, String cidr, String vpnName) {
 689         LoggingFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 690             for (VpnTarget rt : routeTargets) {
 691                 String rtValue = rt.getVrfRTValue();
 692                 switch (rt.getVrfRTType()) {
 693                     case ImportExtcommunity:
 694                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr,  vpnName,
 695                                 tx, false/*isAssociationRemoved*/);
 696                         break;
 697                     case ExportExtcommunity:
 698                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName,
 699                                 tx, false/*isAssociationRemoved*/);
 700                         break;
 701                     case Both:
 702                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr, vpnName,
 703                                 tx, false/*isAssociationRemoved*/);
 704                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName,
 705                                 tx, false/*isAssociationRemoved*/);
 706                         break;
 707                     default:
 708                         LOG.error("updateRouteTargetsToSubnetAssociation: Invalid rt-type {} for vpn {}"
 709                                 + " subnet-cidr {}", rt.getVrfRTType(), vpnName, cidr);
 710                         break;
 711                 }
 712             }
 713         }), LOG, "updateRouteTargetsToSubnetAssociation: Failed for vpn {} subnet-cidr {}", vpnName, cidr);
 714         LOG.info("updateRouteTargetsToSubnetAssociation: Updated RT to subnet association for vpn {} cidr {}",
 715                 vpnName, cidr);
 716     }
 717
 718     @Override
 719     public void removeRouteTargetsToSubnetAssociation(Set<VpnTarget> routeTargets, String cidr, String vpnName) {
 720         LoggingFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, tx -> {
 721             for (VpnTarget rt : routeTargets) {
 722                 String rtValue = rt.getVrfRTValue();
 723                 switch (rt.getVrfRTType()) {
 724                     case ImportExtcommunity:
 725                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr, vpnName, tx,
 726                                 true/*isAssociationRemoved*/);
 727                         break;
 728                     case ExportExtcommunity:
 729                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName, tx,
 730                                 true/*isAssociationRemoved*/);
 731                         break;
 732                     case Both:
 733                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr, vpnName, tx,
 734                                 true/*isAssociationRemoved*/);
 735                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName, tx,
 736                                 true/*isAssociationRemoved*/);
 737                         break;
 738                     default:
 739                         LOG.error("removeRouteTargetsToSubnetAssociation: Invalid route-target type {} for vpn {}"
 740                                 + " subnet-cidr {}", rt.getVrfRTType(), vpnName, cidr);
 741                         break;
 742                 }
 743             }
 744         }), LOG, "removeRouteTargetsToSubnetAssociation: Failed for vpn {} subnet-cidr {}", vpnName, cidr);
 745         LOG.info("removeRouteTargetsToSubnetAssociation: vpn {} cidr {}", vpnName, cidr);
 746     }
 747
 748     private boolean doesExistingVpnsHaveConflictingSubnet(Set<VpnTarget> routeTargets, String subnetCidr) {
 749         Set<RouteTarget> routeTargetSet = vpnUtil.getRouteTargetSet(routeTargets);
 750         for (RouteTarget routerTarget : routeTargetSet) {
 751             if (routerTarget.getAssociatedSubnet() != null) {
 752                 for (int i = 0; i < routerTarget.getAssociatedSubnet().size(); i++) {
 753                     AssociatedSubnet associatedSubnet = routerTarget.getAssociatedSubnet().get(i);
 754                     if (VpnUtil.areSubnetsOverlapping(associatedSubnet.getCidr(), subnetCidr)) {
 755                         return true;
 756                     }
 757                     if (routerTarget.getRtType() == RouteTarget.RtType.ERT) {
 758                         /* Check if there are multiple exports for the input iRT value (iRT in routeTargets)
 759                          *  Example : (1) iRT=A eRT=B subnet-range=S1; OK
 760                          *            (2) iRT=A eRT=B subnet-range=S1; OK
 761                          *            (3) iRT=B eRT=A subnet-range=S2; NOK
 762                          * Check if (1) and (2) are importing the same subnet-range routes to (3) */
 763                         List<AssociatedVpn> multipleAssociatedVpn = associatedSubnet.getAssociatedVpn();
 764                         if (multipleAssociatedVpn != null && multipleAssociatedVpn.size() > 1) {
 765                             LOG.error("doesExistingVpnsHaveConflictingSubnet: There is an indirect complete  overlap"
 766                                     + " for subnet CIDR {} for rt {} rtType {}", subnetCidr, routerTarget.getRt(),
 767                                     routerTarget.getRtType());
 768                             return true;
 769                         }
 770                         for (int j = i + 1; j < routerTarget.getAssociatedSubnet().size(); j++) {
 771                             if (VpnUtil.areSubnetsOverlapping(associatedSubnet.getCidr(),
 772                                     routerTarget.getAssociatedSubnet().get(j).getCidr())) {
 773                                 LOG.error("doesExistingVpnsHaveConflictingSubnet: There is an indirect paartial"
 774                                                 + " overlap for subnet CIDR {} for rt {} rtType {}", subnetCidr,
 775                                         routerTarget.getRt(), routerTarget.getRtType());
 776                                 return true;
 777                             }
 778                         }
 779                     }
 780                 }
 781                 /* Check if there are indirect EXPORTS for the eRT value (eRT in routeTargets)
 782                  *  Example : (1) iRT=A eRT=B subnet-range=S1; OK
 783                  *            (2) iRT=B eRT=A subnet-range=S2; OK
 784                  *            (3) iRT=A eRT=B subnet-range=S1; NOK
 785                  * If associatedSubnet is non-null for a routeTarget in (2),
 786                  * it may have already imported routes from (1) */
 787                 if (routerTarget.getRtType() == RouteTarget.RtType.IRT) {
 788                     try {
 789                         Optional<RouteTarget> indirectRts = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 790                                 LogicalDatastoreType.OPERATIONAL, VpnUtil.getRouteTargetsIdentifier(
 791                                         routerTarget.getRt(), RouteTarget.RtType.ERT));
 792                         if (indirectRts.isPresent() && indirectRts.get().getAssociatedSubnet() != null
 793                                 && routerTarget.getAssociatedSubnet() != null) {
 794                             for (AssociatedSubnet associatedSubnet : indirectRts.get().getAssociatedSubnet()) {
 795                                 if (VpnUtil.areSubnetsOverlapping(associatedSubnet.getCidr(), subnetCidr)) {
 796                                     LOG.error("doesExistingVpnsHaveConflictingSubnet: There is an indirect overlap for"
 797                                             + " subnet CIDR {} for rt {} rtType {}", subnetCidr, routerTarget.getRt(),
 798                                             routerTarget.getRtType());
 799                                     return true;
 800                                 }
 801                             }
 802                         }
 803                     } catch (InterruptedException | ExecutionException e) {
 804                         LOG.error("doesExistingVpnsHaveConflictingSubnet: Failed to read route targets to subnet"
 805                                 + "association for rt {} type {} subnet-cidr {}", routerTarget.getRt(),
 806                                 RouteTarget.RtType.ERT, subnetCidr);
 807                         return true; //Fail subnet association to avoid further damage to the data-stores
 808                     }
 809                 }
 810             } else {
 811                 LOG.info("doesExistingVpnsHaveConflictingSubnet: No prior associated subnets for rtVal {} rtType {}",
 812                         routerTarget.getRt(), routerTarget.getRtType());
 813             }
 814         }
 815         return false;
 816     }
 817
 818     private void addSubnetAssociationOperationToTx(String rt, RouteTarget.RtType rtType, String cidr,
 819                                                    String vpnName, TypedReadWriteTransaction<Operational> tx,
 820                                                    boolean isAssociationRemoved)
 821             throws InterruptedException, ExecutionException {
 822         if (isAssociationRemoved) {
 823             //Remove RT-Subnet-Vpn Association
 824             Optional<AssociatedSubnet> associatedSubnet =
 825                 tx.read(VpnUtil.getAssociatedSubnetIdentifier(rt, rtType, cidr)).get();
 826             boolean deleteParent = false;
 827             if (associatedSubnet.isPresent()) {
 828                 List<AssociatedVpn> associatedVpns = new ArrayList<>(associatedSubnet.get().nonnullAssociatedVpn());
 829                 if (associatedVpns == null || associatedVpns.isEmpty()) {
 830                     deleteParent = true;
 831                 } else {
 832                     for (Iterator<AssociatedVpn> iterator = associatedVpns.iterator(); iterator.hasNext();) {
 833                         AssociatedVpn associatedVpn = iterator.next();
 834                         if (Objects.equals(associatedVpn.getName(), vpnName)) {
 835                             iterator.remove();
 836                             break;
 837                         }
 838                     }
 839                     if (associatedVpns.isEmpty()) {
 840                         deleteParent = true;
 841                     }
 842                 }
 843             }
 844             if (deleteParent) {
 845                 deleteParentForSubnetToVpnAssociation(rt, rtType, cidr, tx);
 846             } else {
 847                 //Some other VPNs are also part of this rtVal, rtType and subnetCidr combination.
 848                 //Delete only this AssociatedVpn Object
 849                 tx.delete(VpnUtil.getAssociatedSubnetAndVpnIdentifier(rt, rtType, cidr, vpnName));
 850                 LOG.debug("addSubnetAssocOperationToTx: Removed vpn {} from association rt {} rtType {} cidr {}",
 851                         vpnName, rt, rtType, cidr);
 852             }
 853         } else {
 854             //Add RT-Subnet-Vpn Association
 855             tx.put(VpnUtil.getAssociatedSubnetAndVpnIdentifier(rt, rtType, cidr, vpnName),
 856                     VpnUtil.buildAssociatedSubnetAndVpn(vpnName), CREATE_MISSING_PARENTS);
 857         }
 858     }
 859
 860     private void deleteParentForSubnetToVpnAssociation(String rt, RouteTarget.RtType rtType,
 861                                                 String cidr, TypedReadWriteTransaction<Operational> tx)
 862             throws InterruptedException, ExecutionException {
 863         //Check if you need to delete rtVal+rtType or just the subnetCidr
 864         InstanceIdentifier<RouteTarget> rtIdentifier = InstanceIdentifier
 865                 .builder(SubnetsAssociatedToRouteTargets.class).child(RouteTarget.class,
 866                         new RouteTargetKey(rt, rtType)).build();
 867         Optional<RouteTarget> rtToSubnetsAssociation = tx.read(rtIdentifier).get();
 868         if (rtToSubnetsAssociation.isPresent()) {
 869             List<AssociatedSubnet> associatedSubnets = new ArrayList<>(rtToSubnetsAssociation.get()
 870                     .nonnullAssociatedSubnet());
 871             if (associatedSubnets != null && !associatedSubnets.isEmpty()) {
 872                 for (Iterator<AssociatedSubnet> iterator = associatedSubnets.iterator(); iterator.hasNext(); ) {
 873                     if (Objects.equals(iterator.next().getCidr(), cidr)) {
 874                         iterator.remove();
 875                         break;
 876                     }
 877                 }
 878                 if (associatedSubnets.isEmpty()) {
 879                     //The entire rt to subnet association is empty
 880                     //Delete the RouteTarget object
 881                     tx.delete(rtIdentifier);
 882                     LOG.debug("deleteParentForSubnetToVpnAssociation: Removed rt {} rtType {} from association,",
 883                             rt, rtType);
 884                 } else {
 885                     //Some other VPNs are also part of this rtVal, rtType combination
 886                     //Delete only this AssociatedSubnet
 887                     tx.delete(VpnUtil.getAssociatedSubnetIdentifier(rt, rtType, cidr));
 888                     LOG.debug("deleteParentForSubnetToVpnAssociation: Removed cidr {} from association rt {}"
 889                             + " rtType {}", cidr, rt, rtType);
 890                 }
 891             }
 892         }
 893     }
 894
 895     @Override
 896     public boolean checkForOverlappingSubnets(Uuid network, List<Subnetmap> subnetmapList, Uuid vpn,
 897                                        Set<VpnTarget> routeTargets, List<String> failedNwList) {
 898         for (Subnetmap subnetmap : subnetmapList) {
 899             //Check if any other subnet that is already part of a different vpn with same rt, has overlapping CIDR
 900             if (checkExistingSubnetWithSameRoutTargets(routeTargets, vpn, subnetmap, failedNwList)) {
 901                 return true;
 902             }
 903         }
 904         return false;
 905     }
 906
 907     private boolean checkExistingSubnetWithSameRoutTargets(Set<VpnTarget> routeTargets, Uuid vpn, Subnetmap subnetmap,
 908                                                    List<String> failedNwList) {
 909         String cidr = String.valueOf(subnetmap.getSubnetIp());
 910         boolean subnetExistsWithSameRt = doesExistingVpnsHaveConflictingSubnet(routeTargets, cidr);
 911         if (subnetExistsWithSameRt) {
 912             failedNwList.add(String.format("Another subnet with the same/overlapping cidr %s as subnet %s"
 913                     + " is already associated to a vpn with routeTargets %s. Ignoring subnet addition to vpn"
 914                     + " %s", cidr, subnetmap.getId().getValue(), routeTargets.toString(), vpn.getValue()));
 915         }
 916         return subnetExistsWithSameRt;
 917     }
 918 }