vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnManagerImpl.java

   1 /*
   2  * Copyright (c) 2015 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  12
  13 import java.util.ArrayList;
  14 import java.util.Collection;
  15 import java.util.Collections;
  16 import java.util.Iterator;
  17 import java.util.List;
  18 import java.util.Objects;
  19 import java.util.Set;
  20 import java.util.concurrent.ExecutionException;
  21 import java.util.concurrent.Future;
  22 import java.util.concurrent.locks.ReentrantLock;
  23 import javax.annotation.PostConstruct;
  24 import javax.inject.Inject;
  25 import javax.inject.Singleton;
  26 import org.eclipse.jdt.annotation.NonNull;
  27 import org.eclipse.jdt.annotation.Nullable;
  28 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  29 import org.opendaylight.controller.md.sal.binding.api.ReadWriteTransaction;
  30 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  31 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  32 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  33 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
  34 import org.opendaylight.genius.infra.Datastore.Configuration;
  35 import org.opendaylight.genius.infra.Datastore.Operational;
  36 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  37 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  38 import org.opendaylight.genius.infra.TypedReadTransaction;
  39 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  40 import org.opendaylight.genius.infra.TypedWriteTransaction;
  41 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  42 import org.opendaylight.genius.mdsalutil.FlowEntity;
  43 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  44 import org.opendaylight.genius.mdsalutil.NwConstants;
  45 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  46 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  47 import org.opendaylight.genius.utils.JvmGlobalLocks;
  48 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  49 import org.opendaylight.infrautils.utils.function.InterruptibleCheckedConsumer;
  50 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  51 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput;
  52 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput.ArpReponderInputBuilder;
  53 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderUtil;
  54 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  55 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  56 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  57 import org.opendaylight.netvirt.neutronvpn.api.enums.IpVersionChoice;
  58 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  59 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  60 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  61 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  62 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  63 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  64 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  65 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  66 import org.opendaylight.serviceutils.upgrade.UpgradeState;
  67 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
  68 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.vpntargets.VpnTarget;
  69 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
  70 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  71 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInput;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInputBuilder;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolOutput;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.SubnetsAssociatedToRouteTargets;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.RouteTarget;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.RouteTargetKey;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.route.target.AssociatedSubnet;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.route.target.associated.subnet.AssociatedVpn;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  90 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  91 import org.opendaylight.yangtools.yang.common.RpcError;
  92 import org.opendaylight.yangtools.yang.common.RpcResult;
  93 import org.opendaylight.yangtools.yang.common.Uint32;
  94 import org.opendaylight.yangtools.yang.common.Uint64;
  95 import org.slf4j.Logger;
  96 import org.slf4j.LoggerFactory;
  97
  98 @Singleton
  99 public class VpnManagerImpl implements IVpnManager {
 100
 101     private static final Logger LOG = LoggerFactory.getLogger(VpnManagerImpl.class);
 102     private final DataBroker dataBroker;
 103     private final ManagedNewTransactionRunner txRunner;
 104     private final IdManagerService idManager;
 105     private final IMdsalApiManager mdsalManager;
 106     private final IElanService elanService;
 107     private final IInterfaceManager interfaceManager;
 108     private final VpnSubnetRouteHandler vpnSubnetRouteHandler;
 109     private final OdlInterfaceRpcService ifaceMgrRpcService;
 110     private final IVpnLinkService ivpnLinkService;
 111     private final IFibManager fibManager;
 112     private final IBgpManager bgpManager;
 113     private final InterVpnLinkCache interVpnLinkCache;
 114     private final DataTreeEventCallbackRegistrar eventCallbacks;
 115     private final UpgradeState upgradeState;
 116     private final ItmRpcService itmRpcService;
 117     private final VpnUtil vpnUtil;
 118
 119     @Inject
 120     public VpnManagerImpl(final DataBroker dataBroker,
 121                           final IdManagerService idManagerService,
 122                           final IMdsalApiManager mdsalManager,
 123                           final IElanService elanService,
 124                           final IInterfaceManager interfaceManager,
 125                           final VpnSubnetRouteHandler vpnSubnetRouteHandler,
 126                           final OdlInterfaceRpcService ifaceMgrRpcService,
 127                           final IVpnLinkService ivpnLinkService,
 128                           final IFibManager fibManager,
 129                           final IBgpManager bgpManager,
 130                           final InterVpnLinkCache interVpnLinkCache,
 131                           final DataTreeEventCallbackRegistrar dataTreeEventCallbackRegistrar,
 132                           final UpgradeState upgradeState,
 133                           final ItmRpcService itmRpcService,
 134                           final VpnUtil vpnUtil) {
 135         this.dataBroker = dataBroker;
 136         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 137         this.idManager = idManagerService;
 138         this.mdsalManager = mdsalManager;
 139         this.elanService = elanService;
 140         this.interfaceManager = interfaceManager;
 141         this.vpnSubnetRouteHandler = vpnSubnetRouteHandler;
 142         this.ifaceMgrRpcService = ifaceMgrRpcService;
 143         this.ivpnLinkService = ivpnLinkService;
 144         this.fibManager = fibManager;
 145         this.bgpManager = bgpManager;
 146         this.interVpnLinkCache = interVpnLinkCache;
 147         this.eventCallbacks = dataTreeEventCallbackRegistrar;
 148         this.upgradeState = upgradeState;
 149         this.itmRpcService = itmRpcService;
 150         this.vpnUtil = vpnUtil;
 151     }
 152
 153     @PostConstruct
 154     public void start() {
 155         LOG.info("{} start", getClass().getSimpleName());
 156         createIdPool();
 157     }
 158
 159     private void createIdPool() {
 160         CreateIdPoolInput createPool = new CreateIdPoolInputBuilder()
 161             .setPoolName(VpnConstants.VPN_IDPOOL_NAME)
 162             .setLow(VpnConstants.VPN_IDPOOL_LOW)
 163             .setHigh(VpnConstants.VPN_IDPOOL_HIGH)
 164             .build();
 165         try {
 166             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPool);
 167             if (result != null && result.get().isSuccessful()) {
 168                 LOG.info("Created IdPool for VPN Service");
 169             }
 170         } catch (InterruptedException | ExecutionException e) {
 171             LOG.error("Failed to create idPool for VPN Service", e);
 172         }
 173
 174         // Now an IdPool for InterVpnLink endpoint's pseudo ports
 175         CreateIdPoolInput createPseudoLporTagPool =
 176             new CreateIdPoolInputBuilder().setPoolName(VpnConstants.PSEUDO_LPORT_TAG_ID_POOL_NAME)
 177                 .setLow(VpnConstants.LOWER_PSEUDO_LPORT_TAG)
 178                 .setHigh(VpnConstants.UPPER_PSEUDO_LPORT_TAG)
 179                 .build();
 180         try {
 181             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPseudoLporTagPool);
 182             if (result.get().isSuccessful()) {
 183                 LOG.debug("Created IdPool for Pseudo Port tags");
 184             } else {
 185                 Collection<RpcError> errors = result.get().getErrors();
 186                 StringBuilder errMsg = new StringBuilder();
 187                 for (RpcError err : errors) {
 188                     errMsg.append(err.getMessage()).append("\n");
 189                 }
 190                 LOG.error("IdPool creation for PseudoPort tags failed. Reasons: {}", errMsg);
 191             }
 192         } catch (InterruptedException | ExecutionException e) {
 193             LOG.error("Failed to create idPool for Pseudo Port tags", e);
 194         }
 195     }
 196
 197     @Override
 198     public void addExtraRoute(String vpnName, String destination, String nextHop, String rd, @Nullable String routerID,
 199         Uint32 l3vni, RouteOrigin origin, @Nullable String intfName, @Nullable Adjacency operationalAdj,
 200         VrfEntry.EncapType encapType, Set<String> prefixListForRefreshFib,
 201         @NonNull TypedWriteTransaction<Configuration> confTx) {
 202         //add extra route to vpn mapping; advertise with nexthop as tunnel ip
 203         vpnUtil.syncUpdate(LogicalDatastoreType.OPERATIONAL,
 204                 VpnExtraRouteHelper.getVpnToExtrarouteVrfIdIdentifier(vpnName, rd != null ? rd : routerID,
 205                         destination),
 206                 VpnUtil.getVpnToExtraroute(destination, Collections.singletonList(nextHop)));
 207
 208         Uint64 dpnId = null;
 209         if (intfName != null && !intfName.isEmpty()) {
 210             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
 211             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 212             if (nextHopIp == null || nextHopIp.isEmpty()) {
 213                 LOG.error("addExtraRoute: NextHop for interface {} is null / empty."
 214                         + " Failed advertising extra route for rd {} prefix {} dpn {}", intfName, rd, destination,
 215                         dpnId);
 216                 return;
 217             }
 218             nextHop = nextHopIp;
 219         }
 220
 221         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 222
 223         // TODO: This is a limitation to be stated in docs. When configuring static route to go to
 224         // another VPN, there can only be one nexthop or, at least, the nexthop to the interVpnLink should be in
 225         // first place.
 226         Optional<InterVpnLinkDataComposite> optVpnLink = interVpnLinkCache.getInterVpnLinkByEndpoint(nextHop);
 227         if (optVpnLink.isPresent() && optVpnLink.get().isActive()) {
 228             InterVpnLinkDataComposite interVpnLink = optVpnLink.get();
 229             // If the nexthop is the endpoint of Vpn2, then prefix must be advertised to Vpn1 in DC-GW, with nexthops
 230             // pointing to the DPNs where Vpn1 is instantiated. LFIB in these DPNS must have a flow entry, with lower
 231             // priority, where if Label matches then sets the lportTag of the Vpn2 endpoint and goes to LportDispatcher
 232             // This is like leaking one of the Vpn2 routes towards Vpn1
 233             String srcVpnUuid = interVpnLink.getVpnNameByIpAddress(nextHop);
 234             String dstVpnUuid = interVpnLink.getOtherVpnNameByIpAddress(nextHop);
 235             String dstVpnRd = vpnUtil.getVpnRd(dstVpnUuid);
 236             Uint32 newLabel = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
 237                     VpnUtil.getNextHopLabelKey(dstVpnRd, destination));
 238             if (newLabel.longValue() == 0) {
 239                 LOG.error("addExtraRoute: Unable to fetch label from Id Manager. Bailing out of adding intervpnlink"
 240                         + " route for destination {}", destination);
 241                 return;
 242             }
 243             ivpnLinkService.leakRoute(interVpnLink, srcVpnUuid, dstVpnUuid, destination, newLabel, RouteOrigin.STATIC);
 244         } else {
 245             Optional<Routes> optVpnExtraRoutes = VpnExtraRouteHelper
 246                     .getVpnExtraroutes(dataBroker, vpnName, rd != null ? rd : routerID, destination);
 247             if (optVpnExtraRoutes.isPresent()) {
 248                 List<String> nhList = optVpnExtraRoutes.get().getNexthopIpList();
 249                 if (nhList != null && nhList.size() > 1) {
 250                     // If nhList is greater than one for vpnextraroute, a call to populatefib doesn't update vrfentry.
 251                     prefixListForRefreshFib.add(destination);
 252                 } else {
 253                     L3vpnInput input = new L3vpnInput().setNextHop(operationalAdj).setNextHopIp(nextHop)
 254                             .setL3vni(l3vni.longValue())
 255                             .setPrimaryRd(primaryRd).setVpnName(vpnName).setDpnId(dpnId)
 256                             .setEncapType(encapType).setRd(rd).setRouteOrigin(origin);
 257                     L3vpnRegistry.getRegisteredPopulator(encapType).populateFib(input, confTx);
 258                 }
 259             }
 260         }
 261     }
 262
 263     @Override
 264     public void delExtraRoute(String vpnName, String destination, String nextHop, String rd, @Nullable String routerID,
 265                               @Nullable String intfName, @NonNull TypedWriteTransaction<Configuration> confTx,
 266                               @NonNull TypedWriteTransaction<Operational> operTx) {
 267         Uint64 dpnId = null;
 268         String tunnelIp = nextHop;
 269         if (intfName != null && !intfName.isEmpty()) {
 270             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
 271             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 272             if (nextHopIp == null || nextHopIp.isEmpty()) {
 273                 LOG.error("delExtraRoute: NextHop for interface {} is null / empty."
 274                         + " Failed advertising extra route for rd {} prefix {} dpn {}", intfName, rd, destination,
 275                         dpnId);
 276             }
 277             tunnelIp = nextHopIp;
 278         }
 279         if (rd != null) {
 280             String primaryRd = vpnUtil.getVpnRd(vpnName);
 281             removePrefixFromBGP(vpnName, primaryRd, rd, intfName, destination,
 282                                 nextHop, tunnelIp, dpnId, confTx, operTx);
 283             LOG.info("delExtraRoute: Removed extra route {} from interface {} for rd {}", destination, intfName, rd);
 284         } else {
 285             // add FIB route directly
 286             fibManager.removeOrUpdateFibEntry(routerID, destination, tunnelIp, confTx);
 287             LOG.info("delExtraRoute: Removed extra route {} from interface {} for rd {}", destination, intfName,
 288                     routerID);
 289         }
 290     }
 291
 292     @Override
 293     @SuppressWarnings("checkstyle:IllegalCatch")
 294     public void removePrefixFromBGP(String vpnName, String primaryRd, String extraRouteRd, String vpnInterfaceName,
 295                                     String prefix, String nextHop, String nextHopTunnelIp, Uint64 dpnId,
 296                                     TypedWriteTransaction<Configuration> confTx,
 297                                     TypedWriteTransaction<Operational> operTx) {
 298         String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 299         // FIXME: separate out to somehow?
 300         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnNamePrefixKey);
 301         lock.lock();
 302         try {
 303             if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, extraRouteRd, vpnInterfaceName, prefix,
 304                     nextHop, nextHopTunnelIp, operTx)) {
 305                 return;
 306             }
 307             fibManager.removeOrUpdateFibEntry(primaryRd, prefix, nextHopTunnelIp, confTx);
 308             if (VpnUtil.isEligibleForBgp(extraRouteRd, vpnName, dpnId, null /*networkName*/)) {
 309                 // TODO: Might be needed to include nextHop here
 310                 bgpManager.withdrawPrefix(extraRouteRd, prefix);
 311             }
 312             LOG.info("removePrefixFromBGP: VPN WITHDRAW: Removed Fib Entry rd {} prefix {} nexthop {}", extraRouteRd,
 313                     prefix, nextHop);
 314         } catch (RuntimeException e) {
 315             LOG.error("removePrefixFromBGP: Delete prefix {} rd {} nextHop {} failed", prefix, extraRouteRd, nextHop);
 316         } finally {
 317             lock.unlock();
 318         }
 319     }
 320
 321     @Override
 322     public boolean isVPNConfigured() {
 323         InstanceIdentifier<VpnInstances> vpnsIdentifier = InstanceIdentifier.builder(VpnInstances.class).build();
 324         try {
 325             Optional<VpnInstances> optionalVpns =
 326                     SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
 327                             vpnsIdentifier);
 328             if (!optionalVpns.isPresent()
 329                     || optionalVpns.get().getVpnInstance() == null
 330                     || optionalVpns.get().getVpnInstance().isEmpty()) {
 331                 LOG.trace("isVPNConfigured: No VPNs configured.");
 332                 return false;
 333             }
 334         } catch (ReadFailedException e) {
 335             throw new RuntimeException("Error reading VPN " + vpnsIdentifier, e);
 336         }
 337         LOG.trace("isVPNConfigured: VPNs are configured on the system.");
 338         return true;
 339     }
 340
 341     @Override
 342     public void addSubnetMacIntoVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 343             Uint64 dpnId, TypedWriteTransaction<Configuration> confTx)
 344             throws ExecutionException, InterruptedException {
 345         setupSubnetMacInVpnInstance(vpnName, subnetVpnName, srcMacAddress, dpnId,
 346             (vpnId, dpId, subnetVpnId) -> addGwMac(srcMacAddress, confTx, vpnId, dpId, subnetVpnId));
 347     }
 348
 349     @Override
 350     public void removeSubnetMacFromVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 351             Uint64 dpnId, TypedReadWriteTransaction<Configuration> confTx)
 352             throws ExecutionException, InterruptedException {
 353         setupSubnetMacInVpnInstance(vpnName, subnetVpnName, srcMacAddress, dpnId,
 354             (vpnId, dpId, subnetVpnId) -> removeGwMac(srcMacAddress, confTx, vpnId, dpId, subnetVpnId));
 355     }
 356
 357     @FunctionalInterface
 358     private interface VpnInstanceSubnetMacSetupMethod {
 359         void process(Uint32 vpnId, Uint64 dpId, Uint32 subnetVpnId) throws InterruptedException, ExecutionException;
 360     }
 361
 362     private void setupSubnetMacInVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 363             Uint64 dpnId, VpnInstanceSubnetMacSetupMethod consumer)
 364             throws ExecutionException, InterruptedException {
 365         if (vpnName == null) {
 366             LOG.warn("Cannot setup subnet MAC {} on DPN {}, null vpnName", srcMacAddress, dpnId);
 367             return;
 368         }
 369
 370         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 371         Uint32 subnetVpnId = vpnUtil.getVpnId(subnetVpnName);
 372         if (dpnId.equals(Uint64.ZERO)) {
 373             /* Apply the MAC on all DPNs in a VPN */
 374             for (Uint64 dpId : vpnUtil.getDpnsOnVpn(vpnName)) {
 375                 consumer.process(vpnId, dpId, subnetVpnId);
 376             }
 377         } else {
 378             consumer.process(vpnId, dpnId, subnetVpnId);
 379         }
 380     }
 381
 382     private void addGwMac(String srcMacAddress, TypedWriteTransaction<Configuration> tx, Uint32 vpnId, Uint64 dpId,
 383         Uint32 subnetVpnId) {
 384         FlowEntity flowEntity = vpnUtil.buildL3vpnGatewayFlow(dpId, srcMacAddress, vpnId, subnetVpnId);
 385         mdsalManager.addFlow(tx, flowEntity);
 386     }
 387
 388     // TODO skitt Fix the exception handling here
 389     @SuppressWarnings("checkstyle:IllegalCatch")
 390     @SuppressFBWarnings("REC_CATCH_EXCEPTION")
 391     private void removeGwMac(String srcMacAddress, TypedReadWriteTransaction<Configuration> tx, Uint32 vpnId,
 392             Uint64 dpId, Uint32 subnetVpnId) throws ExecutionException, InterruptedException {
 393         mdsalManager.removeFlow(tx, dpId,
 394             VpnUtil.getL3VpnGatewayFlowRef(NwConstants.L3_GW_MAC_TABLE, dpId, vpnId, srcMacAddress, subnetVpnId),
 395             NwConstants.L3_GW_MAC_TABLE);
 396     }
 397
 398     @Override
 399     public void addRouterGwMacFlow(String routerName, String routerGwMac, Uint64 dpnId, Uuid extNetworkId,
 400             String subnetVpnName, TypedWriteTransaction<Configuration> confTx)
 401             throws ExecutionException, InterruptedException {
 402         setupRouterGwMacFlow(routerName, routerGwMac, dpnId, extNetworkId,
 403             vpnId -> addSubnetMacIntoVpnInstance(vpnId, subnetVpnName, routerGwMac, dpnId, confTx), "Installing");
 404     }
 405
 406     @Override
 407     public void removeRouterGwMacFlow(String routerName, String routerGwMac, Uint64 dpnId, Uuid extNetworkId,
 408             String subnetVpnName, TypedReadWriteTransaction<Configuration> confTx)
 409             throws ExecutionException, InterruptedException {
 410         setupRouterGwMacFlow(routerName, routerGwMac, dpnId, extNetworkId,
 411             vpnId -> removeSubnetMacFromVpnInstance(vpnId, subnetVpnName, routerGwMac, dpnId, confTx), "Removing");
 412     }
 413
 414     private void setupRouterGwMacFlow(String routerName, String routerGwMac, Uint64 dpnId, Uuid extNetworkId,
 415             InterruptibleCheckedConsumer<String, ExecutionException> consumer, String operation)
 416             throws ExecutionException, InterruptedException {
 417         if (routerGwMac == null) {
 418             LOG.warn("Failed to handle router GW flow in GW-MAC table. MAC address is missing for router-id {}",
 419                 routerName);
 420             return;
 421         }
 422
 423         if (dpnId == null || Uint64.ZERO.equals(dpnId)) {
 424             LOG.info("setupRouterGwMacFlow: DPN id is missing for router-id {}",
 425                 routerName);
 426             return;
 427         }
 428
 429         Uuid vpnId = vpnUtil.getExternalNetworkVpnId(extNetworkId);
 430         if (vpnId == null) {
 431             LOG.warn("Network {} is not associated with VPN", extNetworkId.getValue());
 432             return;
 433         }
 434
 435         LOG.info("{} router GW MAC flow for router-id {} on switch {}", operation, routerName, dpnId);
 436         consumer.accept(vpnId.getValue());
 437     }
 438
 439     @Override
 440     public void addArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 441             Uint64 dpnId, Uuid extNetworkId) {
 442
 443         if (dpnId == null || Uint64.ZERO.equals(dpnId)) {
 444             LOG.warn("Failed to install arp responder flows for router {}. DPN id is missing.", id);
 445             return;
 446         }
 447
 448         String extInterfaceName = elanService.getExternalElanInterface(extNetworkId.getValue(), dpnId);
 449         if (extInterfaceName != null) {
 450             doAddArpResponderFlowsToExternalNetworkIps(
 451                     id, fixedIps, macAddress, dpnId, extInterfaceName);
 452             return;
 453         }
 454
 455         LOG.warn("Failed to install responder flows for {}. No external interface found for DPN id {}", id, dpnId);
 456
 457         if (!upgradeState.isUpgradeInProgress()) {
 458             return;
 459         }
 460
 461         // The following through the end of the function deals with an upgrade scenario where the neutron configuration
 462         // is restored before the OVS switches reconnect. In such a case, the elan-dpn-interfaces entries will be
 463         // missing from the operational data store. In order to mitigate this we use DataTreeEventCallbackRegistrar
 464         // to wait for the exact operational md-sal object we need to contain the external interface we need.
 465
 466         LOG.info("Upgrade in process, waiting for an external interface to appear on dpn {} for elan {}",
 467                 dpnId, extNetworkId.getValue());
 468
 469         InstanceIdentifier<DpnInterfaces> dpnInterfacesIid =
 470                             elanService.getElanDpnInterfaceOperationalDataPath(extNetworkId.getValue(), dpnId);
 471
 472         eventCallbacks.onAddOrUpdate(LogicalDatastoreType.OPERATIONAL, dpnInterfacesIid, (unused, alsoUnused) -> {
 473             LOG.info("Reattempting write of arp responder for external interfaces for external network {}",
 474                     extNetworkId);
 475             DpnInterfaces dpnInterfaces = elanService.getElanInterfaceInfoByElanDpn(extNetworkId.getValue(), dpnId);
 476             if (dpnInterfaces == null) {
 477                 LOG.error("Could not retrieve DpnInterfaces for {}, {}", extNetworkId.getValue(), dpnId);
 478                 return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 479             }
 480
 481             String extIfc = null;
 482             @Nullable List<String> interfaces = dpnInterfaces.getInterfaces();
 483             if (interfaces != null) {
 484                 for (String dpnInterface : interfaces) {
 485                     if (interfaceManager.isExternalInterface(dpnInterface)) {
 486                         extIfc = dpnInterface;
 487                         break;
 488                     }
 489                 }
 490             }
 491
 492             if (extIfc == null) {
 493                 if (upgradeState.isUpgradeInProgress()) {
 494                     LOG.info("External interface not found yet in elan {} on dpn {}, keep waiting",
 495                             extNetworkId.getValue(), dpnInterfaces);
 496                     return DataTreeEventCallbackRegistrar.NextAction.CALL_AGAIN;
 497                 } else {
 498                     return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 499                 }
 500             }
 501
 502             final String extIfcFinal = extIfc;
 503             doAddArpResponderFlowsToExternalNetworkIps(id, fixedIps, macAddress, dpnId, extIfcFinal);
 504
 505             return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 506         });
 507
 508     }
 509
 510     @Override
 511     public void addArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 512             Uint64 dpnId, String extInterfaceName, int lportTag) {
 513         if (fixedIps == null || fixedIps.isEmpty()) {
 514             LOG.debug("No external IPs defined for {}", id);
 515             return;
 516         }
 517
 518         LOG.info("Installing ARP responder flows for {} fixed-ips {} on switch {}", id, fixedIps, dpnId);
 519
 520         for (String fixedIp : fixedIps) {
 521             IpVersionChoice ipVersionChoice = VpnUtil.getIpVersionFromString(fixedIp);
 522             if (ipVersionChoice == IpVersionChoice.IPV6) {
 523                 continue;
 524             }
 525             installArpResponderFlowsToExternalNetworkIp(macAddress, dpnId, extInterfaceName, lportTag,
 526                     fixedIp);
 527         }
 528     }
 529
 530     private void doAddArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 531             Uint64 dpnId, String extInterfaceName) {
 532         Interface extInterfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, extInterfaceName);
 533         if (extInterfaceState == null) {
 534             LOG.debug("No interface state found for interface {}. Delaying responder flows for {}", extInterfaceName,
 535                     id);
 536             return;
 537         }
 538
 539         Integer lportTag = extInterfaceState.getIfIndex();
 540         if (lportTag == null) {
 541             LOG.debug("No Lport tag found for interface {}. Delaying flows for router-id {}", extInterfaceName, id);
 542             return;
 543         }
 544
 545         if (macAddress == null) {
 546
 547             LOG.debug("Failed to install arp responder flows for router-gateway-ip {} for router {}."
 548                     + "External Gw MacAddress is missing.", fixedIps,  id);
 549             return;
 550         }
 551
 552         addArpResponderFlowsToExternalNetworkIps(id, fixedIps, macAddress, dpnId, extInterfaceName, lportTag);
 553     }
 554
 555     @Override
 556     public void removeArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 557             Uint64 dpnId, Uuid extNetworkId) {
 558
 559         if (dpnId == null || Uint64.ZERO.equals(dpnId)) {
 560             LOG.warn("Failed to remove arp responder flows for router {}. DPN id is missing.", id);
 561             return;
 562         }
 563
 564         String extInterfaceName = elanService.getExternalElanInterface(extNetworkId.getValue(), dpnId);
 565         if (extInterfaceName == null) {
 566             LOG.warn("Failed to remove responder flows for {}. No external interface found for DPN id {}", id, dpnId);
 567             return;
 568         }
 569
 570         Interface extInterfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, extInterfaceName);
 571         if (extInterfaceState == null) {
 572             LOG.debug("No interface state found for interface {}. Delaying responder flows for {}", extInterfaceName,
 573                     id);
 574             return;
 575         }
 576
 577         Integer lportTag = extInterfaceState.getIfIndex();
 578         if (lportTag == null) {
 579             LOG.debug("No Lport tag found for interface {}. Delaying flows for router-id {}", extInterfaceName, id);
 580             return;
 581         }
 582
 583         if (macAddress == null) {
 584
 585             LOG.debug("Failed to remove arp responder flows for router-gateway-ip {} for router {}."
 586                     + "External Gw MacAddress is missing.", fixedIps,  id);
 587             return;
 588         }
 589
 590         removeArpResponderFlowsToExternalNetworkIps(id, fixedIps, dpnId,
 591                 extInterfaceName, lportTag);
 592     }
 593
 594     @Override
 595     public void removeArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps,
 596             Uint64 dpnId, String extInterfaceName, int lportTag) {
 597         if (fixedIps == null || fixedIps.isEmpty()) {
 598             LOG.debug("No external IPs defined for {}", id);
 599             return;
 600         }
 601
 602         LOG.info("Removing ARP responder flows for {} fixed-ips {} on switch {}", id, fixedIps, dpnId);
 603
 604         for (String fixedIp : fixedIps) {
 605             removeArpResponderFlowsToExternalNetworkIp(dpnId, lportTag, fixedIp, extInterfaceName);
 606         }
 607     }
 608
 609     @Override
 610     public String getPrimaryRdFromVpnInstance(VpnInstance vpnInstance) {
 611         return VpnUtil.getPrimaryRd(vpnInstance);
 612     }
 613
 614     private void installArpResponderFlowsToExternalNetworkIp(String macAddress, Uint64 dpnId,
 615             String extInterfaceName, int lportTag, String fixedIp) {
 616         // reset the split-horizon bit to allow traffic to be sent back to the
 617         // provider port
 618         List<Instruction> instructions = new ArrayList<>();
 619         instructions.add(
 620                 new InstructionWriteMetadata(Uint64.ZERO,
 621                         MetaDataUtil.METADATA_MASK_SH_FLAG).buildInstruction(1));
 622         instructions.addAll(
 623                 ArpResponderUtil.getExtInterfaceInstructions(interfaceManager, itmRpcService, extInterfaceName,
 624                         fixedIp, macAddress));
 625         ArpReponderInputBuilder builder = new ArpReponderInputBuilder().setDpId(dpnId.toJava())
 626                 .setInterfaceName(extInterfaceName).setSpa(fixedIp).setSha(macAddress).setLportTag(lportTag);
 627         builder.setInstructions(instructions);
 628         elanService.addArpResponderFlow(builder.buildForInstallFlow());
 629     }
 630
 631     private void removeArpResponderFlowsToExternalNetworkIp(Uint64 dpnId, Integer lportTag, String fixedIp,
 632             String extInterfaceName) {
 633         ArpResponderInput arpInput = new ArpReponderInputBuilder()
 634                 .setDpId(dpnId.toJava()).setInterfaceName(extInterfaceName)
 635                 .setSpa(fixedIp).setLportTag(lportTag).buildForRemoveFlow();
 636         elanService.removeArpResponderFlow(arpInput);
 637     }
 638
 639     @Override
 640     public void onSubnetAddedToVpn(Subnetmap subnetmap, boolean isBgpVpn, Long elanTag) {
 641         vpnSubnetRouteHandler.onSubnetAddedToVpn(subnetmap, isBgpVpn, elanTag);
 642     }
 643
 644     @Override
 645     public void onSubnetDeletedFromVpn(Subnetmap subnetmap, boolean isBgpVpn) {
 646         vpnSubnetRouteHandler.onSubnetDeletedFromVpn(subnetmap, isBgpVpn);
 647     }
 648
 649     @Override
 650     @Nullable
 651     public VpnInstance getVpnInstance(DataBroker broker, String vpnInstanceName) {
 652         return vpnUtil.getVpnInstance(vpnInstanceName);
 653     }
 654
 655     @Override
 656     public String getVpnRd(TypedReadTransaction<Configuration> confTx, String vpnName) {
 657         return VpnUtil.getVpnRd(confTx, vpnName);
 658     }
 659
 660     @Override
 661     public String getVpnRd(DataBroker broker, String vpnName) {
 662         return vpnUtil.getVpnRd(vpnName);
 663     }
 664
 665     @Override
 666     public VpnPortipToPort getNeutronPortFromVpnPortFixedIp(TypedReadTransaction<Configuration> confTx, String vpnName,
 667         String fixedIp) {
 668         return VpnUtil.getNeutronPortFromVpnPortFixedIp(confTx, vpnName, fixedIp);
 669     }
 670
 671     @Override
 672     @Nullable
 673     public VpnPortipToPort getNeutronPortFromVpnPortFixedIp(DataBroker broker, String vpnName, String fixedIp) {
 674         return vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, fixedIp);
 675     }
 676
 677     @Override
 678     public Set<VpnTarget> getRtListForVpn(String vpnName) {
 679         return vpnUtil.getRtListForVpn(vpnName);
 680     }
 681
 682     @Override
 683     public void updateRouteTargetsToSubnetAssociation(Set<VpnTarget> routeTargets, String cidr, String vpnName) {
 684         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
 685             for (VpnTarget rt : routeTargets) {
 686                 String rtValue = rt.getVrfRTValue();
 687                 switch (rt.getVrfRTType()) {
 688                     case ImportExtcommunity:
 689                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr,  vpnName,
 690                                 tx, false/*isAssociationRemoved*/);
 691                         break;
 692                     case ExportExtcommunity:
 693                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName,
 694                                 tx, false/*isAssociationRemoved*/);
 695                         break;
 696                     case Both:
 697                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr, vpnName,
 698                                 tx, false/*isAssociationRemoved*/);
 699                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName,
 700                                 tx, false/*isAssociationRemoved*/);
 701                         break;
 702                     default:
 703                         LOG.error("updateRouteTargetsToSubnetAssociation: Invalid rt-type {} for vpn {}"
 704                                 + " subnet-cidr {}", rt.getVrfRTType(), vpnName, cidr);
 705                         break;
 706                 }
 707             }
 708         }), LOG, "updateRouteTargetsToSubnetAssociation: Failed for vpn {} subnet-cidr {}", vpnName, cidr);
 709         LOG.info("updateRouteTargetsToSubnetAssociation: Updated RT to subnet association for vpn {} cidr {}",
 710                 vpnName, cidr);
 711     }
 712
 713     @Override
 714     public void removeRouteTargetsToSubnetAssociation(Set<VpnTarget> routeTargets, String cidr, String vpnName) {
 715         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
 716             for (VpnTarget rt : routeTargets) {
 717                 String rtValue = rt.getVrfRTValue();
 718                 switch (rt.getVrfRTType()) {
 719                     case ImportExtcommunity:
 720                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr, vpnName, tx,
 721                                 true/*isAssociationRemoved*/);
 722                         break;
 723                     case ExportExtcommunity:
 724                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName, tx,
 725                                 true/*isAssociationRemoved*/);
 726                         break;
 727                     case Both:
 728                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr, vpnName, tx,
 729                                 true/*isAssociationRemoved*/);
 730                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName, tx,
 731                                 true/*isAssociationRemoved*/);
 732                         break;
 733                     default:
 734                         LOG.error("removeRouteTargetsToSubnetAssociation: Invalid route-target type {} for vpn {}"
 735                                 + " subnet-cidr {}", rt.getVrfRTType(), vpnName, cidr);
 736                         break;
 737                 }
 738             }
 739         }), LOG, "removeRouteTargetsToSubnetAssociation: Failed for vpn {} subnet-cidr {}", vpnName, cidr);
 740         LOG.info("removeRouteTargetsToSubnetAssociation: vpn {} cidr {}", vpnName, cidr);
 741     }
 742
 743     private boolean doesExistingVpnsHaveConflictingSubnet(Set<VpnTarget> routeTargets, String subnetCidr) {
 744         Set<RouteTarget> routeTargetSet = vpnUtil.getRouteTargetSet(routeTargets);
 745         for (RouteTarget routerTarget : routeTargetSet) {
 746             if (routerTarget.getAssociatedSubnet() != null) {
 747                 for (int i = 0; i < routerTarget.getAssociatedSubnet().size(); i++) {
 748                     AssociatedSubnet associatedSubnet = routerTarget.getAssociatedSubnet().get(i);
 749                     if (VpnUtil.areSubnetsOverlapping(associatedSubnet.getCidr(), subnetCidr)) {
 750                         return true;
 751                     }
 752                     if (routerTarget.getRtType() == RouteTarget.RtType.ERT) {
 753                         /* Check if there are multiple exports for the input iRT value (iRT in routeTargets)
 754                          *  Example : (1) iRT=A eRT=B subnet-range=S1; OK
 755                          *            (2) iRT=A eRT=B subnet-range=S1; OK
 756                          *            (3) iRT=B eRT=A subnet-range=S2; NOK
 757                          * Check if (1) and (2) are importing the same subnet-range routes to (3) */
 758                         List<AssociatedVpn> multipleAssociatedVpn = associatedSubnet.getAssociatedVpn();
 759                         if (multipleAssociatedVpn != null && multipleAssociatedVpn.size() > 1) {
 760                             LOG.error("doesExistingVpnsHaveConflictingSubnet: There is an indirect complete  overlap"
 761                                     + " for subnet CIDR {} for rt {} rtType {}", subnetCidr, routerTarget.getRt(),
 762                                     routerTarget.getRtType());
 763                             return true;
 764                         }
 765                         for (int j = i + 1; j < routerTarget.getAssociatedSubnet().size(); j++) {
 766                             if (VpnUtil.areSubnetsOverlapping(associatedSubnet.getCidr(),
 767                                     routerTarget.getAssociatedSubnet().get(j).getCidr())) {
 768                                 LOG.error("doesExistingVpnsHaveConflictingSubnet: There is an indirect paartial"
 769                                                 + " overlap for subnet CIDR {} for rt {} rtType {}", subnetCidr,
 770                                         routerTarget.getRt(), routerTarget.getRtType());
 771                                 return true;
 772                             }
 773                         }
 774                     }
 775                 }
 776                 /* Check if there are indirect EXPORTS for the eRT value (eRT in routeTargets)
 777                  *  Example : (1) iRT=A eRT=B subnet-range=S1; OK
 778                  *            (2) iRT=B eRT=A subnet-range=S2; OK
 779                  *            (3) iRT=A eRT=B subnet-range=S1; NOK
 780                  * If associatedSubnet is non-null for a routeTarget in (2),
 781                  * it may have already imported routes from (1) */
 782                 if (routerTarget.getRtType() == RouteTarget.RtType.IRT) {
 783                     try {
 784                         Optional<RouteTarget> indirectRts = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 785                                 LogicalDatastoreType.OPERATIONAL, VpnUtil.getRouteTargetsIdentifier(
 786                                         routerTarget.getRt(), RouteTarget.RtType.ERT));
 787                         if (indirectRts.isPresent() && indirectRts.get().getAssociatedSubnet() != null
 788                                 && routerTarget.getAssociatedSubnet() != null) {
 789                             for (AssociatedSubnet associatedSubnet : indirectRts.get().getAssociatedSubnet()) {
 790                                 if (VpnUtil.areSubnetsOverlapping(associatedSubnet.getCidr(), subnetCidr)) {
 791                                     LOG.error("doesExistingVpnsHaveConflictingSubnet: There is an indirect overlap for"
 792                                             + " subnet CIDR {} for rt {} rtType {}", subnetCidr, routerTarget.getRt(),
 793                                             routerTarget.getRtType());
 794                                     return true;
 795                                 }
 796                             }
 797                         }
 798                     } catch (ReadFailedException e) {
 799                         LOG.error("doesExistingVpnsHaveConflictingSubnet: Failed to read route targets to subnet"
 800                                 + "association for rt {} type {} subnet-cidr {}", routerTarget.getRt(),
 801                                 RouteTarget.RtType.ERT, subnetCidr);
 802                         return true; //Fail subnet association to avoid further damage to the data-stores
 803                     }
 804                 }
 805             } else {
 806                 LOG.info("doesExistingVpnsHaveConflictingSubnet: No prior associated subnets for rtVal {} rtType {}",
 807                         routerTarget.getRt(), routerTarget.getRtType());
 808             }
 809         }
 810         return false;
 811     }
 812
 813     private void addSubnetAssociationOperationToTx(String rt, RouteTarget.RtType rtType, String cidr,
 814                                                    String vpnName, ReadWriteTransaction tx,
 815                                                    boolean isAssociationRemoved)
 816             throws InterruptedException, ExecutionException {
 817         if (isAssociationRemoved) {
 818             //Remove RT-Subnet-Vpn Association
 819             Optional<AssociatedSubnet> associatedSubnet = tx.read(LogicalDatastoreType.OPERATIONAL,
 820                     VpnUtil.getAssociatedSubnetIdentifier(rt, rtType, cidr)).get();
 821             boolean deleteParent = false;
 822             if (associatedSubnet.isPresent()) {
 823                 List<AssociatedVpn> associatedVpns = associatedSubnet.get().getAssociatedVpn();
 824                 if (associatedVpns == null || associatedVpns.isEmpty()) {
 825                     deleteParent = true;
 826                 } else {
 827                     for (Iterator<AssociatedVpn> iterator = associatedVpns.iterator(); iterator.hasNext();) {
 828                         AssociatedVpn associatedVpn = iterator.next();
 829                         if (Objects.equals(associatedVpn.getName(), vpnName)) {
 830                             iterator.remove();
 831                             break;
 832                         }
 833                     }
 834                     if (associatedVpns.isEmpty()) {
 835                         deleteParent = true;
 836                     }
 837                 }
 838             }
 839             if (deleteParent) {
 840                 deleteParentForSubnetToVpnAssocication(rt, rtType, cidr, tx);
 841             } else {
 842                 //Some other VPNs are also part of this rtVal, rtType and subnetCidr combination.
 843                 //Delete only this AssociatedVpn Object
 844                 tx.delete(LogicalDatastoreType.OPERATIONAL,
 845                         VpnUtil.getAssociatedSubnetAndVpnIdentifier(rt, rtType, cidr, vpnName));
 846                 LOG.debug("addSubnetAssocOperationToTx: Removed vpn {} from association rt {} rtType {} cidr {}",
 847                         vpnName, rt, rtType, cidr);
 848             }
 849         } else {
 850             //Add RT-Subnet-Vpn Association
 851             tx.put(LogicalDatastoreType.OPERATIONAL,
 852                     VpnUtil.getAssociatedSubnetAndVpnIdentifier(rt, rtType, cidr, vpnName),
 853                     VpnUtil.buildAssociatedSubnetAndVpn(vpnName), true);
 854         }
 855     }
 856
 857     private void deleteParentForSubnetToVpnAssocication(String rt, RouteTarget.RtType rtType,
 858                                                 String cidr, ReadWriteTransaction tx)
 859             throws InterruptedException, ExecutionException {
 860         //Check if you need to delete rtVal+rtType or just the subnetCidr
 861         InstanceIdentifier<RouteTarget> rtIdentifier = InstanceIdentifier
 862                 .builder(SubnetsAssociatedToRouteTargets.class).child(RouteTarget.class,
 863                         new RouteTargetKey(rt, rtType)).build();
 864         Optional<RouteTarget> rtToSubnetsAssociation = tx.read(LogicalDatastoreType.OPERATIONAL,
 865                 rtIdentifier).get();
 866         if (rtToSubnetsAssociation.isPresent()) {
 867             List<AssociatedSubnet> associatedSubnets = rtToSubnetsAssociation.get().getAssociatedSubnet();
 868             if (associatedSubnets != null && !associatedSubnets.isEmpty()) {
 869                 for (Iterator<AssociatedSubnet> iterator = associatedSubnets.iterator(); iterator.hasNext(); ) {
 870                     if (Objects.equals(iterator.next().getCidr(), cidr)) {
 871                         iterator.remove();
 872                         break;
 873                     }
 874                 }
 875                 if (associatedSubnets.isEmpty()) {
 876                     //The entire rt to subnet association is empty
 877                     //Delete the RouteTarget object
 878                     tx.delete(LogicalDatastoreType.OPERATIONAL, rtIdentifier);
 879                     LOG.debug("deleteParentForSubnetToVpnAssocication: Removed rt {} rtType {} from association,",
 880                             rt, rtType);
 881                 } else {
 882                     //Some other VPNs are also part of this rtVal, rtType combination
 883                     //Delete only this AssociatedSubnet
 884                     tx.delete(LogicalDatastoreType.OPERATIONAL, VpnUtil.getAssociatedSubnetIdentifier(rt, rtType,
 885                             cidr));
 886                     LOG.debug("deleteParentForSubnetToVpnAssocication: Removed cidr {} from association rt {}"
 887                             + " rtType {}", cidr, rt, rtType);
 888                 }
 889             }
 890         }
 891     }
 892
 893     @Override
 894     public boolean checkForOverlappingSubnets(Uuid network, List<Subnetmap> subnetmapList, Uuid vpn,
 895                                        Set<VpnTarget> routeTargets, List<String> failedNwList) {
 896         for (int i = 0; i < subnetmapList.size(); i++) {
 897             //Check if any other subnet that is already part of a different vpn with same rt, has overlapping CIDR
 898             if (checkExistingSubnetWithSameRoutTargets(routeTargets, vpn, subnetmapList.get(i), failedNwList)) {
 899                 return true;
 900             }
 901         }
 902         return false;
 903     }
 904
 905     private boolean checkExistingSubnetWithSameRoutTargets(Set<VpnTarget> routeTargets, Uuid vpn, Subnetmap subnetmap,
 906                                                    List<String> failedNwList) {
 907         String cidr = String.valueOf(subnetmap.getSubnetIp());
 908         boolean subnetExistsWithSameRt = doesExistingVpnsHaveConflictingSubnet(routeTargets, cidr);
 909         if (subnetExistsWithSameRt) {
 910             failedNwList.add(String.format("Another subnet with the same/overlapping cidr %s as subnet %s"
 911                     + " is already associated to a vpn with routeTargets %s. Ignoring subnet addition to vpn"
 912                     + " %s", cidr, subnetmap.getId().getValue(), routeTargets.toString(), vpn.getValue()));
 913         }
 914         return subnetExistsWithSameRt;
 915     }
 916 }