vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnManagerImpl.java

   1 /*
   2  * Copyright (c) 2015 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  12 import java.math.BigInteger;
  13 import java.util.ArrayList;
  14 import java.util.Collection;
  15 import java.util.Collections;
  16 import java.util.Iterator;
  17 import java.util.List;
  18 import java.util.Objects;
  19 import java.util.Set;
  20 import java.util.concurrent.ExecutionException;
  21 import java.util.concurrent.Future;
  22 import javax.annotation.PostConstruct;
  23 import javax.inject.Inject;
  24 import javax.inject.Singleton;
  25 import org.eclipse.jdt.annotation.NonNull;
  26 import org.eclipse.jdt.annotation.Nullable;
  27 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  28 import org.opendaylight.controller.md.sal.binding.api.ReadWriteTransaction;
  29 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  30 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  31 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  32 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
  33 import org.opendaylight.genius.infra.Datastore.Configuration;
  34 import org.opendaylight.genius.infra.Datastore.Operational;
  35 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  36 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  37 import org.opendaylight.genius.infra.TypedReadTransaction;
  38 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  39 import org.opendaylight.genius.infra.TypedWriteTransaction;
  40 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  41 import org.opendaylight.genius.mdsalutil.FlowEntity;
  42 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  43 import org.opendaylight.genius.mdsalutil.NwConstants;
  44 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  45 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  46 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  47 import org.opendaylight.infrautils.utils.function.InterruptibleCheckedConsumer;
  48 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  49 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput;
  50 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput.ArpReponderInputBuilder;
  51 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderUtil;
  52 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  53 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  54 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  55 import org.opendaylight.netvirt.neutronvpn.api.enums.IpVersionChoice;
  56 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  57 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  58 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  59 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  60 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  61 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  62 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  63 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  64 import org.opendaylight.serviceutils.upgrade.UpgradeState;
  65 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
  66 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.vpntargets.VpnTarget;
  67 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
  68 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  69 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInput;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInputBuilder;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolOutput;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.SubnetsAssociatedToRouteTargets;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.RouteTarget;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.RouteTargetKey;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.route.target.AssociatedSubnet;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.route.target.associated.subnet.AssociatedVpn;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  88 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  89 import org.opendaylight.yangtools.yang.common.RpcError;
  90 import org.opendaylight.yangtools.yang.common.RpcResult;
  91 import org.slf4j.Logger;
  92 import org.slf4j.LoggerFactory;
  93
  94 @Singleton
  95 public class VpnManagerImpl implements IVpnManager {
  96
  97     private static final Logger LOG = LoggerFactory.getLogger(VpnManagerImpl.class);
  98     private final DataBroker dataBroker;
  99     private final ManagedNewTransactionRunner txRunner;
 100     private final IdManagerService idManager;
 101     private final IMdsalApiManager mdsalManager;
 102     private final IElanService elanService;
 103     private final IInterfaceManager interfaceManager;
 104     private final VpnSubnetRouteHandler vpnSubnetRouteHandler;
 105     private final OdlInterfaceRpcService ifaceMgrRpcService;
 106     private final IVpnLinkService ivpnLinkService;
 107     private final IFibManager fibManager;
 108     private final IBgpManager bgpManager;
 109     private final InterVpnLinkCache interVpnLinkCache;
 110     private final DataTreeEventCallbackRegistrar eventCallbacks;
 111     private final UpgradeState upgradeState;
 112     private final ItmRpcService itmRpcService;
 113     private final VpnUtil vpnUtil;
 114
 115     @Inject
 116     public VpnManagerImpl(final DataBroker dataBroker,
 117                           final IdManagerService idManagerService,
 118                           final IMdsalApiManager mdsalManager,
 119                           final IElanService elanService,
 120                           final IInterfaceManager interfaceManager,
 121                           final VpnSubnetRouteHandler vpnSubnetRouteHandler,
 122                           final OdlInterfaceRpcService ifaceMgrRpcService,
 123                           final IVpnLinkService ivpnLinkService,
 124                           final IFibManager fibManager,
 125                           final IBgpManager bgpManager,
 126                           final InterVpnLinkCache interVpnLinkCache,
 127                           final DataTreeEventCallbackRegistrar dataTreeEventCallbackRegistrar,
 128                           final UpgradeState upgradeState,
 129                           final ItmRpcService itmRpcService,
 130                           final VpnUtil vpnUtil) {
 131         this.dataBroker = dataBroker;
 132         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 133         this.idManager = idManagerService;
 134         this.mdsalManager = mdsalManager;
 135         this.elanService = elanService;
 136         this.interfaceManager = interfaceManager;
 137         this.vpnSubnetRouteHandler = vpnSubnetRouteHandler;
 138         this.ifaceMgrRpcService = ifaceMgrRpcService;
 139         this.ivpnLinkService = ivpnLinkService;
 140         this.fibManager = fibManager;
 141         this.bgpManager = bgpManager;
 142         this.interVpnLinkCache = interVpnLinkCache;
 143         this.eventCallbacks = dataTreeEventCallbackRegistrar;
 144         this.upgradeState = upgradeState;
 145         this.itmRpcService = itmRpcService;
 146         this.vpnUtil = vpnUtil;
 147     }
 148
 149     @PostConstruct
 150     public void start() {
 151         LOG.info("{} start", getClass().getSimpleName());
 152         createIdPool();
 153     }
 154
 155     private void createIdPool() {
 156         CreateIdPoolInput createPool = new CreateIdPoolInputBuilder()
 157             .setPoolName(VpnConstants.VPN_IDPOOL_NAME)
 158             .setLow(VpnConstants.VPN_IDPOOL_LOW)
 159             .setHigh(VpnConstants.VPN_IDPOOL_HIGH)
 160             .build();
 161         try {
 162             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPool);
 163             if (result != null && result.get().isSuccessful()) {
 164                 LOG.info("Created IdPool for VPN Service");
 165             }
 166         } catch (InterruptedException | ExecutionException e) {
 167             LOG.error("Failed to create idPool for VPN Service", e);
 168         }
 169
 170         // Now an IdPool for InterVpnLink endpoint's pseudo ports
 171         CreateIdPoolInput createPseudoLporTagPool =
 172             new CreateIdPoolInputBuilder().setPoolName(VpnConstants.PSEUDO_LPORT_TAG_ID_POOL_NAME)
 173                 .setLow(VpnConstants.LOWER_PSEUDO_LPORT_TAG)
 174                 .setHigh(VpnConstants.UPPER_PSEUDO_LPORT_TAG)
 175                 .build();
 176         try {
 177             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPseudoLporTagPool);
 178             if (result.get().isSuccessful()) {
 179                 LOG.debug("Created IdPool for Pseudo Port tags");
 180             } else {
 181                 Collection<RpcError> errors = result.get().getErrors();
 182                 StringBuilder errMsg = new StringBuilder();
 183                 for (RpcError err : errors) {
 184                     errMsg.append(err.getMessage()).append("\n");
 185                 }
 186                 LOG.error("IdPool creation for PseudoPort tags failed. Reasons: {}", errMsg);
 187             }
 188         } catch (InterruptedException | ExecutionException e) {
 189             LOG.error("Failed to create idPool for Pseudo Port tags", e);
 190         }
 191     }
 192
 193     @Override
 194     public void addExtraRoute(String vpnName, String destination, String nextHop, String rd, @Nullable String routerID,
 195         Long l3vni, RouteOrigin origin, @Nullable String intfName, @Nullable Adjacency operationalAdj,
 196         VrfEntry.EncapType encapType, Set<String> prefixListForRefreshFib,
 197         @NonNull TypedWriteTransaction<Configuration> confTx) {
 198         //add extra route to vpn mapping; advertise with nexthop as tunnel ip
 199         vpnUtil.syncUpdate(LogicalDatastoreType.OPERATIONAL,
 200                 VpnExtraRouteHelper.getVpnToExtrarouteVrfIdIdentifier(vpnName, rd != null ? rd : routerID,
 201                         destination),
 202                 VpnUtil.getVpnToExtraroute(destination, Collections.singletonList(nextHop)));
 203
 204         BigInteger dpnId = null;
 205         if (intfName != null && !intfName.isEmpty()) {
 206             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
 207             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 208             if (nextHopIp == null || nextHopIp.isEmpty()) {
 209                 LOG.error("addExtraRoute: NextHop for interface {} is null / empty."
 210                         + " Failed advertising extra route for rd {} prefix {} dpn {}", intfName, rd, destination,
 211                         dpnId);
 212                 return;
 213             }
 214             nextHop = nextHopIp;
 215         }
 216
 217         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 218
 219         // TODO: This is a limitation to be stated in docs. When configuring static route to go to
 220         // another VPN, there can only be one nexthop or, at least, the nexthop to the interVpnLink should be in
 221         // first place.
 222         Optional<InterVpnLinkDataComposite> optVpnLink = interVpnLinkCache.getInterVpnLinkByEndpoint(nextHop);
 223         if (optVpnLink.isPresent() && optVpnLink.get().isActive()) {
 224             InterVpnLinkDataComposite interVpnLink = optVpnLink.get();
 225             // If the nexthop is the endpoint of Vpn2, then prefix must be advertised to Vpn1 in DC-GW, with nexthops
 226             // pointing to the DPNs where Vpn1 is instantiated. LFIB in these DPNS must have a flow entry, with lower
 227             // priority, where if Label matches then sets the lportTag of the Vpn2 endpoint and goes to LportDispatcher
 228             // This is like leaking one of the Vpn2 routes towards Vpn1
 229             String srcVpnUuid = interVpnLink.getVpnNameByIpAddress(nextHop);
 230             String dstVpnUuid = interVpnLink.getOtherVpnNameByIpAddress(nextHop);
 231             String dstVpnRd = vpnUtil.getVpnRd(dstVpnUuid);
 232             long newLabel = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
 233                     VpnUtil.getNextHopLabelKey(dstVpnRd, destination));
 234             if (newLabel == 0) {
 235                 LOG.error("addExtraRoute: Unable to fetch label from Id Manager. Bailing out of adding intervpnlink"
 236                         + " route for destination {}", destination);
 237                 return;
 238             }
 239             ivpnLinkService.leakRoute(interVpnLink, srcVpnUuid, dstVpnUuid, destination, newLabel, RouteOrigin.STATIC);
 240         } else {
 241             Optional<Routes> optVpnExtraRoutes = VpnExtraRouteHelper
 242                     .getVpnExtraroutes(dataBroker, vpnName, rd != null ? rd : routerID, destination);
 243             if (optVpnExtraRoutes.isPresent()) {
 244                 List<String> nhList = optVpnExtraRoutes.get().getNexthopIpList();
 245                 if (nhList != null && nhList.size() > 1) {
 246                     // If nhList is greater than one for vpnextraroute, a call to populatefib doesn't update vrfentry.
 247                     prefixListForRefreshFib.add(destination);
 248                 } else {
 249                     L3vpnInput input = new L3vpnInput().setNextHop(operationalAdj).setNextHopIp(nextHop).setL3vni(l3vni)
 250                             .setPrimaryRd(primaryRd).setVpnName(vpnName).setDpnId(dpnId)
 251                             .setEncapType(encapType).setRd(rd).setRouteOrigin(origin);
 252                     L3vpnRegistry.getRegisteredPopulator(encapType).populateFib(input, confTx);
 253                 }
 254             }
 255         }
 256     }
 257
 258     @Override
 259     public void delExtraRoute(String vpnName, String destination, String nextHop, String rd, @Nullable String routerID,
 260                               @Nullable String intfName, @NonNull TypedWriteTransaction<Configuration> confTx,
 261                               @NonNull TypedWriteTransaction<Operational> operTx) {
 262         BigInteger dpnId = null;
 263         String tunnelIp = nextHop;
 264         if (intfName != null && !intfName.isEmpty()) {
 265             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
 266             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 267             if (nextHopIp == null || nextHopIp.isEmpty()) {
 268                 LOG.error("delExtraRoute: NextHop for interface {} is null / empty."
 269                         + " Failed advertising extra route for rd {} prefix {} dpn {}", intfName, rd, destination,
 270                         dpnId);
 271             }
 272             tunnelIp = nextHopIp;
 273         }
 274         if (rd != null) {
 275             String primaryRd = vpnUtil.getVpnRd(vpnName);
 276             removePrefixFromBGP(vpnName, primaryRd, rd, intfName, destination,
 277                                 nextHop, tunnelIp, dpnId, confTx, operTx);
 278             LOG.info("delExtraRoute: Removed extra route {} from interface {} for rd {}", destination, intfName, rd);
 279         } else {
 280             // add FIB route directly
 281             fibManager.removeOrUpdateFibEntry(routerID, destination, tunnelIp, confTx);
 282             LOG.info("delExtraRoute: Removed extra route {} from interface {} for rd {}", destination, intfName,
 283                     routerID);
 284         }
 285     }
 286
 287     @Override
 288     @SuppressWarnings("checkstyle:IllegalCatch")
 289     public void removePrefixFromBGP(String vpnName, String primaryRd, String extraRouteRd, String vpnInterfaceName,
 290                                     String prefix, String nextHop, String nextHopTunnelIp, BigInteger dpnId,
 291                                     TypedWriteTransaction<Configuration> confTx,
 292                                     TypedWriteTransaction<Operational> operTx) {
 293         try {
 294             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 295             synchronized (vpnNamePrefixKey.intern()) {
 296                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, extraRouteRd, vpnInterfaceName, prefix,
 297                         nextHop, nextHopTunnelIp, operTx)) {
 298                     return;
 299                 }
 300                 fibManager.removeOrUpdateFibEntry(primaryRd, prefix, nextHopTunnelIp, confTx);
 301                 if (VpnUtil.isEligibleForBgp(extraRouteRd, vpnName, dpnId, null /*networkName*/)) {
 302                     // TODO: Might be needed to include nextHop here
 303                     bgpManager.withdrawPrefix(extraRouteRd, prefix);
 304                 }
 305             }
 306             LOG.info("removePrefixFromBGP: VPN WITHDRAW: Removed Fib Entry rd {} prefix {} nexthop {}", extraRouteRd,
 307                     prefix, nextHop);
 308         } catch (RuntimeException e) {
 309             LOG.error("removePrefixFromBGP: Delete prefix {} rd {} nextHop {} failed", prefix, extraRouteRd, nextHop);
 310         }
 311     }
 312
 313     @Override
 314     public boolean isVPNConfigured() {
 315         InstanceIdentifier<VpnInstances> vpnsIdentifier = InstanceIdentifier.builder(VpnInstances.class).build();
 316         try {
 317             Optional<VpnInstances> optionalVpns =
 318                     SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
 319                             vpnsIdentifier);
 320             if (!optionalVpns.isPresent()
 321                     || optionalVpns.get().getVpnInstance() == null
 322                     || optionalVpns.get().getVpnInstance().isEmpty()) {
 323                 LOG.trace("isVPNConfigured: No VPNs configured.");
 324                 return false;
 325             }
 326         } catch (ReadFailedException e) {
 327             throw new RuntimeException("Error reading VPN " + vpnsIdentifier, e);
 328         }
 329         LOG.trace("isVPNConfigured: VPNs are configured on the system.");
 330         return true;
 331     }
 332
 333     @Override
 334     public void addSubnetMacIntoVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 335             BigInteger dpnId, TypedWriteTransaction<Configuration> confTx)
 336             throws ExecutionException, InterruptedException {
 337         setupSubnetMacInVpnInstance(vpnName, subnetVpnName, srcMacAddress, dpnId,
 338             (vpnId, dpId, subnetVpnId) -> addGwMac(srcMacAddress, confTx, vpnId, dpId, subnetVpnId));
 339     }
 340
 341     @Override
 342     public void removeSubnetMacFromVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 343             BigInteger dpnId, TypedReadWriteTransaction<Configuration> confTx)
 344             throws ExecutionException, InterruptedException {
 345         setupSubnetMacInVpnInstance(vpnName, subnetVpnName, srcMacAddress, dpnId,
 346             (vpnId, dpId, subnetVpnId) -> removeGwMac(srcMacAddress, confTx, vpnId, dpId, subnetVpnId));
 347     }
 348
 349     @FunctionalInterface
 350     private interface VpnInstanceSubnetMacSetupMethod {
 351         void process(long vpnId, BigInteger dpId, long subnetVpnId) throws InterruptedException, ExecutionException;
 352     }
 353
 354     private void setupSubnetMacInVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 355             BigInteger dpnId, VpnInstanceSubnetMacSetupMethod consumer)
 356             throws ExecutionException, InterruptedException {
 357         if (vpnName == null) {
 358             LOG.warn("Cannot setup subnet MAC {} on DPN {}, null vpnName", srcMacAddress, dpnId);
 359             return;
 360         }
 361
 362         long vpnId = vpnUtil.getVpnId(vpnName);
 363         long subnetVpnId = vpnUtil.getVpnId(subnetVpnName);
 364         if (dpnId.equals(BigInteger.ZERO)) {
 365             /* Apply the MAC on all DPNs in a VPN */
 366             for (BigInteger dpId : vpnUtil.getDpnsOnVpn(vpnName)) {
 367                 consumer.process(vpnId, dpId, subnetVpnId);
 368             }
 369         } else {
 370             consumer.process(vpnId, dpnId, subnetVpnId);
 371         }
 372     }
 373
 374     private void addGwMac(String srcMacAddress, TypedWriteTransaction<Configuration> tx, long vpnId, BigInteger dpId,
 375         long subnetVpnId) {
 376         FlowEntity flowEntity = vpnUtil.buildL3vpnGatewayFlow(dpId, srcMacAddress, vpnId, subnetVpnId);
 377         mdsalManager.addFlow(tx, flowEntity);
 378     }
 379
 380     // TODO skitt Fix the exception handling here
 381     @SuppressWarnings("checkstyle:IllegalCatch")
 382     @SuppressFBWarnings("REC_CATCH_EXCEPTION")
 383     private void removeGwMac(String srcMacAddress, TypedReadWriteTransaction<Configuration> tx, long vpnId,
 384             BigInteger dpId, long subnetVpnId) throws ExecutionException, InterruptedException {
 385         mdsalManager.removeFlow(tx, dpId,
 386             VpnUtil.getL3VpnGatewayFlowRef(NwConstants.L3_GW_MAC_TABLE, dpId, vpnId, srcMacAddress, subnetVpnId),
 387             NwConstants.L3_GW_MAC_TABLE);
 388     }
 389
 390     @Override
 391     public void addRouterGwMacFlow(String routerName, String routerGwMac, BigInteger dpnId, Uuid extNetworkId,
 392             String subnetVpnName, TypedWriteTransaction<Configuration> confTx)
 393             throws ExecutionException, InterruptedException {
 394         setupRouterGwMacFlow(routerName, routerGwMac, dpnId, extNetworkId,
 395             vpnId -> addSubnetMacIntoVpnInstance(vpnId, subnetVpnName, routerGwMac, dpnId, confTx), "Installing");
 396     }
 397
 398     @Override
 399     public void removeRouterGwMacFlow(String routerName, String routerGwMac, BigInteger dpnId, Uuid extNetworkId,
 400             String subnetVpnName, TypedReadWriteTransaction<Configuration> confTx)
 401             throws ExecutionException, InterruptedException {
 402         setupRouterGwMacFlow(routerName, routerGwMac, dpnId, extNetworkId,
 403             vpnId -> removeSubnetMacFromVpnInstance(vpnId, subnetVpnName, routerGwMac, dpnId, confTx), "Removing");
 404     }
 405
 406     private void setupRouterGwMacFlow(String routerName, String routerGwMac, BigInteger dpnId, Uuid extNetworkId,
 407             InterruptibleCheckedConsumer<String, ExecutionException> consumer, String operation)
 408             throws ExecutionException, InterruptedException {
 409         if (routerGwMac == null) {
 410             LOG.warn("Failed to handle router GW flow in GW-MAC table. MAC address is missing for router-id {}",
 411                 routerName);
 412             return;
 413         }
 414
 415         if (dpnId == null || BigInteger.ZERO.equals(dpnId)) {
 416             LOG.info("setupRouterGwMacFlow: DPN id is missing for router-id {}",
 417                 routerName);
 418             return;
 419         }
 420
 421         Uuid vpnId = vpnUtil.getExternalNetworkVpnId(extNetworkId);
 422         if (vpnId == null) {
 423             LOG.warn("Network {} is not associated with VPN", extNetworkId.getValue());
 424             return;
 425         }
 426
 427         LOG.info("{} router GW MAC flow for router-id {} on switch {}", operation, routerName, dpnId);
 428         consumer.accept(vpnId.getValue());
 429     }
 430
 431     @Override
 432     public void addArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 433             BigInteger dpnId, Uuid extNetworkId) {
 434
 435         if (dpnId == null || BigInteger.ZERO.equals(dpnId)) {
 436             LOG.warn("Failed to install arp responder flows for router {}. DPN id is missing.", id);
 437             return;
 438         }
 439
 440         String extInterfaceName = elanService.getExternalElanInterface(extNetworkId.getValue(), dpnId);
 441         if (extInterfaceName != null) {
 442             doAddArpResponderFlowsToExternalNetworkIps(
 443                     id, fixedIps, macAddress, dpnId, extInterfaceName);
 444             return;
 445         }
 446
 447         LOG.warn("Failed to install responder flows for {}. No external interface found for DPN id {}", id, dpnId);
 448
 449         if (!upgradeState.isUpgradeInProgress()) {
 450             return;
 451         }
 452
 453         // The following through the end of the function deals with an upgrade scenario where the neutron configuration
 454         // is restored before the OVS switches reconnect. In such a case, the elan-dpn-interfaces entries will be
 455         // missing from the operational data store. In order to mitigate this we use DataTreeEventCallbackRegistrar
 456         // to wait for the exact operational md-sal object we need to contain the external interface we need.
 457
 458         LOG.info("Upgrade in process, waiting for an external interface to appear on dpn {} for elan {}",
 459                 dpnId, extNetworkId.getValue());
 460
 461         InstanceIdentifier<DpnInterfaces> dpnInterfacesIid =
 462                             elanService.getElanDpnInterfaceOperationalDataPath(extNetworkId.getValue(), dpnId);
 463
 464         eventCallbacks.onAddOrUpdate(LogicalDatastoreType.OPERATIONAL, dpnInterfacesIid, (unused, alsoUnused) -> {
 465             LOG.info("Reattempting write of arp responder for external interfaces for external network {}",
 466                     extNetworkId);
 467             DpnInterfaces dpnInterfaces = elanService.getElanInterfaceInfoByElanDpn(extNetworkId.getValue(), dpnId);
 468             if (dpnInterfaces == null) {
 469                 LOG.error("Could not retrieve DpnInterfaces for {}, {}", extNetworkId.getValue(), dpnId);
 470                 return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 471             }
 472
 473             String extIfc = null;
 474             @Nullable List<String> interfaces = dpnInterfaces.getInterfaces();
 475             if (interfaces != null) {
 476                 for (String dpnInterface : interfaces) {
 477                     if (interfaceManager.isExternalInterface(dpnInterface)) {
 478                         extIfc = dpnInterface;
 479                         break;
 480                     }
 481                 }
 482             }
 483
 484             if (extIfc == null) {
 485                 if (upgradeState.isUpgradeInProgress()) {
 486                     LOG.info("External interface not found yet in elan {} on dpn {}, keep waiting",
 487                             extNetworkId.getValue(), dpnInterfaces);
 488                     return DataTreeEventCallbackRegistrar.NextAction.CALL_AGAIN;
 489                 } else {
 490                     return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 491                 }
 492             }
 493
 494             final String extIfcFinal = extIfc;
 495             doAddArpResponderFlowsToExternalNetworkIps(id, fixedIps, macAddress, dpnId, extIfcFinal);
 496
 497             return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 498         });
 499
 500     }
 501
 502     @Override
 503     public void addArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 504             BigInteger dpnId, String extInterfaceName, int lportTag) {
 505         if (fixedIps == null || fixedIps.isEmpty()) {
 506             LOG.debug("No external IPs defined for {}", id);
 507             return;
 508         }
 509
 510         LOG.info("Installing ARP responder flows for {} fixed-ips {} on switch {}", id, fixedIps, dpnId);
 511
 512         for (String fixedIp : fixedIps) {
 513             IpVersionChoice ipVersionChoice = VpnUtil.getIpVersionFromString(fixedIp);
 514             if (ipVersionChoice == IpVersionChoice.IPV6) {
 515                 continue;
 516             }
 517             installArpResponderFlowsToExternalNetworkIp(macAddress, dpnId, extInterfaceName, lportTag,
 518                     fixedIp);
 519         }
 520     }
 521
 522     private void doAddArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 523             BigInteger dpnId, String extInterfaceName) {
 524         Interface extInterfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, extInterfaceName);
 525         if (extInterfaceState == null) {
 526             LOG.debug("No interface state found for interface {}. Delaying responder flows for {}", extInterfaceName,
 527                     id);
 528             return;
 529         }
 530
 531         Integer lportTag = extInterfaceState.getIfIndex();
 532         if (lportTag == null) {
 533             LOG.debug("No Lport tag found for interface {}. Delaying flows for router-id {}", extInterfaceName, id);
 534             return;
 535         }
 536
 537         if (macAddress == null) {
 538
 539             LOG.debug("Failed to install arp responder flows for router-gateway-ip {} for router {}."
 540                     + "External Gw MacAddress is missing.", fixedIps,  id);
 541             return;
 542         }
 543
 544         addArpResponderFlowsToExternalNetworkIps(id, fixedIps, macAddress, dpnId, extInterfaceName, lportTag);
 545     }
 546
 547     @Override
 548     public void removeArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 549             BigInteger dpnId, Uuid extNetworkId) {
 550
 551         if (dpnId == null || BigInteger.ZERO.equals(dpnId)) {
 552             LOG.warn("Failed to remove arp responder flows for router {}. DPN id is missing.", id);
 553             return;
 554         }
 555
 556         String extInterfaceName = elanService.getExternalElanInterface(extNetworkId.getValue(), dpnId);
 557         if (extInterfaceName == null) {
 558             LOG.warn("Failed to remove responder flows for {}. No external interface found for DPN id {}", id, dpnId);
 559             return;
 560         }
 561
 562         Interface extInterfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, extInterfaceName);
 563         if (extInterfaceState == null) {
 564             LOG.debug("No interface state found for interface {}. Delaying responder flows for {}", extInterfaceName,
 565                     id);
 566             return;
 567         }
 568
 569         Integer lportTag = extInterfaceState.getIfIndex();
 570         if (lportTag == null) {
 571             LOG.debug("No Lport tag found for interface {}. Delaying flows for router-id {}", extInterfaceName, id);
 572             return;
 573         }
 574
 575         if (macAddress == null) {
 576
 577             LOG.debug("Failed to remove arp responder flows for router-gateway-ip {} for router {}."
 578                     + "External Gw MacAddress is missing.", fixedIps,  id);
 579             return;
 580         }
 581
 582         removeArpResponderFlowsToExternalNetworkIps(id, fixedIps, dpnId,
 583                 extInterfaceName, lportTag);
 584     }
 585
 586     @Override
 587     public void removeArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps,
 588             BigInteger dpnId, String extInterfaceName, int lportTag) {
 589         if (fixedIps == null || fixedIps.isEmpty()) {
 590             LOG.debug("No external IPs defined for {}", id);
 591             return;
 592         }
 593
 594         LOG.info("Removing ARP responder flows for {} fixed-ips {} on switch {}", id, fixedIps, dpnId);
 595
 596         for (String fixedIp : fixedIps) {
 597             removeArpResponderFlowsToExternalNetworkIp(dpnId, lportTag, fixedIp, extInterfaceName);
 598         }
 599     }
 600
 601     @Override
 602     public String getPrimaryRdFromVpnInstance(VpnInstance vpnInstance) {
 603         return VpnUtil.getPrimaryRd(vpnInstance);
 604     }
 605
 606     private void installArpResponderFlowsToExternalNetworkIp(String macAddress, BigInteger dpnId,
 607             String extInterfaceName, int lportTag, String fixedIp) {
 608         // reset the split-horizon bit to allow traffic to be sent back to the
 609         // provider port
 610         List<Instruction> instructions = new ArrayList<>();
 611         instructions.add(
 612                 new InstructionWriteMetadata(BigInteger.ZERO, MetaDataUtil.METADATA_MASK_SH_FLAG).buildInstruction(1));
 613         instructions.addAll(
 614                 ArpResponderUtil.getExtInterfaceInstructions(interfaceManager, itmRpcService, extInterfaceName,
 615                         fixedIp, macAddress));
 616         ArpReponderInputBuilder builder = new ArpReponderInputBuilder().setDpId(dpnId)
 617                 .setInterfaceName(extInterfaceName).setSpa(fixedIp).setSha(macAddress).setLportTag(lportTag);
 618         builder.setInstructions(instructions);
 619         elanService.addArpResponderFlow(builder.buildForInstallFlow());
 620     }
 621
 622     private void removeArpResponderFlowsToExternalNetworkIp(BigInteger dpnId, Integer lportTag, String fixedIp,
 623             String extInterfaceName) {
 624         ArpResponderInput arpInput = new ArpReponderInputBuilder().setDpId(dpnId).setInterfaceName(extInterfaceName)
 625                 .setSpa(fixedIp).setLportTag(lportTag).buildForRemoveFlow();
 626         elanService.removeArpResponderFlow(arpInput);
 627     }
 628
 629     @Override
 630     public void onSubnetAddedToVpn(Subnetmap subnetmap, boolean isBgpVpn, Long elanTag) {
 631         vpnSubnetRouteHandler.onSubnetAddedToVpn(subnetmap, isBgpVpn, elanTag);
 632     }
 633
 634     @Override
 635     public void onSubnetDeletedFromVpn(Subnetmap subnetmap, boolean isBgpVpn) {
 636         vpnSubnetRouteHandler.onSubnetDeletedFromVpn(subnetmap, isBgpVpn);
 637     }
 638
 639     @Override
 640     @Nullable
 641     public VpnInstance getVpnInstance(DataBroker broker, String vpnInstanceName) {
 642         return vpnUtil.getVpnInstance(vpnInstanceName);
 643     }
 644
 645     @Override
 646     public String getVpnRd(TypedReadTransaction<Configuration> confTx, String vpnName) {
 647         return VpnUtil.getVpnRd(confTx, vpnName);
 648     }
 649
 650     @Override
 651     public String getVpnRd(DataBroker broker, String vpnName) {
 652         return vpnUtil.getVpnRd(vpnName);
 653     }
 654
 655     @Override
 656     public VpnPortipToPort getNeutronPortFromVpnPortFixedIp(TypedReadTransaction<Configuration> confTx, String vpnName,
 657         String fixedIp) {
 658         return VpnUtil.getNeutronPortFromVpnPortFixedIp(confTx, vpnName, fixedIp);
 659     }
 660
 661     @Override
 662     @Nullable
 663     public VpnPortipToPort getNeutronPortFromVpnPortFixedIp(DataBroker broker, String vpnName, String fixedIp) {
 664         return vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, fixedIp);
 665     }
 666
 667     @Override
 668     public Set<VpnTarget> getRtListForVpn(String vpnName) {
 669         return vpnUtil.getRtListForVpn(vpnName);
 670     }
 671
 672     @Override
 673     public void updateRouteTargetsToSubnetAssociation(Set<VpnTarget> routeTargets, String cidr, String vpnName) {
 674         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
 675             for (VpnTarget rt : routeTargets) {
 676                 String rtValue = rt.getVrfRTValue();
 677                 switch (rt.getVrfRTType()) {
 678                     case ImportExtcommunity:
 679                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr,  vpnName,
 680                                 tx, false/*isAssociationRemoved*/);
 681                         break;
 682                     case ExportExtcommunity:
 683                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName,
 684                                 tx, false/*isAssociationRemoved*/);
 685                         break;
 686                     case Both:
 687                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr, vpnName,
 688                                 tx, false/*isAssociationRemoved*/);
 689                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName,
 690                                 tx, false/*isAssociationRemoved*/);
 691                         break;
 692                     default:
 693                         LOG.error("updateRouteTargetsToSubnetAssociation: Invalid rt-type {} for vpn {}"
 694                                 + " subnet-cidr {}", rt.getVrfRTType(), vpnName, cidr);
 695                         break;
 696                 }
 697             }
 698         }), LOG, "updateRouteTargetsToSubnetAssociation: Failed for vpn {} subnet-cidr {}", vpnName, cidr);
 699         LOG.info("updateRouteTargetsToSubnetAssociation: Updated RT to subnet association for vpn {} cidr {}",
 700                 vpnName, cidr);
 701     }
 702
 703     @Override
 704     public void removeRouteTargetsToSubnetAssociation(Set<VpnTarget> routeTargets, String cidr, String vpnName) {
 705         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
 706             for (VpnTarget rt : routeTargets) {
 707                 String rtValue = rt.getVrfRTValue();
 708                 switch (rt.getVrfRTType()) {
 709                     case ImportExtcommunity:
 710                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr, vpnName, tx,
 711                                 true/*isAssociationRemoved*/);
 712                         break;
 713                     case ExportExtcommunity:
 714                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName, tx,
 715                                 true/*isAssociationRemoved*/);
 716                         break;
 717                     case Both:
 718                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr, vpnName, tx,
 719                                 true/*isAssociationRemoved*/);
 720                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName, tx,
 721                                 true/*isAssociationRemoved*/);
 722                         break;
 723                     default:
 724                         LOG.error("removeRouteTargetsToSubnetAssociation: Invalid route-target type {} for vpn {}"
 725                                 + " subnet-cidr {}", rt.getVrfRTType(), vpnName, cidr);
 726                         break;
 727                 }
 728             }
 729         }), LOG, "removeRouteTargetsToSubnetAssociation: Failed for vpn {} subnet-cidr {}", vpnName, cidr);
 730         LOG.info("removeRouteTargetsToSubnetAssociation: vpn {} cidr {}", vpnName, cidr);
 731     }
 732
 733     private boolean doesExistingVpnsHaveConflictingSubnet(Set<VpnTarget> routeTargets, String subnetCidr) {
 734         Set<RouteTarget> routeTargetSet = vpnUtil.getRouteTargetSet(routeTargets);
 735         for (RouteTarget routerTarget : routeTargetSet) {
 736             if (routerTarget.getAssociatedSubnet() != null) {
 737                 for (int i = 0; i < routerTarget.getAssociatedSubnet().size(); i++) {
 738                     AssociatedSubnet associatedSubnet = routerTarget.getAssociatedSubnet().get(i);
 739                     if (VpnUtil.areSubnetsOverlapping(associatedSubnet.getCidr(), subnetCidr)) {
 740                         return true;
 741                     }
 742                     if (routerTarget.getRtType() == RouteTarget.RtType.ERT) {
 743                         /* Check if there are multiple exports for the input iRT value (iRT in routeTargets)
 744                          *  Example : (1) iRT=A eRT=B subnet-range=S1; OK
 745                          *            (2) iRT=A eRT=B subnet-range=S1; OK
 746                          *            (3) iRT=B eRT=A subnet-range=S2; NOK
 747                          * Check if (1) and (2) are importing the same subnet-range routes to (3) */
 748                         List<AssociatedVpn> multipleAssociatedVpn = associatedSubnet.getAssociatedVpn();
 749                         if (multipleAssociatedVpn != null && multipleAssociatedVpn.size() > 1) {
 750                             LOG.error("doesExistingVpnsHaveConflictingSubnet: There is an indirect complete  overlap"
 751                                     + " for subnet CIDR {} for rt {} rtType {}", subnetCidr, routerTarget.getRt(),
 752                                     routerTarget.getRtType());
 753                             return true;
 754                         }
 755                         for (int j = i + 1; j < routerTarget.getAssociatedSubnet().size(); j++) {
 756                             if (VpnUtil.areSubnetsOverlapping(associatedSubnet.getCidr(),
 757                                     routerTarget.getAssociatedSubnet().get(j).getCidr())) {
 758                                 LOG.error("doesExistingVpnsHaveConflictingSubnet: There is an indirect paartial"
 759                                                 + " overlap for subnet CIDR {} for rt {} rtType {}", subnetCidr,
 760                                         routerTarget.getRt(), routerTarget.getRtType());
 761                                 return true;
 762                             }
 763                         }
 764                     }
 765                 }
 766                 /* Check if there are indirect EXPORTS for the eRT value (eRT in routeTargets)
 767                  *  Example : (1) iRT=A eRT=B subnet-range=S1; OK
 768                  *            (2) iRT=B eRT=A subnet-range=S2; OK
 769                  *            (3) iRT=A eRT=B subnet-range=S1; NOK
 770                  * If associatedSubnet is non-null for a routeTarget in (2),
 771                  * it may have already imported routes from (1) */
 772                 if (routerTarget.getRtType() == RouteTarget.RtType.IRT) {
 773                     try {
 774                         Optional<RouteTarget> indirectRts = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 775                                 LogicalDatastoreType.OPERATIONAL, VpnUtil.getRouteTargetsIdentifier(
 776                                         routerTarget.getRt(), RouteTarget.RtType.ERT));
 777                         if (indirectRts.isPresent() && indirectRts.get().getAssociatedSubnet() != null
 778                                 && routerTarget.getAssociatedSubnet() != null) {
 779                             for (AssociatedSubnet associatedSubnet : indirectRts.get().getAssociatedSubnet()) {
 780                                 if (VpnUtil.areSubnetsOverlapping(associatedSubnet.getCidr(), subnetCidr)) {
 781                                     LOG.error("doesExistingVpnsHaveConflictingSubnet: There is an indirect overlap for"
 782                                             + " subnet CIDR {} for rt {} rtType {}", subnetCidr, routerTarget.getRt(),
 783                                             routerTarget.getRtType());
 784                                     return true;
 785                                 }
 786                             }
 787                         }
 788                     } catch (ReadFailedException e) {
 789                         LOG.error("doesExistingVpnsHaveConflictingSubnet: Failed to read route targets to subnet"
 790                                 + "association for rt {} type {} subnet-cidr {}", routerTarget.getRt(),
 791                                 RouteTarget.RtType.ERT, subnetCidr);
 792                         return true; //Fail subnet association to avoid further damage to the data-stores
 793                     }
 794                 }
 795             } else {
 796                 LOG.info("doesExistingVpnsHaveConflictingSubnet: No prior associated subnets for rtVal {} rtType {}",
 797                         routerTarget.getRt(), routerTarget.getRtType());
 798             }
 799         }
 800         return false;
 801     }
 802
 803     private void addSubnetAssociationOperationToTx(String rt, RouteTarget.RtType rtType, String cidr,
 804                                                    String vpnName, ReadWriteTransaction tx,
 805                                                    boolean isAssociationRemoved)
 806             throws InterruptedException, ExecutionException {
 807         if (isAssociationRemoved) {
 808             //Remove RT-Subnet-Vpn Association
 809             Optional<AssociatedSubnet> associatedSubnet = tx.read(LogicalDatastoreType.OPERATIONAL,
 810                     VpnUtil.getAssociatedSubnetIdentifier(rt, rtType, cidr)).get();
 811             boolean deleteParent = false;
 812             if (associatedSubnet.isPresent()) {
 813                 List<AssociatedVpn> associatedVpns = associatedSubnet.get().getAssociatedVpn();
 814                 if (associatedVpns == null || associatedVpns.isEmpty()) {
 815                     deleteParent = true;
 816                 } else {
 817                     for (Iterator<AssociatedVpn> iterator = associatedVpns.iterator(); iterator.hasNext();) {
 818                         AssociatedVpn associatedVpn = iterator.next();
 819                         if (Objects.equals(associatedVpn.getName(), vpnName)) {
 820                             iterator.remove();
 821                             break;
 822                         }
 823                     }
 824                     if (associatedVpns.isEmpty()) {
 825                         deleteParent = true;
 826                     }
 827                 }
 828             }
 829             if (deleteParent) {
 830                 deleteParentForSubnetToVpnAssocication(rt, rtType, cidr, tx);
 831             } else {
 832                 //Some other VPNs are also part of this rtVal, rtType and subnetCidr combination.
 833                 //Delete only this AssociatedVpn Object
 834                 tx.delete(LogicalDatastoreType.OPERATIONAL,
 835                         VpnUtil.getAssociatedSubnetAndVpnIdentifier(rt, rtType, cidr, vpnName));
 836                 LOG.debug("addSubnetAssocOperationToTx: Removed vpn {} from association rt {} rtType {} cidr {}",
 837                         vpnName, rt, rtType, cidr);
 838             }
 839         } else {
 840             //Add RT-Subnet-Vpn Association
 841             tx.put(LogicalDatastoreType.OPERATIONAL,
 842                     VpnUtil.getAssociatedSubnetAndVpnIdentifier(rt, rtType, cidr, vpnName),
 843                     VpnUtil.buildAssociatedSubnetAndVpn(vpnName), true);
 844         }
 845     }
 846
 847     private void deleteParentForSubnetToVpnAssocication(String rt, RouteTarget.RtType rtType,
 848                                                 String cidr, ReadWriteTransaction tx)
 849             throws InterruptedException, ExecutionException {
 850         //Check if you need to delete rtVal+rtType or just the subnetCidr
 851         InstanceIdentifier<RouteTarget> rtIdentifier = InstanceIdentifier
 852                 .builder(SubnetsAssociatedToRouteTargets.class).child(RouteTarget.class,
 853                         new RouteTargetKey(rt, rtType)).build();
 854         Optional<RouteTarget> rtToSubnetsAssociation = tx.read(LogicalDatastoreType.OPERATIONAL,
 855                 rtIdentifier).get();
 856         if (rtToSubnetsAssociation.isPresent()) {
 857             List<AssociatedSubnet> associatedSubnets = rtToSubnetsAssociation.get().getAssociatedSubnet();
 858             if (associatedSubnets != null && !associatedSubnets.isEmpty()) {
 859                 for (Iterator<AssociatedSubnet> iterator = associatedSubnets.iterator(); iterator.hasNext(); ) {
 860                     if (Objects.equals(iterator.next().getCidr(), cidr)) {
 861                         iterator.remove();
 862                         break;
 863                     }
 864                 }
 865                 if (associatedSubnets.isEmpty()) {
 866                     //The entire rt to subnet association is empty
 867                     //Delete the RouteTarget object
 868                     tx.delete(LogicalDatastoreType.OPERATIONAL, rtIdentifier);
 869                     LOG.debug("deleteParentForSubnetToVpnAssocication: Removed rt {} rtType {} from association,",
 870                             rt, rtType);
 871                 } else {
 872                     //Some other VPNs are also part of this rtVal, rtType combination
 873                     //Delete only this AssociatedSubnet
 874                     tx.delete(LogicalDatastoreType.OPERATIONAL, VpnUtil.getAssociatedSubnetIdentifier(rt, rtType,
 875                             cidr));
 876                     LOG.debug("deleteParentForSubnetToVpnAssocication: Removed cidr {} from association rt {}"
 877                             + " rtType {}", cidr, rt, rtType);
 878                 }
 879             }
 880         }
 881     }
 882
 883     @Override
 884     public boolean checkForOverlappingSubnets(Uuid network, List<Subnetmap> subnetmapList, Uuid vpn,
 885                                        Set<VpnTarget> routeTargets, List<String> failedNwList) {
 886         for (int i = 0; i < subnetmapList.size(); i++) {
 887             //Check if any other subnet that is already part of a different vpn with same rt, has overlapping CIDR
 888             if (checkExistingSubnetWithSameRoutTargets(routeTargets, vpn, subnetmapList.get(i), failedNwList)) {
 889                 return true;
 890             }
 891         }
 892         return false;
 893     }
 894
 895     private boolean checkExistingSubnetWithSameRoutTargets(Set<VpnTarget> routeTargets, Uuid vpn, Subnetmap subnetmap,
 896                                                    List<String> failedNwList) {
 897         String cidr = String.valueOf(subnetmap.getSubnetIp());
 898         boolean subnetExistsWithSameRt = doesExistingVpnsHaveConflictingSubnet(routeTargets, cidr);
 899         if (subnetExistsWithSameRt) {
 900             failedNwList.add(String.format("Another subnet with the same/overlapping cidr %s as subnet %s"
 901                     + " is already associated to a vpn with routeTargets %s. Ignoring subnet addition to vpn"
 902                     + " %s", cidr, subnetmap.getId().getValue(), routeTargets.toString(), vpn.getValue()));
 903         }
 904         return subnetExistsWithSameRt;
 905     }
 906 }