vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnManagerImpl.java

   1 /*
   2  * Copyright (c) 2015 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import java.math.BigInteger;
  12 import java.util.ArrayList;
  13 import java.util.Collection;
  14 import java.util.Collections;
  15 import java.util.Iterator;
  16 import java.util.List;
  17 import java.util.Set;
  18 import java.util.concurrent.ExecutionException;
  19 import java.util.concurrent.Future;
  20 import javax.annotation.PostConstruct;
  21 import javax.inject.Inject;
  22 import javax.inject.Singleton;
  23 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  24 import org.opendaylight.controller.md.sal.binding.api.ReadWriteTransaction;
  25 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  26 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  27 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  28 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  29 import org.opendaylight.genius.datastoreutils.listeners.DataTreeEventCallbackRegistrar;
  30 import org.opendaylight.genius.infra.Datastore.Configuration;
  31 import org.opendaylight.genius.infra.Datastore.Operational;
  32 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  33 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  34 import org.opendaylight.genius.infra.TransactionAdapter;
  35 import org.opendaylight.genius.infra.TypedReadTransaction;
  36 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  37 import org.opendaylight.genius.infra.TypedWriteTransaction;
  38 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  39 import org.opendaylight.genius.mdsalutil.FlowEntity;
  40 import org.opendaylight.genius.mdsalutil.MetaDataUtil;
  41 import org.opendaylight.genius.mdsalutil.NwConstants;
  42 import org.opendaylight.genius.mdsalutil.UpgradeState;
  43 import org.opendaylight.genius.mdsalutil.instructions.InstructionWriteMetadata;
  44 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  45 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  46 import org.opendaylight.infrautils.utils.function.InterruptibleCheckedConsumer;
  47 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  48 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput;
  49 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderInput.ArpReponderInputBuilder;
  50 import org.opendaylight.netvirt.elan.arp.responder.ArpResponderUtil;
  51 import org.opendaylight.netvirt.elanmanager.api.IElanService;
  52 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  53 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  54 import org.opendaylight.netvirt.neutronvpn.api.enums.IpVersionChoice;
  55 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  56 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  57 import org.opendaylight.netvirt.vpnmanager.api.VpnExtraRouteHelper;
  58 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.IVpnLinkService;
  59 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkCache;
  60 import org.opendaylight.netvirt.vpnmanager.api.intervpnlink.InterVpnLinkDataComposite;
  61 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  62 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  63 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInstances;
  64 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.af.config.vpntargets.VpnTarget;
  65 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.instances.VpnInstance;
  66 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  67 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.flow.types.rev131026.instruction.list.Instruction;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInput;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolInputBuilder;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.CreateIdPoolOutput;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.rpcs.rev160406.ItmRpcService;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.elan.rev150602.elan.dpn.interfaces.elan.dpn.interfaces.list.DpnInterfaces;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.SubnetsAssociatedToRouteTargets;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.RouteTarget;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.RouteTargetKey;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.route.target.AssociatedSubnet;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.subnets.associated.to.route.targets.route.target.associated.subnet.AssociatedVpn;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.to.extraroutes.vpn.extra.routes.Routes;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
  86 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
  87 import org.opendaylight.yangtools.yang.common.RpcError;
  88 import org.opendaylight.yangtools.yang.common.RpcResult;
  89 import org.slf4j.Logger;
  90 import org.slf4j.LoggerFactory;
  91
  92 @Singleton
  93 public class VpnManagerImpl implements IVpnManager {
  94
  95     private static final Logger LOG = LoggerFactory.getLogger(VpnManagerImpl.class);
  96     private final DataBroker dataBroker;
  97     private final ManagedNewTransactionRunner txRunner;
  98     private final IdManagerService idManager;
  99     private final IMdsalApiManager mdsalManager;
 100     private final IElanService elanService;
 101     private final IInterfaceManager interfaceManager;
 102     private final VpnSubnetRouteHandler vpnSubnetRouteHandler;
 103     private final OdlInterfaceRpcService ifaceMgrRpcService;
 104     private final IVpnLinkService ivpnLinkService;
 105     private final IFibManager fibManager;
 106     private final IBgpManager bgpManager;
 107     private final InterVpnLinkCache interVpnLinkCache;
 108     private final DataTreeEventCallbackRegistrar eventCallbacks;
 109     private final UpgradeState upgradeState;
 110     private final ItmRpcService itmRpcService;
 111     private final VpnUtil vpnUtil;
 112
 113     @Inject
 114     public VpnManagerImpl(final DataBroker dataBroker,
 115                           final IdManagerService idManagerService,
 116                           final IMdsalApiManager mdsalManager,
 117                           final IElanService elanService,
 118                           final IInterfaceManager interfaceManager,
 119                           final VpnSubnetRouteHandler vpnSubnetRouteHandler,
 120                           final OdlInterfaceRpcService ifaceMgrRpcService,
 121                           final IVpnLinkService ivpnLinkService,
 122                           final IFibManager fibManager,
 123                           final IBgpManager bgpManager,
 124                           final InterVpnLinkCache interVpnLinkCache,
 125                           final DataTreeEventCallbackRegistrar dataTreeEventCallbackRegistrar,
 126                           final UpgradeState upgradeState,
 127                           final ItmRpcService itmRpcService,
 128                           final VpnUtil vpnUtil) {
 129         this.dataBroker = dataBroker;
 130         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 131         this.idManager = idManagerService;
 132         this.mdsalManager = mdsalManager;
 133         this.elanService = elanService;
 134         this.interfaceManager = interfaceManager;
 135         this.vpnSubnetRouteHandler = vpnSubnetRouteHandler;
 136         this.ifaceMgrRpcService = ifaceMgrRpcService;
 137         this.ivpnLinkService = ivpnLinkService;
 138         this.fibManager = fibManager;
 139         this.bgpManager = bgpManager;
 140         this.interVpnLinkCache = interVpnLinkCache;
 141         this.eventCallbacks = dataTreeEventCallbackRegistrar;
 142         this.upgradeState = upgradeState;
 143         this.itmRpcService = itmRpcService;
 144         this.vpnUtil = vpnUtil;
 145     }
 146
 147     @PostConstruct
 148     public void start() {
 149         LOG.info("{} start", getClass().getSimpleName());
 150         createIdPool();
 151     }
 152
 153     private void createIdPool() {
 154         CreateIdPoolInput createPool = new CreateIdPoolInputBuilder()
 155             .setPoolName(VpnConstants.VPN_IDPOOL_NAME)
 156             .setLow(VpnConstants.VPN_IDPOOL_LOW)
 157             .setHigh(VpnConstants.VPN_IDPOOL_HIGH)
 158             .build();
 159         try {
 160             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPool);
 161             if (result != null && result.get().isSuccessful()) {
 162                 LOG.info("Created IdPool for VPN Service");
 163             }
 164         } catch (InterruptedException | ExecutionException e) {
 165             LOG.error("Failed to create idPool for VPN Service", e);
 166         }
 167
 168         // Now an IdPool for InterVpnLink endpoint's pseudo ports
 169         CreateIdPoolInput createPseudoLporTagPool =
 170             new CreateIdPoolInputBuilder().setPoolName(VpnConstants.PSEUDO_LPORT_TAG_ID_POOL_NAME)
 171                 .setLow(VpnConstants.LOWER_PSEUDO_LPORT_TAG)
 172                 .setHigh(VpnConstants.UPPER_PSEUDO_LPORT_TAG)
 173                 .build();
 174         try {
 175             Future<RpcResult<CreateIdPoolOutput>> result = idManager.createIdPool(createPseudoLporTagPool);
 176             if (result.get().isSuccessful()) {
 177                 LOG.debug("Created IdPool for Pseudo Port tags");
 178             } else {
 179                 Collection<RpcError> errors = result.get().getErrors();
 180                 StringBuilder errMsg = new StringBuilder();
 181                 for (RpcError err : errors) {
 182                     errMsg.append(err.getMessage()).append("\n");
 183                 }
 184                 LOG.error("IdPool creation for PseudoPort tags failed. Reasons: {}", errMsg);
 185             }
 186         } catch (InterruptedException | ExecutionException e) {
 187             LOG.error("Failed to create idPool for Pseudo Port tags", e);
 188         }
 189     }
 190
 191     @Override
 192     public void addExtraRoute(String vpnName, String destination, String nextHop, String rd, String routerID,
 193         Long l3vni, RouteOrigin origin, String intfName, Adjacency operationalAdj, VrfEntry.EncapType encapType,
 194         TypedWriteTransaction<Configuration> confTx) {
 195         addExtraRoute(vpnName, destination, nextHop, rd, routerID, l3vni, origin, intfName, operationalAdj, encapType,
 196             TransactionAdapter.toWriteTransaction(confTx));
 197     }
 198
 199     @Override
 200     public void addExtraRoute(String vpnName, String destination, String nextHop, String rd, String routerID,
 201             Long l3vni, RouteOrigin origin, String intfName, Adjacency operationalAdj,
 202             VrfEntry.EncapType encapType, WriteTransaction writeConfigTxn) {
 203         if (writeConfigTxn == null) {
 204             String finalNextHop = nextHop;
 205             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx ->
 206                 addExtraRoute(vpnName, destination, finalNextHop, rd, routerID, l3vni, origin, intfName, operationalAdj,
 207                         encapType, tx)),
 208                     LOG, "Error adding extra route");
 209             return;
 210         }
 211
 212         //add extra route to vpn mapping; advertise with nexthop as tunnel ip
 213         vpnUtil.syncUpdate(LogicalDatastoreType.OPERATIONAL,
 214                 VpnExtraRouteHelper.getVpnToExtrarouteVrfIdIdentifier(vpnName, rd != null ? rd : routerID,
 215                         destination),
 216                 VpnUtil.getVpnToExtraroute(destination, Collections.singletonList(nextHop)));
 217
 218         BigInteger dpnId = null;
 219         if (intfName != null && !intfName.isEmpty()) {
 220             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
 221             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 222             if (nextHopIp == null || nextHopIp.isEmpty()) {
 223                 LOG.error("addExtraRoute: NextHop for interface {} is null / empty."
 224                         + " Failed advertising extra route for rd {} prefix {} dpn {}", intfName, rd, destination,
 225                         dpnId);
 226                 return;
 227             }
 228             nextHop = nextHopIp;
 229         }
 230
 231         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 232
 233         // TODO: This is a limitation to be stated in docs. When configuring static route to go to
 234         // another VPN, there can only be one nexthop or, at least, the nexthop to the interVpnLink should be in
 235         // first place.
 236         Optional<InterVpnLinkDataComposite> optVpnLink = interVpnLinkCache.getInterVpnLinkByEndpoint(nextHop);
 237         if (optVpnLink.isPresent() && optVpnLink.get().isActive()) {
 238             InterVpnLinkDataComposite interVpnLink = optVpnLink.get();
 239             // If the nexthop is the endpoint of Vpn2, then prefix must be advertised to Vpn1 in DC-GW, with nexthops
 240             // pointing to the DPNs where Vpn1 is instantiated. LFIB in these DPNS must have a flow entry, with lower
 241             // priority, where if Label matches then sets the lportTag of the Vpn2 endpoint and goes to LportDispatcher
 242             // This is like leaking one of the Vpn2 routes towards Vpn1
 243             String srcVpnUuid = interVpnLink.getVpnNameByIpAddress(nextHop);
 244             String dstVpnUuid = interVpnLink.getOtherVpnNameByIpAddress(nextHop);
 245             String dstVpnRd = vpnUtil.getVpnRd(dstVpnUuid);
 246             long newLabel = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
 247                     VpnUtil.getNextHopLabelKey(dstVpnRd, destination));
 248             if (newLabel == 0) {
 249                 LOG.error("addExtraRoute: Unable to fetch label from Id Manager. Bailing out of adding intervpnlink"
 250                         + " route for destination {}", destination);
 251                 return;
 252             }
 253             ivpnLinkService.leakRoute(interVpnLink, srcVpnUuid, dstVpnUuid, destination, newLabel, RouteOrigin.STATIC);
 254         } else {
 255             Optional<Routes> optVpnExtraRoutes = VpnExtraRouteHelper
 256                     .getVpnExtraroutes(dataBroker, vpnName, rd != null ? rd : routerID, destination);
 257             if (optVpnExtraRoutes.isPresent()) {
 258                 List<String> nhList = optVpnExtraRoutes.get().getNexthopIpList();
 259                 if (nhList != null && nhList.size() > 1) {
 260                     // If nhList is greater than one for vpnextraroute, a call to populatefib doesn't update vrfentry.
 261                     fibManager.refreshVrfEntry(primaryRd, destination);
 262                 } else {
 263                     L3vpnInput input = new L3vpnInput().setNextHop(operationalAdj).setNextHopIp(nextHop).setL3vni(l3vni)
 264                             .setPrimaryRd(primaryRd).setVpnName(vpnName).setDpnId(dpnId)
 265                             .setEncapType(encapType).setRd(rd).setRouteOrigin(origin);
 266                     L3vpnRegistry.getRegisteredPopulator(encapType).populateFib(input, writeConfigTxn);
 267                 }
 268             }
 269         }
 270     }
 271
 272     @Override
 273     public void delExtraRoute(String vpnName, String destination, String nextHop, String rd, String routerID,
 274         String intfName, TypedWriteTransaction<Configuration> confTx, TypedWriteTransaction<Operational> operTx) {
 275         delExtraRoute(vpnName, destination, nextHop, rd, routerID, intfName,
 276             TransactionAdapter.toWriteTransaction(confTx), TransactionAdapter.toWriteTransaction(operTx));
 277     }
 278
 279     @Override
 280     public void delExtraRoute(String vpnName, String destination, String nextHop, String rd, String routerID,
 281             String intfName, WriteTransaction writeConfigTxn, WriteTransaction writeOperTx) {
 282         if (writeConfigTxn == null) {
 283             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 284                 delExtraRoute(vpnName, destination, nextHop, rd, routerID, intfName, tx, writeOperTx);
 285             }), LOG, "Error deleting extra route");
 286             return;
 287         }
 288         if (writeOperTx == null) {
 289             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
 290                 delExtraRoute(vpnName, destination, nextHop, rd, routerID, intfName, writeConfigTxn, tx);
 291             }), LOG, "Error deleting extra route");
 292             return;
 293         }
 294         BigInteger dpnId = null;
 295         String tunnelIp = nextHop;
 296         if (intfName != null && !intfName.isEmpty()) {
 297             dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, intfName);
 298             String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 299             if (nextHopIp == null || nextHopIp.isEmpty()) {
 300                 LOG.error("delExtraRoute: NextHop for interface {} is null / empty."
 301                         + " Failed advertising extra route for rd {} prefix {} dpn {}", intfName, rd, destination,
 302                         dpnId);
 303             }
 304             tunnelIp = nextHopIp;
 305         }
 306         if (rd != null) {
 307             String primaryRd = vpnUtil.getVpnRd(vpnName);
 308             removePrefixFromBGP(vpnName, primaryRd, rd, intfName, destination, nextHop, tunnelIp, dpnId,
 309                     writeConfigTxn, writeOperTx);
 310             LOG.info("delExtraRoute: Removed extra route {} from interface {} for rd {}", destination, intfName, rd);
 311         } else {
 312             // add FIB route directly
 313             fibManager.removeOrUpdateFibEntry(routerID, destination, tunnelIp, writeConfigTxn);
 314             LOG.info("delExtraRoute: Removed extra route {} from interface {} for rd {}", destination, intfName,
 315                     routerID);
 316         }
 317     }
 318
 319     @Override
 320     public void removePrefixFromBGP(String vpnName, String primaryRd, String extraRouteRd, String vpnInterfaceName,
 321                                     String prefix, String nextHop, String nextHopTunnelIp, BigInteger dpnId,
 322                                     TypedWriteTransaction<Configuration> confTx,
 323                                     TypedWriteTransaction<Operational> operTx) {
 324         removePrefixFromBGP(vpnName, primaryRd, extraRouteRd, vpnInterfaceName, prefix, nextHop, nextHopTunnelIp,
 325                 dpnId, TransactionAdapter.toWriteTransaction(confTx), TransactionAdapter.toWriteTransaction(operTx));
 326     }
 327
 328     // TODO Clean up the exception handling
 329     @Override
 330     @SuppressWarnings("checkstyle:IllegalCatch")
 331     public void removePrefixFromBGP(String vpnName, String primaryRd, String extraRouteRd, String vpnInterfaceName,
 332         String prefix, String nextHop, String nextHopTunnelIp, BigInteger dpnId, WriteTransaction writeConfigTxn,
 333         WriteTransaction writeOperTx) {
 334         try {
 335             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 336             synchronized (vpnNamePrefixKey.intern()) {
 337                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, extraRouteRd, vpnInterfaceName, prefix,
 338                         nextHop, nextHopTunnelIp, writeOperTx)) {
 339                     return;
 340                 }
 341                 fibManager.removeOrUpdateFibEntry(primaryRd, prefix, nextHopTunnelIp, writeConfigTxn);
 342                 if (VpnUtil.isEligibleForBgp(extraRouteRd, vpnName, dpnId, null /*networkName*/)) {
 343                     // TODO: Might be needed to include nextHop here
 344                     bgpManager.withdrawPrefix(extraRouteRd, prefix);
 345                 }
 346             }
 347             LOG.info("removePrefixFromBGP: VPN WITHDRAW: Removed Fib Entry rd {} prefix {} nexthop {}", extraRouteRd,
 348                     prefix, nextHop);
 349         } catch (RuntimeException e) {
 350             LOG.error("removePrefixFromBGP: Delete prefix {} rd {} nextHop {} failed", prefix, extraRouteRd, nextHop);
 351         }
 352     }
 353
 354     @Override
 355     public boolean isVPNConfigured() {
 356         InstanceIdentifier<VpnInstances> vpnsIdentifier = InstanceIdentifier.builder(VpnInstances.class).build();
 357         try {
 358             Optional<VpnInstances> optionalVpns =
 359                     SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
 360                             vpnsIdentifier);
 361             if (!optionalVpns.isPresent()
 362                     || optionalVpns.get().getVpnInstance() == null
 363                     || optionalVpns.get().getVpnInstance().isEmpty()) {
 364                 LOG.trace("isVPNConfigured: No VPNs configured.");
 365                 return false;
 366             }
 367         } catch (ReadFailedException e) {
 368             throw new RuntimeException("Error reading VPN " + vpnsIdentifier, e);
 369         }
 370         LOG.trace("isVPNConfigured: VPNs are configured on the system.");
 371         return true;
 372     }
 373
 374     @Override
 375     public void addSubnetMacIntoVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 376             BigInteger dpnId, TypedWriteTransaction<Configuration> confTx)
 377             throws ExecutionException, InterruptedException {
 378         setupSubnetMacInVpnInstance(vpnName, subnetVpnName, srcMacAddress, dpnId,
 379             (vpnId, dpId, subnetVpnId) -> addGwMac(srcMacAddress, confTx, vpnId, dpId, subnetVpnId));
 380     }
 381
 382     @Override
 383     public void addSubnetMacIntoVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 384             BigInteger dpnId, WriteTransaction tx) throws ExecutionException, InterruptedException {
 385         setupSubnetMacInVpnInstance(vpnName, subnetVpnName, srcMacAddress, dpnId,
 386             (vpnId, dpId, subnetVpnId) -> addGwMac(srcMacAddress, tx, vpnId, dpId, subnetVpnId));
 387     }
 388
 389     @Override
 390     public void removeSubnetMacFromVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 391             BigInteger dpnId, TypedReadWriteTransaction<Configuration> confTx)
 392             throws ExecutionException, InterruptedException {
 393         setupSubnetMacInVpnInstance(vpnName, subnetVpnName, srcMacAddress, dpnId,
 394             (vpnId, dpId, subnetVpnId) -> removeGwMac(srcMacAddress, confTx, vpnId, dpId, subnetVpnId));
 395     }
 396
 397     @Override
 398     public void removeSubnetMacFromVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 399             BigInteger dpnId, WriteTransaction tx) throws ExecutionException, InterruptedException {
 400         setupSubnetMacInVpnInstance(vpnName, subnetVpnName, srcMacAddress, dpnId,
 401             (vpnId, dpId, subnetVpnId) -> removeGwMac(srcMacAddress, tx, vpnId, dpId, subnetVpnId));
 402     }
 403
 404     @FunctionalInterface
 405     private interface VpnInstanceSubnetMacSetupMethod {
 406         void process(long vpnId, BigInteger dpId, long subnetVpnId) throws InterruptedException, ExecutionException;
 407     }
 408
 409     private void setupSubnetMacInVpnInstance(String vpnName, String subnetVpnName, String srcMacAddress,
 410             BigInteger dpnId, VpnInstanceSubnetMacSetupMethod consumer)
 411             throws ExecutionException, InterruptedException {
 412         if (vpnName == null) {
 413             LOG.warn("Cannot setup subnet MAC {} on DPN {}, null vpnName", srcMacAddress, dpnId);
 414             return;
 415         }
 416
 417         long vpnId = vpnUtil.getVpnId(vpnName);
 418         long subnetVpnId = vpnUtil.getVpnId(subnetVpnName);
 419         if (dpnId.equals(BigInteger.ZERO)) {
 420             /* Apply the MAC on all DPNs in a VPN */
 421             for (BigInteger dpId : vpnUtil.getDpnsOnVpn(vpnName)) {
 422                 consumer.process(vpnId, dpId, subnetVpnId);
 423             }
 424         } else {
 425             consumer.process(vpnId, dpnId, subnetVpnId);
 426         }
 427     }
 428
 429     private void addGwMac(String srcMacAddress, WriteTransaction tx, long vpnId, BigInteger dpId, long subnetVpnId) {
 430         FlowEntity flowEntity = vpnUtil.buildL3vpnGatewayFlow(dpId, srcMacAddress, vpnId, subnetVpnId);
 431         mdsalManager.addFlowToTx(flowEntity, tx);
 432     }
 433
 434     private void addGwMac(String srcMacAddress, TypedWriteTransaction<Configuration> tx, long vpnId, BigInteger dpId,
 435         long subnetVpnId) {
 436         FlowEntity flowEntity = vpnUtil.buildL3vpnGatewayFlow(dpId, srcMacAddress, vpnId, subnetVpnId);
 437         mdsalManager.addFlow(tx, flowEntity);
 438     }
 439
 440     private void removeGwMac(String srcMacAddress, WriteTransaction tx, long vpnId, BigInteger dpId, long subnetVpnId) {
 441         FlowEntity flowEntity = vpnUtil.buildL3vpnGatewayFlow(dpId, srcMacAddress, vpnId, subnetVpnId);
 442         mdsalManager.removeFlowToTx(flowEntity, tx);
 443     }
 444
 445     private void removeGwMac(String srcMacAddress, TypedReadWriteTransaction<Configuration> tx, long vpnId,
 446             BigInteger dpId, long subnetVpnId) throws ExecutionException, InterruptedException {
 447         mdsalManager.removeFlow(tx, dpId,
 448             VpnUtil.getL3VpnGatewayFlowRef(NwConstants.L3_GW_MAC_TABLE, dpId, vpnId, srcMacAddress, subnetVpnId),
 449             NwConstants.L3_GW_MAC_TABLE);
 450     }
 451
 452     @Override
 453     public void addRouterGwMacFlow(String routerName, String routerGwMac, BigInteger dpnId, Uuid extNetworkId,
 454             String subnetVpnName, TypedWriteTransaction<Configuration> confTx)
 455             throws ExecutionException, InterruptedException {
 456         setupRouterGwMacFlow(routerName, routerGwMac, dpnId, extNetworkId,
 457             vpnId -> addSubnetMacIntoVpnInstance(vpnId, subnetVpnName, routerGwMac, dpnId, confTx), "Installing");
 458     }
 459
 460     @Override
 461     public void addRouterGwMacFlow(String routerName, String routerGwMac, BigInteger dpnId, Uuid extNetworkId,
 462             String subnetVpnName, WriteTransaction writeTx) throws ExecutionException, InterruptedException {
 463         setupRouterGwMacFlow(routerName, routerGwMac, dpnId, extNetworkId, writeTx,
 464             (vpnId, tx) -> addSubnetMacIntoVpnInstance(vpnId, subnetVpnName, routerGwMac, dpnId, tx), "Installing");
 465     }
 466
 467     @Override
 468     public void removeRouterGwMacFlow(String routerName, String routerGwMac, BigInteger dpnId, Uuid extNetworkId,
 469             String subnetVpnName, TypedReadWriteTransaction<Configuration> confTx)
 470             throws ExecutionException, InterruptedException {
 471         setupRouterGwMacFlow(routerName, routerGwMac, dpnId, extNetworkId,
 472             vpnId -> removeSubnetMacFromVpnInstance(vpnId, subnetVpnName, routerGwMac, dpnId, confTx), "Removing");
 473     }
 474
 475     @Override
 476     public void removeRouterGwMacFlow(String routerName, String routerGwMac, BigInteger dpnId, Uuid extNetworkId,
 477             String subnetVpnName, WriteTransaction writeTx) throws ExecutionException, InterruptedException {
 478         setupRouterGwMacFlow(routerName, routerGwMac, dpnId, extNetworkId, writeTx,
 479             (vpnId, tx) -> removeSubnetMacFromVpnInstance(vpnId, subnetVpnName, routerGwMac, dpnId, tx), "Removing");
 480     }
 481
 482     @FunctionalInterface
 483     private interface RouterGwMacFlowSetupMethod {
 484         void process(String vpnId, WriteTransaction tx) throws InterruptedException, ExecutionException;
 485     }
 486
 487     private void setupRouterGwMacFlow(String routerName, String routerGwMac, BigInteger dpnId, Uuid extNetworkId,
 488             WriteTransaction writeTx, RouterGwMacFlowSetupMethod consumer, String operation)
 489             throws ExecutionException, InterruptedException {
 490         if (routerGwMac == null) {
 491             LOG.warn("Failed to handle router GW flow in GW-MAC table. MAC address is missing for router-id {}",
 492                     routerName);
 493             return;
 494         }
 495
 496         if (dpnId == null || BigInteger.ZERO.equals(dpnId)) {
 497             LOG.info("setupRouterGwMacFlow: DPN id is missing for router-id {}",
 498                     routerName);
 499             return;
 500         }
 501
 502         Uuid vpnId = vpnUtil.getExternalNetworkVpnId(extNetworkId);
 503         if (vpnId == null) {
 504             LOG.warn("Network {} is not associated with VPN", extNetworkId.getValue());
 505             return;
 506         }
 507
 508         LOG.info("{} router GW MAC flow for router-id {} on switch {}", operation, routerName, dpnId);
 509         if (writeTx == null) {
 510             ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 511                 tx -> consumer.process(vpnId.getValue(), tx)), LOG, "Commit transaction");
 512         } else {
 513             consumer.process(vpnId.getValue(), writeTx);
 514         }
 515     }
 516
 517     private void setupRouterGwMacFlow(String routerName, String routerGwMac, BigInteger dpnId, Uuid extNetworkId,
 518             InterruptibleCheckedConsumer<String, ExecutionException> consumer, String operation)
 519             throws ExecutionException, InterruptedException {
 520         if (routerGwMac == null) {
 521             LOG.warn("Failed to handle router GW flow in GW-MAC table. MAC address is missing for router-id {}",
 522                 routerName);
 523             return;
 524         }
 525
 526         if (dpnId == null || BigInteger.ZERO.equals(dpnId)) {
 527             LOG.info("setupRouterGwMacFlow: DPN id is missing for router-id {}",
 528                 routerName);
 529             return;
 530         }
 531
 532         Uuid vpnId = vpnUtil.getExternalNetworkVpnId(extNetworkId);
 533         if (vpnId == null) {
 534             LOG.warn("Network {} is not associated with VPN", extNetworkId.getValue());
 535             return;
 536         }
 537
 538         LOG.info("{} router GW MAC flow for router-id {} on switch {}", operation, routerName, dpnId);
 539         consumer.accept(vpnId.getValue());
 540     }
 541
 542     @Override
 543     public void addArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 544         BigInteger dpnId, Uuid extNetworkId, WriteTransaction writeTx) {
 545         addArpResponderFlowsToExternalNetworkIps(id, fixedIps, macAddress, dpnId, extNetworkId);
 546     }
 547
 548     public void addArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 549             BigInteger dpnId, Uuid extNetworkId) {
 550
 551         if (dpnId == null || BigInteger.ZERO.equals(dpnId)) {
 552             LOG.warn("Failed to install arp responder flows for router {}. DPN id is missing.", id);
 553             return;
 554         }
 555
 556         String extInterfaceName = elanService.getExternalElanInterface(extNetworkId.getValue(), dpnId);
 557         if (extInterfaceName != null) {
 558             doAddArpResponderFlowsToExternalNetworkIps(
 559                     id, fixedIps, macAddress, dpnId, extInterfaceName);
 560             return;
 561         }
 562
 563         LOG.warn("Failed to install responder flows for {}. No external interface found for DPN id {}", id, dpnId);
 564
 565         if (!upgradeState.isUpgradeInProgress()) {
 566             return;
 567         }
 568
 569         // The following through the end of the function deals with an upgrade scenario where the neutron configuration
 570         // is restored before the OVS switches reconnect. In such a case, the elan-dpn-interfaces entries will be
 571         // missing from the operational data store. In order to mitigate this we use DataTreeEventCallbackRegistrar
 572         // to wait for the exact operational md-sal object we need to contain the external interface we need.
 573
 574         LOG.info("Upgrade in process, waiting for an external interface to appear on dpn {} for elan {}",
 575                 dpnId, extNetworkId.getValue());
 576
 577         InstanceIdentifier<DpnInterfaces> dpnInterfacesIid =
 578                             elanService.getElanDpnInterfaceOperationalDataPath(extNetworkId.getValue(), dpnId);
 579
 580         eventCallbacks.onAddOrUpdate(LogicalDatastoreType.OPERATIONAL, dpnInterfacesIid, (unused, alsoUnused) -> {
 581             LOG.info("Reattempting write of arp responder for external interfaces for external network {}",
 582                     extNetworkId);
 583             DpnInterfaces dpnInterfaces = elanService.getElanInterfaceInfoByElanDpn(extNetworkId.getValue(), dpnId);
 584             if (dpnInterfaces == null) {
 585                 LOG.error("Could not retrieve DpnInterfaces for {}, {}", extNetworkId.getValue(), dpnId);
 586                 return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 587             }
 588
 589             String extIfc = null;
 590             for (String dpnInterface : dpnInterfaces.getInterfaces()) {
 591                 if (interfaceManager.isExternalInterface(dpnInterface)) {
 592                     extIfc = dpnInterface;
 593                     break;
 594                 }
 595             }
 596
 597             if (extIfc == null) {
 598                 if (upgradeState.isUpgradeInProgress()) {
 599                     LOG.info("External interface not found yet in elan {} on dpn {}, keep waiting",
 600                             extNetworkId.getValue(), dpnInterfaces);
 601                     return DataTreeEventCallbackRegistrar.NextAction.CALL_AGAIN;
 602                 } else {
 603                     return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 604                 }
 605             }
 606
 607             final String extIfcFinal = extIfc;
 608             doAddArpResponderFlowsToExternalNetworkIps(id, fixedIps, macAddress, dpnId, extIfcFinal);
 609
 610             return DataTreeEventCallbackRegistrar.NextAction.UNREGISTER;
 611         });
 612
 613     }
 614
 615     @Override
 616     public void addArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 617             BigInteger dpnId, String extInterfaceName, int lportTag) {
 618         if (fixedIps == null || fixedIps.isEmpty()) {
 619             LOG.debug("No external IPs defined for {}", id);
 620             return;
 621         }
 622
 623         LOG.info("Installing ARP responder flows for {} fixed-ips {} on switch {}", id, fixedIps, dpnId);
 624
 625         for (String fixedIp : fixedIps) {
 626             IpVersionChoice ipVersionChoice = VpnUtil.getIpVersionFromString(fixedIp);
 627             if (ipVersionChoice == IpVersionChoice.IPV6) {
 628                 continue;
 629             }
 630             installArpResponderFlowsToExternalNetworkIp(macAddress, dpnId, extInterfaceName, lportTag,
 631                     fixedIp);
 632         }
 633     }
 634
 635     private void doAddArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 636             BigInteger dpnId, String extInterfaceName) {
 637         Interface extInterfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, extInterfaceName);
 638         if (extInterfaceState == null) {
 639             LOG.debug("No interface state found for interface {}. Delaying responder flows for {}", extInterfaceName,
 640                     id);
 641             return;
 642         }
 643
 644         Integer lportTag = extInterfaceState.getIfIndex();
 645         if (lportTag == null) {
 646             LOG.debug("No Lport tag found for interface {}. Delaying flows for router-id {}", extInterfaceName, id);
 647             return;
 648         }
 649
 650         if (macAddress == null) {
 651
 652             LOG.debug("Failed to install arp responder flows for router-gateway-ip {} for router {}."
 653                     + "External Gw MacAddress is missing.", fixedIps,  id);
 654             return;
 655         }
 656
 657         addArpResponderFlowsToExternalNetworkIps(id, fixedIps, macAddress, dpnId, extInterfaceName, lportTag);
 658     }
 659
 660     @Override
 661     public void removeArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps, String macAddress,
 662             BigInteger dpnId, Uuid extNetworkId) {
 663
 664         if (dpnId == null || BigInteger.ZERO.equals(dpnId)) {
 665             LOG.warn("Failed to remove arp responder flows for router {}. DPN id is missing.", id);
 666             return;
 667         }
 668
 669         String extInterfaceName = elanService.getExternalElanInterface(extNetworkId.getValue(), dpnId);
 670         if (extInterfaceName == null) {
 671             LOG.warn("Failed to remove responder flows for {}. No external interface found for DPN id {}", id, dpnId);
 672             return;
 673         }
 674
 675         Interface extInterfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, extInterfaceName);
 676         if (extInterfaceState == null) {
 677             LOG.debug("No interface state found for interface {}. Delaying responder flows for {}", extInterfaceName,
 678                     id);
 679             return;
 680         }
 681
 682         Integer lportTag = extInterfaceState.getIfIndex();
 683         if (lportTag == null) {
 684             LOG.debug("No Lport tag found for interface {}. Delaying flows for router-id {}", extInterfaceName, id);
 685             return;
 686         }
 687
 688         if (macAddress == null) {
 689
 690             LOG.debug("Failed to remove arp responder flows for router-gateway-ip {} for router {}."
 691                     + "External Gw MacAddress is missing.", fixedIps,  id);
 692             return;
 693         }
 694
 695         removeArpResponderFlowsToExternalNetworkIps(id, fixedIps, dpnId,
 696                 extInterfaceName, lportTag);
 697     }
 698
 699     @Override
 700     public void removeArpResponderFlowsToExternalNetworkIps(String id, Collection<String> fixedIps,
 701             BigInteger dpnId, String extInterfaceName, int lportTag) {
 702         if (fixedIps == null || fixedIps.isEmpty()) {
 703             LOG.debug("No external IPs defined for {}", id);
 704             return;
 705         }
 706
 707         LOG.info("Removing ARP responder flows for {} fixed-ips {} on switch {}", id, fixedIps, dpnId);
 708
 709         for (String fixedIp : fixedIps) {
 710             removeArpResponderFlowsToExternalNetworkIp(dpnId, lportTag, fixedIp, extInterfaceName);
 711         }
 712     }
 713
 714     @Override
 715     public String getPrimaryRdFromVpnInstance(VpnInstance vpnInstance) {
 716         return VpnUtil.getPrimaryRd(vpnInstance);
 717     }
 718
 719     private void installArpResponderFlowsToExternalNetworkIp(String macAddress, BigInteger dpnId,
 720             String extInterfaceName, int lportTag, String fixedIp) {
 721         // reset the split-horizon bit to allow traffic to be sent back to the
 722         // provider port
 723         List<Instruction> instructions = new ArrayList<>();
 724         instructions.add(
 725                 new InstructionWriteMetadata(BigInteger.ZERO, MetaDataUtil.METADATA_MASK_SH_FLAG).buildInstruction(1));
 726         instructions.addAll(
 727                 ArpResponderUtil.getExtInterfaceInstructions(interfaceManager, itmRpcService, extInterfaceName,
 728                         fixedIp, macAddress));
 729         ArpReponderInputBuilder builder = new ArpReponderInputBuilder().setDpId(dpnId)
 730                 .setInterfaceName(extInterfaceName).setSpa(fixedIp).setSha(macAddress).setLportTag(lportTag);
 731         builder.setInstructions(instructions);
 732         elanService.addArpResponderFlow(builder.buildForInstallFlow());
 733     }
 734
 735     private void removeArpResponderFlowsToExternalNetworkIp(BigInteger dpnId, Integer lportTag, String fixedIp,
 736             String extInterfaceName) {
 737         ArpResponderInput arpInput = new ArpReponderInputBuilder().setDpId(dpnId).setInterfaceName(extInterfaceName)
 738                 .setSpa(fixedIp).setLportTag(lportTag).buildForRemoveFlow();
 739         elanService.removeArpResponderFlow(arpInput);
 740     }
 741
 742     @Override
 743     public void onSubnetAddedToVpn(Subnetmap subnetmap, boolean isBgpVpn, Long elanTag) {
 744         vpnSubnetRouteHandler.onSubnetAddedToVpn(subnetmap, isBgpVpn, elanTag);
 745     }
 746
 747     @Override
 748     public void onSubnetDeletedFromVpn(Subnetmap subnetmap, boolean isBgpVpn) {
 749         vpnSubnetRouteHandler.onSubnetDeletedFromVpn(subnetmap, isBgpVpn);
 750     }
 751
 752     @Override
 753     public VpnInstance getVpnInstance(DataBroker broker, String vpnInstanceName) {
 754         return vpnUtil.getVpnInstance(vpnInstanceName);
 755     }
 756
 757     @Override
 758     public String getVpnRd(TypedReadTransaction<Configuration> confTx, String vpnName) {
 759         return VpnUtil.getVpnRd(confTx, vpnName);
 760     }
 761
 762     @Override
 763     public String getVpnRd(DataBroker broker, String vpnName) {
 764         return vpnUtil.getVpnRd(vpnName);
 765     }
 766
 767     @Override
 768     public VpnPortipToPort getNeutronPortFromVpnPortFixedIp(TypedReadTransaction<Configuration> confTx, String vpnName,
 769         String fixedIp) {
 770         return vpnUtil.getNeutronPortFromVpnPortFixedIp(confTx, vpnName, fixedIp);
 771     }
 772
 773     @Override
 774     public VpnPortipToPort getNeutronPortFromVpnPortFixedIp(DataBroker broker, String vpnName, String fixedIp) {
 775         return vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, fixedIp);
 776     }
 777
 778     @Override
 779     public Set<VpnTarget> getRtListForVpn(String vpnName) {
 780         return vpnUtil.getRtListForVpn(vpnName);
 781     }
 782
 783     @Override
 784     public void updateRouteTargetsToSubnetAssociation(Set<VpnTarget> routeTargets, String cidr, String vpnName) {
 785         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
 786             for (VpnTarget rt : routeTargets) {
 787                 String rtValue = rt.getVrfRTValue();
 788                 switch (rt.getVrfRTType()) {
 789                     case ImportExtcommunity:
 790                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr,  vpnName,
 791                                 tx, false/*isAssociationRemoved*/);
 792                         break;
 793                     case ExportExtcommunity:
 794                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName,
 795                                 tx, false/*isAssociationRemoved*/);
 796                         break;
 797                     case Both:
 798                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr, vpnName,
 799                                 tx, false/*isAssociationRemoved*/);
 800                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName,
 801                                 tx, false/*isAssociationRemoved*/);
 802                         break;
 803                     default:
 804                         LOG.error("updateRouteTargetsToSubnetAssociation: Invalid rt-type {} for vpn {}"
 805                                 + " subnet-cidr {}", rt.getVrfRTType(), vpnName, cidr);
 806                         break;
 807                 }
 808             }
 809         }), LOG, "updateRouteTargetsToSubnetAssociation: Failed for vpn {} subnet-cidr {}", vpnName, cidr);
 810         LOG.info("updateRouteTargetsToSubnetAssociation: Updated RT to subnet association for vpn {} cidr {}",
 811                 vpnName, cidr);
 812     }
 813
 814     @Override
 815     public void removeRouteTargetsToSubnetAssociation(Set<VpnTarget> routeTargets, String cidr, String vpnName) {
 816         ListenableFutures.addErrorLogging(txRunner.callWithNewReadWriteTransactionAndSubmit(tx -> {
 817             for (VpnTarget rt : routeTargets) {
 818                 String rtValue = rt.getVrfRTValue();
 819                 switch (rt.getVrfRTType()) {
 820                     case ImportExtcommunity:
 821                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr, vpnName, tx,
 822                                 true/*isAssociationRemoved*/);
 823                         break;
 824                     case ExportExtcommunity:
 825                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName, tx,
 826                                 true/*isAssociationRemoved*/);
 827                         break;
 828                     case Both:
 829                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.IRT, cidr, vpnName, tx,
 830                                 true/*isAssociationRemoved*/);
 831                         addSubnetAssociationOperationToTx(rtValue, RouteTarget.RtType.ERT, cidr, vpnName, tx,
 832                                 true/*isAssociationRemoved*/);
 833                         break;
 834                     default:
 835                         LOG.error("removeRouteTargetsToSubnetAssociation: Invalid route-target type {} for vpn {}"
 836                                 + " subnet-cidr {}", rt.getVrfRTType(), vpnName, cidr);
 837                         break;
 838                 }
 839             }
 840         }), LOG, "removeRouteTargetsToSubnetAssociation: Failed for vpn {} subnet-cidr {}", vpnName, cidr);
 841         LOG.info("removeRouteTargetsToSubnetAssociation: vpn {} cidr {}", vpnName, cidr);
 842     }
 843
 844     private boolean doesExistingVpnsHaveConflictingSubnet(Set<VpnTarget> routeTargets, String subnetCidr) {
 845         Set<RouteTarget> routeTargetSet = vpnUtil.getRouteTargetSet(routeTargets);
 846         for (RouteTarget routerTarget : routeTargetSet) {
 847             if (routerTarget.getAssociatedSubnet() != null) {
 848                 for (int i = 0; i < routerTarget.getAssociatedSubnet().size(); i++) {
 849                     AssociatedSubnet associatedSubnet = routerTarget.getAssociatedSubnet().get(i);
 850                     if (VpnUtil.areSubnetsOverlapping(associatedSubnet.getCidr(), subnetCidr)) {
 851                         return true;
 852                     }
 853                     if (routerTarget.getRtType() == RouteTarget.RtType.ERT) {
 854                         /* Check if there are multiple exports for the input iRT value (iRT in routeTargets)
 855                          *  Example : (1) iRT=A eRT=B subnet-range=S1; OK
 856                          *            (2) iRT=A eRT=B subnet-range=S1; OK
 857                          *            (3) iRT=B eRT=A subnet-range=S2; NOK
 858                          * Check if (1) and (2) are importing the same subnet-range routes to (3) */
 859                         List<AssociatedVpn> multipleAssociatedVpn = associatedSubnet.getAssociatedVpn();
 860                         if (multipleAssociatedVpn != null && multipleAssociatedVpn.size() > 1) {
 861                             LOG.error("doesExistingVpnsHaveConflictingSubnet: There is an indirect complete  overlap"
 862                                     + " for subnet CIDR {} for rt {} rtType {}", subnetCidr, routerTarget.getRt(),
 863                                     routerTarget.getRtType());
 864                             return true;
 865                         }
 866                         for (int j = i + 1; j < routerTarget.getAssociatedSubnet().size(); j++) {
 867                             if (VpnUtil.areSubnetsOverlapping(associatedSubnet.getCidr(),
 868                                     routerTarget.getAssociatedSubnet().get(j).getCidr())) {
 869                                 LOG.error("doesExistingVpnsHaveConflictingSubnet: There is an indirect paartial"
 870                                                 + " overlap for subnet CIDR {} for rt {} rtType {}", subnetCidr,
 871                                         routerTarget.getRt(), routerTarget.getRtType());
 872                                 return true;
 873                             }
 874                         }
 875                     }
 876                 }
 877                 /* Check if there are indirect EXPORTS for the eRT value (eRT in routeTargets)
 878                  *  Example : (1) iRT=A eRT=B subnet-range=S1; OK
 879                  *            (2) iRT=B eRT=A subnet-range=S2; OK
 880                  *            (3) iRT=A eRT=B subnet-range=S1; NOK
 881                  * If associatedSubnet is non-null for a routeTarget in (2),
 882                  * it may have already imported routes from (1) */
 883                 if (routerTarget.getRtType() == RouteTarget.RtType.IRT) {
 884                     try {
 885                         Optional<RouteTarget> indirectRts = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 886                                 LogicalDatastoreType.OPERATIONAL, VpnUtil.getRouteTargetsIdentifier(
 887                                         routerTarget.getRt(), RouteTarget.RtType.ERT));
 888                         if (indirectRts.isPresent() && routerTarget.getAssociatedSubnet() != null) {
 889                             for (AssociatedSubnet associatedSubnet : indirectRts.get().getAssociatedSubnet()) {
 890                                 if (VpnUtil.areSubnetsOverlapping(associatedSubnet.getCidr(), subnetCidr)) {
 891                                     LOG.error("doesExistingVpnsHaveConflictingSubnet: There is an indirect overlap for"
 892                                             + " subnet CIDR {} for rt {} rtType {}", subnetCidr, routerTarget.getRt(),
 893                                             routerTarget.getRtType());
 894                                     return true;
 895                                 }
 896                             }
 897                         }
 898                     } catch (ReadFailedException e) {
 899                         LOG.error("doesExistingVpnsHaveConflictingSubnet: Failed to read route targets to subnet"
 900                                 + "association for rt {} type {} subnet-cidr {}", routerTarget.getRt(),
 901                                 RouteTarget.RtType.ERT, subnetCidr);
 902                         return true; //Fail subnet association to avoid further damage to the data-stores
 903                     }
 904                 }
 905             } else {
 906                 LOG.info("doesExistingVpnsHaveConflictingSubnet: No prior associated subnets for rtVal {} rtType {}",
 907                         routerTarget.getRt(), routerTarget.getRtType());
 908             }
 909         }
 910         return false;
 911     }
 912
 913     private void addSubnetAssociationOperationToTx(String rt, RouteTarget.RtType rtType, String cidr,
 914                                                    String vpnName, ReadWriteTransaction tx,
 915                                                    boolean isAssociationRemoved)
 916             throws InterruptedException, ExecutionException {
 917         if (isAssociationRemoved) {
 918             //Remove RT-Subnet-Vpn Association
 919             Optional<AssociatedSubnet> associatedSubnet = tx.read(LogicalDatastoreType.OPERATIONAL,
 920                     VpnUtil.getAssociatedSubnetIdentifier(rt, rtType, cidr)).get();
 921             boolean deleteParent = false;
 922             if (associatedSubnet.isPresent()) {
 923                 List<AssociatedVpn> associatedVpns = associatedSubnet.get().getAssociatedVpn();
 924                 if (associatedVpns == null || associatedVpns.isEmpty()) {
 925                     deleteParent = true;
 926                 } else {
 927                     for (Iterator<AssociatedVpn> iterator = associatedVpns.iterator(); iterator.hasNext();) {
 928                         AssociatedVpn associatedVpn = (AssociatedVpn) iterator.next();
 929                         if (associatedVpn.getName().equals(vpnName)) {
 930                             iterator.remove();
 931                             break;
 932                         }
 933                     }
 934                     if (associatedVpns.isEmpty()) {
 935                         deleteParent = true;
 936                     }
 937                 }
 938             }
 939             if (deleteParent) {
 940                 deleteParentForSubnetToVpnAssocication(rt, rtType, cidr, tx);
 941             } else {
 942                 //Some other VPNs are also part of this rtVal, rtType and subnetCidr combination.
 943                 //Delete only this AssociatedVpn Object
 944                 tx.delete(LogicalDatastoreType.OPERATIONAL,
 945                         VpnUtil.getAssociatedSubnetAndVpnIdentifier(rt, rtType, cidr, vpnName));
 946                 LOG.debug("addSubnetAssocOperationToTx: Removed vpn {} from association rt {} rtType {} cidr {}",
 947                         vpnName, rt, rtType, cidr);
 948             }
 949         } else {
 950             //Add RT-Subnet-Vpn Association
 951             tx.put(LogicalDatastoreType.OPERATIONAL,
 952                     VpnUtil.getAssociatedSubnetAndVpnIdentifier(rt, rtType, cidr, vpnName),
 953                     VpnUtil.buildAssociatedSubnetAndVpn(vpnName), true);
 954         }
 955     }
 956
 957     private void deleteParentForSubnetToVpnAssocication(String rt, RouteTarget.RtType rtType,
 958                                                 String cidr, ReadWriteTransaction tx)
 959             throws InterruptedException, ExecutionException {
 960         //Check if you need to delete rtVal+rtType or just the subnetCidr
 961         InstanceIdentifier<RouteTarget> rtIdentifier = InstanceIdentifier
 962                 .builder(SubnetsAssociatedToRouteTargets.class).child(RouteTarget.class,
 963                         new RouteTargetKey(rt, rtType)).build();
 964         Optional<RouteTarget> rtToSubnetsAssociation = tx.read(LogicalDatastoreType.OPERATIONAL,
 965                 rtIdentifier).get();
 966         if (rtToSubnetsAssociation.isPresent()) {
 967             List<AssociatedSubnet> associatedSubnets = rtToSubnetsAssociation.get().getAssociatedSubnet();
 968             if (associatedSubnets != null && !associatedSubnets.isEmpty()) {
 969                 for (Iterator<AssociatedSubnet> iterator = associatedSubnets.iterator(); iterator.hasNext(); ) {
 970                     if (iterator.next().getCidr().equals(cidr)) {
 971                         iterator.remove();
 972                         break;
 973                     }
 974                 }
 975                 if (associatedSubnets.isEmpty()) {
 976                     //The entire rt to subnet association is empty
 977                     //Delete the RouteTarget object
 978                     tx.delete(LogicalDatastoreType.OPERATIONAL, rtIdentifier);
 979                     LOG.debug("deleteParentForSubnetToVpnAssocication: Removed rt {} rtType {} from association,",
 980                             rt, rtType);
 981                 } else {
 982                     //Some other VPNs are also part of this rtVal, rtType combination
 983                     //Delete only this AssociatedSubnet
 984                     tx.delete(LogicalDatastoreType.OPERATIONAL, VpnUtil.getAssociatedSubnetIdentifier(rt, rtType,
 985                             cidr));
 986                     LOG.debug("deleteParentForSubnetToVpnAssocication: Removed cidr {} from association rt {}"
 987                             + " rtType {}", cidr, rt, rtType);
 988                 }
 989             }
 990         }
 991     }
 992
 993     @Override
 994     public boolean checkForOverlappingSubnets(Uuid network, List<Subnetmap> subnetmapList, Uuid vpn,
 995                                        Set<VpnTarget> routeTargets, List<String> failedNwList) {
 996         for (int i = 0; i < subnetmapList.size(); i++) {
 997             //Check if any other subnet that is already part of a different vpn with same rt, has overlapping CIDR
 998             if (checkExistingSubnetWithSameRoutTargets(routeTargets, vpn, subnetmapList.get(i), failedNwList)) {
 999                 return true;
1000             }
1001         }
1002         return false;
1003     }
1004
1005     private boolean checkExistingSubnetWithSameRoutTargets(Set<VpnTarget> routeTargets, Uuid vpn, Subnetmap subnetmap,
1006                                                    List<String> failedNwList) {
1007         String cidr = String.valueOf(subnetmap.getSubnetIp());
1008         boolean subnetExistsWithSameRt = doesExistingVpnsHaveConflictingSubnet(routeTargets, cidr);
1009         if (subnetExistsWithSameRt) {
1010             failedNwList.add(String.format("Another subnet with the same/overlapping cidr %s as subnet %s"
1011                     + " is already associated to a vpn with routeTargets %s. Ignoring subnet addition to vpn"
1012                     + " %s", cidr, subnetmap.getId().getValue(), routeTargets.toString(), vpn.getValue()));
1013         }
1014         return subnetExistsWithSameRt;
1015     }
1016 }