X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=vpnmanager%2Fimpl%2Fsrc%2Fmain%2Fjava%2Forg%2Fopendaylight%2Fnetvirt%2Fvpnmanager%2FVpnInterfaceManager.java;h=bc996c6075581197dfb221f453c9e2396a5a8a07;hb=cf1ea9172b94fc17e45391b62bf4ca24ac5c5fe0;hp=509e553b08b0832dd468809094e5a6019ae2a9ad;hpb=5db0a8a1aa8c6c9ce8bf7c39f7044dd2f9a59fb6;p=netvirt.git diff --git a/vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java b/vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java index 509e553b08..bc996c6075 100755 --- a/vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java +++ b/vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java @@ -7,46 +7,56 @@ */ package org.opendaylight.netvirt.vpnmanager; -import com.google.common.base.Optional; +import static java.util.Collections.emptyList; +import static org.opendaylight.genius.infra.Datastore.CONFIGURATION; +import static org.opendaylight.genius.infra.Datastore.OPERATIONAL; + import com.google.common.base.Preconditions; import com.google.common.collect.Iterators; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.math.BigInteger; import java.util.ArrayList; import java.util.Collections; -import java.util.Iterator; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; import java.util.function.Predicate; import java.util.stream.Collectors; -import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import javax.inject.Inject; import javax.inject.Singleton; - -import org.opendaylight.controller.md.sal.binding.api.DataBroker; -import org.opendaylight.controller.md.sal.binding.api.WriteTransaction; -import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType; -import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase; +import org.eclipse.jdt.annotation.Nullable; +import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker; +import org.opendaylight.genius.infra.Datastore.Configuration; +import org.opendaylight.genius.infra.Datastore.Operational; import org.opendaylight.genius.infra.ManagedNewTransactionRunner; import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl; +import org.opendaylight.genius.infra.TypedReadWriteTransaction; +import org.opendaylight.genius.infra.TypedWriteTransaction; import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager; -import org.opendaylight.genius.mdsalutil.MDSALUtil; import org.opendaylight.genius.mdsalutil.NWUtil; import org.opendaylight.genius.mdsalutil.NwConstants; -import org.opendaylight.genius.mdsalutil.cache.DataObjectCache; +import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache; import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager; +import org.opendaylight.genius.utils.JvmGlobalLocks; import org.opendaylight.infrautils.caches.CacheProvider; import org.opendaylight.infrautils.jobcoordinator.JobCoordinator; +import org.opendaylight.infrautils.utils.concurrent.Executors; import org.opendaylight.infrautils.utils.concurrent.ListenableFutures; +import org.opendaylight.mdsal.binding.api.DataBroker; +import org.opendaylight.mdsal.common.api.LogicalDatastoreType; +import org.opendaylight.mdsal.common.api.TransactionCommitFailedException; import org.opendaylight.netvirt.bgpmanager.api.IBgpManager; import org.opendaylight.netvirt.fibmanager.api.FibHelper; import org.opendaylight.netvirt.fibmanager.api.IFibManager; @@ -58,10 +68,7 @@ import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler; import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput; import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator; import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry; -import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces; -import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface; -import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey; -import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames; +import org.opendaylight.serviceutils.tools.listener.AbstractAsyncDataTreeChangeListener; import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface; import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid; import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService; @@ -80,23 +87,14 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev15033 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfaces; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesBuilder; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesKey; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder; @@ -105,15 +103,28 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.routers.ExternalIps; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.Adjacencies; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.VpnInterfaces; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.Adjacency; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.Adjacency.AdjacencyType; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.AdjacencyBuilder; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.AdjacencyKey; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.VpnInterface; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.VpnInterfaceKey; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.vpn._interface.VpnInstanceNames; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort; import org.opendaylight.yangtools.yang.binding.InstanceIdentifier; import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder; +import org.opendaylight.yangtools.yang.common.Uint32; +import org.opendaylight.yangtools.yang.common.Uint64; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @Singleton -public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase { +public class VpnInterfaceManager extends AbstractAsyncDataTreeChangeListener { private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class); private static final short DJC_MAX_RETRIES = 3; @@ -130,13 +141,14 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase vpnIntfMap = new ConcurrentHashMap<>(); private final Map> unprocessedVpnInterfaces = new ConcurrentHashMap<>(); - private final DataObjectCache vpnInstanceOpDataEntryCache; + private final InstanceIdDataObjectCache vpnInstanceOpDataEntryCache; @Inject public VpnInterfaceManager(final DataBroker dataBroker, @@ -150,8 +162,11 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase(VpnInstanceOpDataEntry.class, dataBroker, + vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker, LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder( VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider); + start(); } public Runnable isNotifyTaskQueued(String intfName) { return vpnIntfMap.remove(intfName); } - @PostConstruct public void start() { LOG.info("{} start", getClass().getSimpleName()); - registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker); } @Override @@ -186,42 +201,39 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase getWildCardPath() { - return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class); - } - - @Override - protected VpnInterfaceManager getDataTreeChangeListener() { - return VpnInterfaceManager.this; + Executors.shutdownAndAwaitTermination(getExecutorService()); } @Override public void add(final InstanceIdentifier identifier, final VpnInterface vpnInterface) { - LOG.info("add: intfName {} onto vpnName {}", - vpnInterface.getName(), - VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames())); + LOG.trace("Received VpnInterface add event: vpnInterface={}", vpnInterface); + LOG.info("add: intfName {} onto vpnName {}", vpnInterface.getName(), + VpnHelper.getVpnInterfaceVpnInstanceNamesString( + new ArrayList(vpnInterface.getVpnInstanceNames().values()))); addVpnInterface(identifier, vpnInterface, null, null); } private boolean canHandleNewVpnInterface(final InstanceIdentifier identifier, final VpnInterface vpnInterface, String vpnName) { - synchronized (vpnName.intern()) { + // FIXME: separate this out somehow? + final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnName); + lock.lock(); + try { if (isVpnInstanceReady(vpnName)) { return true; } addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName); return false; + } finally { + lock.unlock(); } } // TODO Clean up the exception handling @SuppressWarnings("checkstyle:IllegalCatch") private void addVpnInterface(final InstanceIdentifier identifier, final VpnInterface vpnInterface, - final List oldAdjs, final List newAdjs) { - for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) { + final @Nullable List oldAdjs, final @Nullable List newAdjs) { + for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames().values()) { String vpnName = vpnInterfaceVpnInstance.getVpnName(); addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName); } @@ -229,7 +241,7 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase identifier, final VpnInterface vpnInterface, final List oldAdjs, final List newAdjs, String vpnName) { - final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class); + final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class); final String interfaceName = key.getName(); if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) { @@ -253,119 +265,126 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase vpnInterfaceOpIdentifier, - final VpnInterface vpnInterface, final List oldAdjs, - final List newAdjs, + final VpnInterface vpnInterface, final @Nullable List oldAdjs, + final @Nullable List newAdjs, final InstanceIdentifier identifier, String vpnName) { - final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class); + final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class); final String interfaceName = key.getName(); - String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName); - if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) { + String primaryRd = vpnUtil.getPrimaryRd(vpnName); + if (!vpnUtil.isVpnPendingDelete(primaryRd)) { Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName); - boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName); + boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName); if (interfaceState != null) { try { - final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState); + final Uint64 dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState); final int ifIndex = interfaceState.getIfIndex(); jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> { - WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction(); - WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction(); - WriteTransaction writeInvTxn = dataBroker.newWriteOnlyTransaction(); - LOG.info("addVpnInterface: VPN Interface add event - intfName {} vpnName {} on dpn {}" , - vpnInterface.getName(), vpnName, vpnInterface.getDpnId()); - processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false, writeConfigTxn, - writeOperTxn, writeInvTxn, interfaceState, vpnName); - if (oldAdjs != null && !oldAdjs.equals(newAdjs)) { - LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}" - + " Update for swapping VPN {} case.", interfaceName, vpnName); - if (newAdjs != null) { - for (Adjacency adj : newAdjs) { - if (oldAdjs.contains(adj)) { - oldAdjs.remove(adj); - } else { - if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet( - dataBroker, adj)) { - addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, - adj, dpnId, writeOperTxn, writeConfigTxn); - } - } - } - } - for (Adjacency adj : oldAdjs) { - if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet( - dataBroker, adj)) { - delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, - writeOperTxn, writeConfigTxn); - } - } - } - ListenableFuture operFuture = writeOperTxn.submit(); - try { - operFuture.get(); - } catch (ExecutionException e) { - LOG.error("addVpnInterface: Exception encountered while submitting operational future for" - + " addVpnInterface {} on vpn {}: {}", vpnInterface.getName(), vpnName, e); - return null; - } + // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in + // (the inventory tx goes in last) List> futures = new ArrayList<>(); - ListenableFuture configFuture = writeConfigTxn.submit(); - futures.add(configFuture); - Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config")); - futures.add(writeInvTxn.submit()); + //set of prefix used, as entry in prefix-to-interface datastore + // is prerequisite for refresh Fib to avoid race condition leading to + // missing remote next hop in bucket actions on bgp-vpn delete + Set prefixListForRefreshFib = new HashSet<>(); + ListenableFuture confFuture = + txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, + confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, + operTx -> futures.add( + txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, invTx -> { + LOG.info( + "addVpnInterface: VPN Interface add event - intfName {} vpnName {}" + + " on dpn {}", + vpnInterface.getName(), vpnName, vpnInterface.getDpnId()); + processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false, + confTx, operTx, invTx, interfaceState, vpnName, + prefixListForRefreshFib); + if (oldAdjs != null && !oldAdjs.equals(newAdjs)) { + LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}" + + " Update for swapping VPN {} case.", interfaceName, vpnName); + if (newAdjs != null) { + for (Adjacency adj : newAdjs) { + if (oldAdjs.contains(adj)) { + oldAdjs.remove(adj); + } else { + if (!isBgpVpnInternetVpn + || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) { + addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, + primaryRd, adj, dpnId, operTx, confTx, invTx, + prefixListForRefreshFib); + } + } + } + } + for (Adjacency adj : oldAdjs) { + if (!isBgpVpnInternetVpn + || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) { + delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, + operTx, confTx); + } + } + } + }))))); + Futures.addCallback(confFuture, + new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib), + MoreExecutors.directExecutor()); + futures.add(confFuture); + Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"), + MoreExecutors.directExecutor()); LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}" - + " processed successfully", interfaceName, vpnName, dpnId); + + " processed successfully", interfaceName, vpnName, dpnId); return futures; }); } catch (NumberFormatException | IllegalStateException e) { LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for " - + "interface {}. Interface addition on vpn {} failed", interfaceName, - vpnName, e); + + "interface {}. Interface addition on vpn {} failed", interfaceName, + vpnName, e); return; } } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) { jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> { - WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction(); - createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName, - writeConfigTxn, vpnName); - LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName, - vpnName, vpnInterface.getDpnId()); - ListenableFuture futures = writeConfigTxn.submit(); - String errorText = "addVpnInterfaceCall: Exception encountered while submitting writeConfigTxn" - + " for interface " + vpnInterface.getName() + " on vpn " + vpnName; - ListenableFutures.addErrorLogging(futures, LOG, errorText); - return Collections.singletonList(futures); + ListenableFuture future = + txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> { + createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName, + confTx, vpnName); + LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName, + vpnName, vpnInterface.getDpnId()); + }); + ListenableFutures.addErrorLogging(future, LOG, + "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName); + return Collections.singletonList(future); }); } else { LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState" - + " is not available", interfaceName, vpnName); + + " is not available", interfaceName, vpnName); } } else { LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped" - + " as vpn is pending delete", interfaceName, vpnName, - vpnInterface.getDpnId()); + + " as vpn is pending delete", interfaceName, vpnName, + vpnInterface.getDpnId()); } } // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation - // see comments below. @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"}) - protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd, + protected void processVpnInterfaceUp(final Uint64 dpId, VpnInterface vpnInterface, final String primaryRd, final int lportTag, boolean isInterfaceUp, - WriteTransaction writeConfigTxn, - WriteTransaction writeOperTxn, - WriteTransaction writeInvTxn, - Interface interfaceState, - final String vpnName) { + TypedWriteTransaction writeConfigTxn, + TypedWriteTransaction writeOperTxn, + TypedReadWriteTransaction writeInvTxn, + Interface interfaceState, final String vpnName, + Set prefixListForRefreshFib) throws ExecutionException, InterruptedException { final String interfaceName = vpnInterface.getName(); - Optional optOpVpnInterface = VpnUtil.getVpnInterfaceOpDataEntry(dataBroker, - interfaceName, vpnName); + Optional optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName, + vpnName); VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null; - boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName); + boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName); if (!isInterfaceUp) { LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}", interfaceName, dpId, vpnName); - long vpnId = VpnUtil.getVpnId(dataBroker, vpnName); - if (vpnId == VpnConstants.INVALID_ID) { + Uint32 vpnId = vpnUtil.getVpnId(vpnName); + if (VpnConstants.INVALID_ID.equals(vpnId)) { LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}" + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName, dpId); @@ -373,15 +392,15 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName); + List adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName); if (adjs == null) { LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies" + " for this vpn interface could not be obtained", interfaceName, dpId, @@ -401,7 +420,7 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase identifier = - VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName()); - InstanceIdentifier vpnInterfaceOpIdentifier = - VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName); - advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName); - // Perform similar operation as interface add event for extraroutes. - InstanceIdentifier path = identifier.augmentation(Adjacencies.class); - Optional optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path); - if (!optAdjacencies.isPresent()) { - LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface); - return; - } - List adjacencies = optAdjacencies.get().getAdjacency(); - for (Adjacency adjacency : adjacencies) { - if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) { - continue; + try { + // Interface is retained in the DPN, but its Link Up. + // Advertise prefixes again for this interface to BGP + InstanceIdentifier identifier = + VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName()); + InstanceIdentifier vpnInterfaceOpIdentifier = + VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName); + advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName); + // Perform similar operation as interface add event for extraroutes. + InstanceIdentifier path = identifier.augmentation(Adjacencies.class); + Optional optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, + LogicalDatastoreType.CONFIGURATION, path); + if (!optAdjacencies.isPresent()) { + LOG.trace("No config adjacencyKeyAdjacencyMap present for vpninterface {}", vpnInterface); + return; } - // if BGPVPN Internet, filter only IPv6 Adjacencies - if (isBgpVpnInternetVpn && !VpnUtil.isAdjacencyEligibleToVpnInternet( - dataBroker, adjacency)) { - continue; + Map adjacencyKeyAdjacencyMap = optAdjacencies.get().nonnullAdjacency(); + for (Adjacency adjacency : adjacencyKeyAdjacencyMap.values()) { + if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) { + continue; + } + // if BGPVPN Internet, filter only IPv6 Adjacencies + if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) { + continue; + } + addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency, + dpId, writeOperTxn, writeConfigTxn, writeInvTxn, prefixListForRefreshFib); } - addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency, - dpId, writeOperTxn, writeConfigTxn); + } catch (InterruptedException | ExecutionException e) { + LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}", + interfaceName, vpnName, primaryRd, dpId); } } } - private void processExternalVpnInterface(String interfaceName, String vpnName, long vpnId, BigInteger dpId, - int lportTag, WriteTransaction writeInvTxn, int addOrRemove) { + private void processExternalVpnInterface(String interfaceName, String vpnName, Uint64 dpId, + int lportTag, int addOrRemove) { Uuid extNetworkId; try { // vpn instance of ext-net interface is the network-id @@ -522,7 +545,7 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase routerIds = VpnUtil.getExternalNetworkRouterIds(dataBroker, extNetworkId); + List routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId); if (routerIds == null || routerIds.isEmpty()) { LOG.info("processExternalVpnInterface: No router is associated with {}." + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}", @@ -534,17 +557,19 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase(router + .getExternalIps().values())), router.getExtGwMacAddress(), + dpId, interfaceName, lportTag); } else { vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName, - VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), + VpnUtil.getIpsListFromExternalIps(new ArrayList(router + .getExternalIps().values())), dpId, interfaceName, lportTag); } } else { @@ -558,7 +583,7 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase identifier, String vpnName, String interfaceName) { if (rd == null) { @@ -583,60 +608,65 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase path = identifier.augmentation(AdjacenciesOp.class); - Optional adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path); - if (adjacencies.isPresent()) { - List nextHops = adjacencies.get().getAdjacency(); - - if (!nextHops.isEmpty()) { - LOG.debug("advertiseAdjacenciesForVpnToBgp: NextHops are {} for interface {} on dpn {} for vpn {}" - + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd); - VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, rd); - long l3vni = vpnInstanceOpData.getL3vni(); - VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni) - ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre; - for (Adjacency nextHop : nextHops) { - if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) { - continue; - } - String gatewayMac = null; - long label = 0; - if (VpnUtil.isL3VpnOverVxLan(l3vni)) { - final VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker, - vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress()); - gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName).get(); - } else { - label = nextHop.getLabel(); - } - try { - LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}" - + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label); - bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp, - encapType, (int)label, l3vni, 0 /*l2vni*/, - gatewayMac); - LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}" - + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd, - nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName); - } catch (Exception e) { - LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {} with rd {}" - + " for interface {} on dpn {}", nextHop.getIpAddress(), - vpnName, rd, interfaceName, dpnId, e); + try { + //Read NextHops + InstanceIdentifier path = identifier.augmentation(AdjacenciesOp.class); + Optional adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, + LogicalDatastoreType.OPERATIONAL, path); + if (adjacencies.isPresent()) { + Map nextHopsMap = adjacencies.get().getAdjacency(); + if (nextHopsMap != null && !nextHopsMap.isEmpty()) { + LOG.debug("advertiseAdjacenciesForVpnToBgp: NextHops are {} for interface {} on dpn {} for vpn {}" + + " rd {}", nextHopsMap, interfaceName, dpnId, vpnName, rd); + VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd); + Uint32 l3vni = vpnInstanceOpData.getL3vni(); + VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni) + ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre; + for (Adjacency nextHop : nextHopsMap.values()) { + if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) { + continue; + } + String gatewayMac = null; + Uint32 label = Uint32.ZERO; + if (VpnUtil.isL3VpnOverVxLan(l3vni)) { + final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp( + vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress()); + gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName) + .get(); + } else { + label = nextHop.getLabel(); + } + try { + LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}" + + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label); + bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp, + encapType, label, l3vni, Uint32.ZERO /*l2vni*/, + gatewayMac); + LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}" + + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd, + nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName); + } catch (Exception e) { + LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}" + + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd, + interfaceName, dpnId, e); + } } } } + } catch (InterruptedException | ExecutionException e) { + LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}" + + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd); } } // TODO Clean up the exception handling @SuppressWarnings("checkstyle:IllegalCatch") private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier identifier, - String vpnName, String interfaceName, WriteTransaction writeConfigTxn) { + String vpnName, String interfaceName, TypedWriteTransaction writeConfigTxn, + TypedWriteTransaction writeOperTx) { //Read NextHops InstanceIdentifier path = identifier.augmentation(AdjacenciesOp.class); - Optional adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path); - - String rd = VpnUtil.getVpnRd(dataBroker, interfaceName); + String rd = vpnUtil.getVpnRd(interfaceName); if (rd == null) { LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}", interfaceName, vpnName); @@ -651,13 +681,21 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase adjacencies = Optional.empty(); + try { + adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, + path); + } catch (InterruptedException | ExecutionException e) { + LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}", + interfaceName, vpnName); + } if (adjacencies.isPresent()) { - List nextHops = adjacencies.get().getAdjacency(); + Map nextHopsMap = adjacencies.get().getAdjacency(); - if (!nextHops.isEmpty()) { + if (nextHopsMap != null && !nextHopsMap.isEmpty()) { LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}", - nextHops, interfaceName, vpnName, rd); - for (Adjacency nextHop : nextHops) { + nextHopsMap, interfaceName, vpnName, rd); + for (Adjacency nextHop : nextHopsMap.values()) { try { if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) { LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}" @@ -667,12 +705,13 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase writeConfigTxn, + TypedWriteTransaction writeOperTxn, + TypedReadWriteTransaction writeInvTxn, + Interface interfaceState, Set prefixListForRefreshFib) + throws ExecutionException, InterruptedException { InstanceIdentifier identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName); // Read NextHops - InstanceIdentifier path = identifier.augmentation(Adjacencies.class); - Optional adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path); - if (!adjacencies.isPresent()) { - addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag, - null/*gwMac*/, writeOperTxn); - return; + Optional vpnInteface = Optional.empty(); + try { + vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker, + LogicalDatastoreType.CONFIGURATION, identifier); + } catch (InterruptedException | ExecutionException e) { + LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}" + + "dpn {}", interfaceName, vpnName, primaryRd, dpnId); + } + Uuid intfnetworkUuid = null; + NetworkType networkType = null; + Long segmentationId = Long.valueOf(-1); + Adjacencies adjacencies = null; + if (vpnInteface.isPresent()) { + intfnetworkUuid = vpnInteface.get().getNetworkId(); + networkType = vpnInteface.get().getNetworkType(); + segmentationId = vpnInteface.get().getSegmentationId().toJava(); + adjacencies = vpnInteface.get().augmentation(Adjacencies.class); + if (adjacencies == null) { + addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag, + null/*gwMac*/, null/*gatewayIp*/, writeOperTxn); + return; + } } - // Get the rd of the vpn instance String nextHopIp = null; + String gatewayIp = null; try { nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId); } catch (Exception e) { @@ -715,16 +771,17 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase gwMac = Optional.absent(); + Optional gwMac = Optional.empty(); String vpnInterfaceSubnetGwMacAddress = null; - VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd); - Long l3vni = vpnInstanceOpData.getL3vni(); + VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd); + Uint32 l3vni = vpnInstanceOpData.getL3vni() != null ? vpnInstanceOpData.getL3vni() : Uint32.ZERO; boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni); VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre; VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType); - List nextHops = adjacencies.get().getAdjacency(); + Map nextHopsMap = adjacencies != null ? adjacencies.getAdjacency() + : Collections.emptyMap(); List value = new ArrayList<>(); - for (Adjacency nextHop : nextHops) { + for (Adjacency nextHop : nextHopsMap.values()) { String rd = primaryRd; String nexthopIpValue = nextHop.getIpAddress().split("/")[0]; if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet @@ -739,19 +796,20 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase gatewayIpOptional = VpnUtil.getVpnSubnetGatewayIp(dataBroker, subnetId); + Optional gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId); if (gatewayIpOptional.isPresent()) { gatewayIp = gatewayIpOptional.get(); } @@ -774,15 +832,15 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase rdToAllocate = VpnUtil - .allocateRdForExtraRouteAndUpdateUsedRdsMap(dataBroker, vpnId, null, - prefix, vpnName, nextHop.getNextHopIpList().get(0), dpnId); + // FIXME: separate this out somehow? + final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey); + lock.lock(); + try { + java.util.Optional rdToAllocate = vpnUtil + .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName, + nextHop.getNextHopIpList().get(0), dpnId); if (rdToAllocate.isPresent()) { rd = rdToAllocate.get(); LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}", @@ -809,6 +870,8 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase writeOperTxn) { VpnInterfaceOpDataEntry opInterface = - VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, Boolean.FALSE, lportTag, gwMac); + VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac, gwIp); InstanceIdentifier interfaceId = VpnUtil .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName); - writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface, - WriteTransaction.CREATE_MISSING_PARENTS); + writeOperTxn.mergeParentStructurePut(interfaceId, opInterface); LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore", interfaceName, dpnId, vpnName); } @@ -873,45 +934,45 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>(); - if (adjList.isEmpty()) { + TypedWriteTransaction writeConfigTxn, + TypedWriteTransaction writeOperTxn) { + + String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue(); + Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern(); + AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class); + Map keyAdjacencyMap = + adjacencies != null && adjacencies.getAdjacency() != null ? adjacencies.getAdjacency() + : Collections.emptyMap(); + if (keyAdjacencyMap.isEmpty()) { LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}", vpnInterface, srcDpnId); return; } String prefix = null; - long label = 0; List value = new ArrayList<>(); - boolean isNextHopAddReqd = false; + boolean isFibNextHopAddReqd = false; String vpnName = vpnInterface.getVpnInstanceName(); - long vpnId = VpnUtil.getVpnId(dataBroker, vpnName); - String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName); + Uint32 vpnId = vpnUtil.getVpnId(vpnName); + String primaryRd = vpnUtil.getPrimaryRd(vpnName); LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}", vpnInterface.getName(), vpnInterface.getDpnId(), - vpnInterface.getVpnInstanceName(), adjList); - for (Adjacency adj : adjList) { + vpnInterface.getVpnInstanceName(), keyAdjacencyMap); + for (Adjacency adj : keyAdjacencyMap.values()) { String rd = adj.getVrfId(); rd = rd != null ? rd : vpnName; prefix = adj.getIpAddress(); - label = adj.getLabel(); + Uint32 label = adj.getLabel(); List nhList = Collections.singletonList(srcTepIp); List nextHopList = adj.getNextHopIpList(); // If TEP is added , update the nexthop of primary adjacency. // Secondary adj nexthop is already pointing to primary adj IP address. - if (nextHopList != null && !nextHopList.isEmpty() && nextHopList.get(0).equalsIgnoreCase(srcTepIp)) { - /* everything right already */ - } else { - isNextHopAddReqd = true; - } - if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) { value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build()); + if (nextHopList != null && !nextHopList.isEmpty()) { + /* everything right already */ + } else { + isFibNextHopAddReqd = true; + } } else { Optional vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix); if (!vrfEntryOptional.isPresent()) { @@ -920,29 +981,28 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase vpnsToImportRoute = - VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName); + List vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName); for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) { String vpnRd = vpn.getVrfId(); if (vpnRd != null) { - fibManager.updateRoutePathForFibEntry(vpnRd, prefix, - srcTepIp, label, true, writeConfigTxn); + fibManager.updateRoutePathForFibEntry(vpnRd, prefix, srcTepIp, label, true, + writeConfigTxn); LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}" + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label, vpnInterface.getName(), srcDpnId, vpnName, @@ -954,7 +1014,7 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase interfaceId = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName); - writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface, - WriteTransaction.CREATE_MISSING_PARENTS); + writeOperTxn.mergeParentStructurePut(interfaceId, opInterface); LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}", vpnInterface.getName(), srcDpnId, vpnName); @@ -985,19 +1043,19 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase writeConfigTxn, + TypedWriteTransaction writeOperTxn) { - AdjacenciesOp adjacencies = vpnInterface.getAugmentation(AdjacenciesOp.class); - List adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>(); + AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class); + List adjList = adjacencies != null ? new ArrayList(adjacencies.getAdjacency().values()) + : new ArrayList<>(); String prefix = null; - long label = 0; boolean isNextHopRemoveReqd = false; - String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue()); - BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId()); + String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue(); + Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern(); String vpnName = vpnInterface.getVpnInstanceName(); - long vpnId = VpnUtil.getVpnId(dataBroker, vpnName); - String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName); + Uint32 vpnId = vpnUtil.getVpnId(vpnName); + String primaryRd = vpnUtil.getVpnRd(vpnName); if (adjList != null) { List value = new ArrayList<>(); LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}", @@ -1009,7 +1067,7 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase nextHopList = adj.getNextHopIpList(); - label = adj.getLabel(); + Uint32 label = adj.getLabel(); if (nextHopList != null && !nextHopList.isEmpty()) { isNextHopRemoveReqd = true; } @@ -1037,18 +1095,17 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase vpnsToImportRoute = - VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName); + List vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName); for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) { String vpnRd = vpn.getVrfId(); if (vpnRd != null) { - fibManager.updateRoutePathForFibEntry(vpnRd, prefix, - srcTepIp, label, false, writeConfigTxn); + fibManager.updateRoutePathForFibEntry(vpnRd, prefix, srcTepIp, label, false, + writeConfigTxn); LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}" + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label, vpnInterface.getName(), srcDpnId, @@ -1068,36 +1125,41 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase interfaceId = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName); - writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface, - WriteTransaction.CREATE_MISSING_PARENTS); + writeOperTxn.mergeParentStructurePut(interfaceId, opInterface); LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}", vpnInterface.getName(), srcDpnId, vpnName); } } + @SuppressWarnings("checkstyle:IllegalCatch") private List getVpnsExportingMyRoute(final String vpnName) { List vpnsToExportRoute = new ArrayList<>(); - - String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnName); - final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd); - if (vpnInstanceOpDataEntry == null) { - LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}" - + " to check for vpns exporting the routes", vpnName); + final VpnInstanceOpDataEntry vpnInstanceOpDataEntry; + String vpnRd = vpnUtil.getVpnRd(vpnName); + try { + VpnInstanceOpDataEntry opDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd); + if (opDataEntry == null) { + LOG.error("getVpnsExportingMyRoute: Null vpn instance op data for vpn {} rd {}" + + " when check for vpns exporting the routes", vpnName, vpnRd); + return vpnsToExportRoute; + } + vpnInstanceOpDataEntry = opDataEntry; + } catch (Exception re) { + LOG.error("getVpnsExportingMyRoute: DSexception when retrieving vpn instance op data for vpn {} rd {}" + + " to check for vpns exporting the routes", vpnName, vpnRd, re); return vpnsToExportRoute; } - Predicate excludeVpn = input -> { if (input.getVpnInstanceName() == null) { LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {} without a name", @@ -1115,7 +1177,7 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase vpnsToExportRoute = getVpnsExportingMyRoute(vpnName); for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) { - List vrfEntries = VpnUtil.getAllVrfEntries(dataBroker, vpn.getVrfId()); - WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction(); + List vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId()); if (vrfEntries != null) { - for (VrfEntry vrfEntry : vrfEntries) { - try { - if (!FibHelper.isControllerManagedNonInterVpnLinkRoute( - RouteOrigin.value(vrfEntry.getOrigin()))) { - LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}" - + " is not a controller managed non intervpn link route. Ignoring.", - vpn.getVrfId(), vrfEntry.getDestPrefix()); - continue; - } - String prefix = vrfEntry.getDestPrefix(); - String gwMac = vrfEntry.getGatewayMacAddress(); - vrfEntry.getRoutePaths().forEach(routePath -> { - String nh = routePath.getNexthopAddress(); - int label = routePath.getLabel().intValue(); - if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value( - vrfEntry.getOrigin()))) { - LOG.info("handleVpnsExportingRoutesImporting: Importing fib entry rd {} prefix {}" - + " nexthop {} label {} to vpn {} vpnRd {}", vpn.getVrfId(), prefix, nh, label, - vpnName, vpnRd); - fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix, - Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label, - 0 /*l3vni*/, gwMac, null /*parentVpnRd*/, RouteOrigin.SELF_IMPORTED, - writeConfigTxn); - } else { - LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry rd {} prefix {}" - + " nexthop {} label {} to vpn {} vpnRd {}", vpn.getVrfId(), prefix, nh, label, - vpnName, vpnRd); - SubnetRoute route = vrfEntry.getAugmentation(SubnetRoute.class); - importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, writeConfigTxn); + ListenableFutures.addErrorLogging( + txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> { + for (VrfEntry vrfEntry : vrfEntries) { + try { + if (!FibHelper.isControllerManagedNonInterVpnLinkRoute( + RouteOrigin.value(vrfEntry.getOrigin()))) { + LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}" + + " is not a controller managed non intervpn link route. Ignoring.", + vpn.getVrfId(), vrfEntry.getDestPrefix()); + continue; + } + String prefix = vrfEntry.getDestPrefix(); + String gwMac = vrfEntry.getGatewayMacAddress(); + vrfEntry.nonnullRoutePaths().values().forEach(routePath -> { + String nh = routePath.getNexthopAddress(); + Uint32 label = routePath.getLabel(); + if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value( + vrfEntry.getOrigin()))) { + LOG.info( + "handleVpnsExportingRoutesImporting: Importing fib entry rd {}" + + " prefix {} nexthop {} label {} to vpn {} vpnRd {}", + vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd); + fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix, + Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label, + Uint32.ZERO /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED, + confTx); + } else { + LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry" + + " rd {} prefix {} nexthop {} label {} to vpn {} vpnRd {}", + vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd); + SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class); + importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(), + confTx); + } + }); + } catch (RuntimeException e) { + LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}" + + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(), + vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd); } - }); - } catch (RuntimeException e) { - LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}" - + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(), - vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd); - } - } - writeConfigTxn.submit(); + } + }), LOG, "Error handing VPN exporting routes"); } else { LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}", vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd); @@ -1173,12 +1238,12 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase identifier, VpnInterface vpnInterface) { - final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class); + LOG.trace("Received VpnInterface remove event: vpnInterface={}", vpnInterface); + final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class); final String interfaceName = key.getName(); - for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) { + for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames().values()) { String vpnName = vpnInterfaceVpnInstance.getVpnName(); removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName); } @@ -1189,14 +1254,14 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase { - WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction(); - deleteFibEntryForRouterInterface(vpnInterface, writeConfigTxn, vpnName); - LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName); - ListenableFuture futures = writeConfigTxn.submit(); - String errorText = "removeVpnInterfaceCall: Exception encountered while submitting writeConfigTxn" - + " for interface " + vpnInterface.getName() + " on vpn " + vpnName; - ListenableFutures.addErrorLogging(futures, LOG, errorText); - return Collections.singletonList(futures); + ListenableFuture future = + txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> { + deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName); + LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName); + }); + ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}", + vpnInterface.getName(), vpnName); + return Collections.singletonList(future); }, DJC_MAX_RETRIES); } else { Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName); @@ -1206,79 +1271,86 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase identifier, - final VpnInterface vpnInterface, final String vpnName, - final String interfaceName, final Interface interfaceState) { + final VpnInterface vpnInterface, final String vpnName, + final String interfaceName, final Interface interfaceState) { LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(), vpnName, vpnInterface.getDpnId()); removeInterfaceFromUnprocessedList(identifier, vpnInterface); jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> { List> futures = new ArrayList<>(3); - ListenableFuture configFuture = txRunner - .callWithNewWriteOnlyTransactionAndSubmit(writeConfigTxn -> { - futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeOperTxn -> { - futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeInvTxn -> { - LOG.info("remove: - intfName {} onto vpnName {} running config-driven", + ListenableFuture configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, + writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, + writeOperTxn -> futures.add( + txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeInvTxn -> { + LOG.info("remove: - intfName {} onto vpnName {} running config-driven", + interfaceName, vpnName); + Uint64 dpId; + int ifIndex; + String gwMacAddress; + InstanceIdentifier interfaceId = + VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName); + Optional optVpnInterface; + try { + optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker, + LogicalDatastoreType.OPERATIONAL, interfaceId); + } catch (InterruptedException | ExecutionException e) { + LOG.error("remove: Failed to read data store for interface {} vpn {}", interfaceName, vpnName); - BigInteger dpId = BigInteger.ZERO; - int ifIndex = 0; - String gwMacAddress = null; - InstanceIdentifier interfaceId = - VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName); - final Optional optVpnInterface = - VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, interfaceId); - if (interfaceState != null) { - try { - dpId = InterfaceUtils.getDpIdFromInterface(interfaceState); - } catch (NumberFormatException | IllegalStateException e) { - LOG.error("remove: Unable to retrieve dpnId from interface operational" - + " data store for interface {} on dpn {} for vpn {} Fetching" - + " from vpn interface op data store. ", interfaceName, - vpnInterface.getDpnId(), vpnName, e); - dpId = BigInteger.ZERO; - } - ifIndex = interfaceState.getIfIndex(); - gwMacAddress = interfaceState.getPhysAddress().getValue(); + return; + } + if (interfaceState != null) { + try { + dpId = InterfaceUtils.getDpIdFromInterface(interfaceState); + } catch (NumberFormatException | IllegalStateException e) { + LOG.error("remove: Unable to retrieve dpnId from interface operational" + + " data store for interface {} on dpn {} for vpn {} Fetching" + + " from vpn interface op data store. ", interfaceName, + vpnInterface.getDpnId(), vpnName, e); + dpId = Uint64.ZERO; + } + ifIndex = interfaceState.getIfIndex(); + gwMacAddress = interfaceState.getPhysAddress().getValue(); + } else { + LOG.info("remove: Interface state not available for {}. Trying to fetch data" + + " from vpn interface op.", interfaceName); + if (optVpnInterface.isPresent()) { + VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get(); + dpId = vpnOpInterface.getDpnId(); + ifIndex = vpnOpInterface.getLportTag().intValue(); + gwMacAddress = vpnOpInterface.getGatewayMacAddress(); } else { - LOG.info("remove: Interface state not available for {}. Trying to fetch data" - + " from vpn interface op.", interfaceName); - if (optVpnInterface.isPresent()) { - VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get(); - dpId = vpnOpInterface.getDpnId(); - ifIndex = vpnOpInterface.getLportTag().intValue(); - gwMacAddress = vpnOpInterface.getGatewayMacAddress(); - } else { - LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped" - + " as interfaceState and vpn interface op is not" - + " available", interfaceName, vpnName); - return; - } + LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped" + + " as interfaceState and vpn interface op is not" + + " available", interfaceName, vpnName); + return; } - processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress, - optVpnInterface.isPresent() ? optVpnInterface.get() : null, false, - writeConfigTxn, writeOperTxn, writeInvTxn); - LOG.info( - "remove: Removal of vpn interface {} on dpn {} for vpn {} processed " - + "successfully", - interfaceName, vpnInterface.getDpnId(), vpnName); - })); - })); - }); + } + processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress, + optVpnInterface.isPresent() ? optVpnInterface.get() : null, false, + writeConfigTxn, writeOperTxn, writeInvTxn); + LOG.info( + "remove: Removal of vpn interface {} on dpn {} for vpn {} processed " + + "successfully", + interfaceName, vpnInterface.getDpnId(), vpnName); + }))))); futures.add(configFuture); - Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config")); + Futures.addCallback(configFuture, new PostVpnInterfaceWorker( + interfaceName, false, "Config"), MoreExecutors.directExecutor()); return futures; }, DJC_MAX_RETRIES); } - protected void processVpnInterfaceDown(BigInteger dpId, + protected void processVpnInterfaceDown(Uint64 dpId, String interfaceName, int lportTag, String gwMac, VpnInterfaceOpDataEntry vpnOpInterface, boolean isInterfaceStateDown, - WriteTransaction writeConfigTxn, - WriteTransaction writeOperTxn, - WriteTransaction writeInvTxn) { + TypedWriteTransaction writeConfigTxn, + TypedWriteTransaction writeOperTxn, + TypedReadWriteTransaction writeInvTxn) + throws ExecutionException, InterruptedException { if (vpnOpInterface == null) { LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}" + " as it is not available in operational data store", interfaceName, dpId); @@ -1288,104 +1360,160 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier( interfaceName, vpnName); if (!isInterfaceStateDown) { - final long vpnId = VpnUtil.getVpnId(dataBroker, vpnName); - VpnUtil.scheduleVpnInterfaceForRemoval(dataBroker, interfaceName, dpId, vpnName, Boolean.TRUE, - null); - final boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName); + final Uint32 vpnId = vpnUtil.getVpnId(vpnName); + vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName, null); + final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName); removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName, vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn); if (interfaceManager.isExternalInterface(interfaceName)) { - processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn, - NwConstants.DEL_FLOW); + processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag, + NwConstants.DEL_FLOW); } if (!isBgpVpnInternetVpn) { - VpnUtil.unbindService(dataBroker, interfaceName, isInterfaceStateDown, jobCoordinator); + vpnUtil.unbindService(interfaceName, isInterfaceStateDown); } LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}" + " successful", interfaceName, dpId, vpnName); } else { // Interface is retained in the DPN, but its Link Down. // Only withdraw the prefixes for this interface from BGP - withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn); + withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn); } } - private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName, - final String vpnName, final long vpnId, String gwMac, - WriteTransaction writeConfigTxn, final WriteTransaction writeOperTxn, - final WriteTransaction writeInvTxn) { + private void removeAdjacenciesFromVpn(final Uint64 dpnId, final int lportTag, final String interfaceName, + final String vpnName, final Uint32 vpnId, String gwMac, + TypedWriteTransaction writeConfigTxn, + TypedWriteTransaction writeOperTxn, + TypedReadWriteTransaction writeInvTxn) + throws ExecutionException, InterruptedException { //Read NextHops - InstanceIdentifier identifier = VpnUtil - .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName); - InstanceIdentifier path = identifier.augmentation(AdjacenciesOp.class); - Optional adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path); - - String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName); - LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}", - interfaceName, dpnId, vpnName, primaryRd); - if (adjacencies.isPresent() && !adjacencies.get().getAdjacency().isEmpty()) { - List nextHops = adjacencies.get().getAdjacency(); - LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}", - interfaceName, dpnId, vpnName, nextHops); - for (Adjacency nextHop : nextHops) { - if (nextHop.isPhysNetworkFunc()) { - LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}", - nextHop.getSubnetId().getValue(), nextHop.getIpAddress()); - fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(), - null/*writeCfgTxn*/); - } else { - String rd = nextHop.getVrfId(); - List nhList; - if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) { - nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName); + try { + InstanceIdentifier identifier = VpnUtil + .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName); + Optional vpnInterfaceOpDataEnteryOptional = + SingleTransactionDataBroker.syncReadOptional(dataBroker, + LogicalDatastoreType.OPERATIONAL, identifier); + boolean isNonPrimaryAdjIp = false; + String primaryRd = vpnUtil.getVpnRd(vpnName); + LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}", + interfaceName, dpnId, vpnName, primaryRd); + if (!vpnInterfaceOpDataEnteryOptional.isPresent()) { + LOG.error("removeAdjacenciesFromVpn: VpnInterfaceOpDataEntry-Oper DS is absent for Interface {} " + + "on vpn {} dpn {}", interfaceName, vpnName, dpnId); + return; + } + AdjacenciesOp adjacencies = vpnInterfaceOpDataEnteryOptional.get().augmentation(AdjacenciesOp.class); + + if (adjacencies != null && !adjacencies.getAdjacency().isEmpty()) { + Map nextHopsMap = adjacencies.getAdjacency(); + LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}", + interfaceName, dpnId, vpnName, nextHopsMap); + for (Adjacency nextHop : nextHopsMap.values()) { + if (nextHop.isPhysNetworkFunc()) { + LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}", + nextHop.getSubnetId().getValue(), nextHop.getIpAddress()); + fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(), + null, null/*writeCfgTxn*/); } else { - // This is a primary adjacency - nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList() - : Collections.emptyList(); - removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac, - interfaceName, writeInvTxn); - } - if (!nhList.isEmpty()) { - if (rd.equals(vpnName)) { - //this is an internal vpn - the rd is assigned to the vpn instance name; - //remove from FIB directly - nhList.forEach((nh) -> removeAdjacencyFromInternalVpn(nextHop, vpnName, - interfaceName, dpnId, writeConfigTxn)); + String rd = nextHop.getVrfId(); + List nhList; + if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) { + nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName); + isNonPrimaryAdjIp = Boolean.TRUE; } else { - removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd, interfaceName, - writeConfigTxn); + // This is a primary adjacency + nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList() + : emptyList(); + removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac, + vpnInterfaceOpDataEnteryOptional.get().getGatewayIpAddress(), + interfaceName, writeInvTxn); + isNonPrimaryAdjIp = Boolean.FALSE; + } + if (!nhList.isEmpty()) { + if (Objects.equals(primaryRd, vpnName)) { + //this is an internal vpn - the rd is assigned to the vpn instance name; + //remove from FIB directly + nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName, + interfaceName, dpnId, writeConfigTxn, writeOperTxn)); + } else { + removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd, + interfaceName, isNonPrimaryAdjIp, writeConfigTxn, writeOperTxn); + } + } else { + LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}" + + " interface {}", nextHop.getIpAddress(), rd, + nextHop.getAdjacencyType().toString(), interfaceName); + bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress()); + fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), null, writeConfigTxn); } - } else { - LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}" - + " interface {}", nextHop.getIpAddress(), rd, - nextHop.getAdjacencyType().toString(), interfaceName); - bgpManager.withdrawPrefix(rd, nextHop.getIpAddress()); - fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn); + } + String ip = nextHop.getIpAddress().split("/")[0]; + LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip); + if (vpnVipToPort != null && vpnVipToPort.getPortName().equals(interfaceName)) { + vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null); + LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry" + + " for Interface {} ip {} on dpn {} for vpn {}", + vpnVipToPort.getPortName(), ip, dpnId, vpnName); + } + // Remove the MIP-IP from VpnPortIpToPort. + if (isNonPrimaryAdjIp) { + VpnPortipToPort persistedIp = vpnUtil.getVpnPortipToPort(vpnName, ip); + if (persistedIp != null && persistedIp.isLearntIp() + && persistedIp.getPortName().equals(interfaceName)) { + VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null); + LOG.info( + "removeAdjacenciesFromVpn: Learnt-IP: {} interface {} of vpn {} removed " + + "from VpnPortipToPort", + persistedIp.getPortFixedip(), persistedIp.getPortName(), vpnName); + } + } + VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip); + if (vpnPortipToPort != null) { + VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null); + LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for " + + "Interface {} ip {} on dpn {} for vpn {}", + vpnPortipToPort.getPortName(), ip, dpnId, vpnName); } } - String ip = nextHop.getIpAddress().split("/")[0]; - LearntVpnVipToPort vpnVipToPort = VpnUtil.getLearntVpnVipToPort(dataBroker, vpnName, ip); - if (vpnVipToPort != null) { - VpnUtil.removeLearntVpnVipToPort(dataBroker, vpnName, ip); - LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed adjacency for Interface {}" - + " ip {} on dpn {} for vpn {} from VpnPortData Entry", vpnVipToPort.getPortName(), - ip, dpnId, vpnName); - } + } else { + // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS + LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies." + + " Removing it.", interfaceName, vpnName, dpnId); + writeOperTxn.delete(identifier); } - } else { - // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS - LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies." - + " Removing it.", interfaceName, vpnName, dpnId); - writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, identifier); + } catch (InterruptedException | ExecutionException e) { + LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}", + interfaceName, dpnId, vpnName); } } private Consumer removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName, - String interfaceName, BigInteger dpnId, - WriteTransaction writeConfigTxn) { + String interfaceName, Uint64 dpnId, + TypedWriteTransaction writeConfigTxn, + TypedWriteTransaction writeOperTx) { return (nh) -> { - fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh, - writeConfigTxn); + String primaryRd = vpnUtil.getVpnRd(vpnName); + String prefix = nextHop.getIpAddress(); + String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix); + LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}", + nextHop, vpnName, interfaceName, dpnId); + // FIXME: separate this out somehow? + final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnNamePrefixKey); + lock.lock(); + try { + if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName, + prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) { + //If extra-route is present behind at least one VM, then do not remove or update + //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and + //prefixtointerface DS + return; + } + fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh, + writeConfigTxn); + } finally { + lock.unlock(); + } LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}" + " nexthop {} for interface {} on dpn {} for internal vpn {}", vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName); @@ -1393,14 +1521,19 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase nhList, String vpnName, String primaryRd, - BigInteger dpnId, String rd, String interfaceName, - WriteTransaction writeConfigTxn) { + Uint64 dpnId, String rd, String interfaceName, boolean isNonPrimaryAdjIp, + TypedWriteTransaction writeConfigTxn, + TypedWriteTransaction writeOperTx) { List vpnsToImportRoute = - VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName); + vpnUtil.getVpnsImportingMyRoute(vpnName); nhList.forEach((nh) -> { //IRT: remove routes from other vpns importing it - vpnManager.removePrefixFromBGP(primaryRd, rd, vpnName, nextHop.getIpAddress(), - nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn); + if (isNonPrimaryAdjIp) { + removeLearntPrefixFromBGP(rd, nextHop.getIpAddress(), nh, writeConfigTxn); + } else { + vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(), + nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx); + } for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) { String vpnRd = vpn.getVrfId(); if (vpnRd != null) { @@ -1415,22 +1548,42 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase writeConfigTxn) { + try { + if (!fibManager.checkFibEntryExist(dataBroker, rd, prefix, nextHop)) { + LOG.info("removeLearntPrefixFromBGP: IP {} with nexthop {} rd {} is already removed.Ignoring this" + + " operation", prefix, nextHop, rd); + return; + } + LOG.info("removeLearntPrefixFromBGP: VPN WITHDRAW: Removing Fib Entry rd {} prefix {} nexthop {}", + rd, prefix, nextHop); + fibManager.removeOrUpdateFibEntry(rd, prefix, nextHop, writeConfigTxn); + bgpManager.withdrawPrefix(rd, prefix); // TODO: Might be needed to include nextHop here + LOG.info("removeLearntPrefixFromBGP: VPN WITHDRAW: Removed Fib Entry rd {} prefix {} nexthop {}", + rd, prefix, nextHop); + } catch (Exception e) { + LOG.error("removeLearntPrefixFromBGP: Delete prefix {} rd {} nextHop {} failed", prefix, rd, nextHop, e); + } + } + + private void removeGwMacAndArpResponderFlows(Adjacency nextHop, Uint32 vpnId, Uint64 dpnId, + int lportTag, String gwMac, String gwIp, String interfaceName, + TypedReadWriteTransaction writeInvTxn) + throws ExecutionException, InterruptedException { final Uuid subnetId = nextHop.getSubnetId(); if (nextHop.getSubnetGatewayMacAddress() == null) { // A valid mac-address was not available for this subnet-gateway-ip // So a connected-mac-address was used for this subnet and we need // to remove the flows for the same here from the L3_GW_MAC_TABLE. - VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName, - vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac); + vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac); } - arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(), + arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, gwIp, subnetId); } - private List getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId, + private List getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, Uint64 dpnId, String interfaceName) { // This is either an extra-route (or) a learned IP via subnet-route List nhList = null; @@ -1440,7 +1593,7 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) { - VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker, vpnName, ipAddress); + VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress); //Check if a router gateway interface is available for the subnet gw is so then use Router interface // else use connected interface if (gwPort != null && gwPort.isSubnetIp()) { @@ -1456,168 +1609,340 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase identifier, final VpnInterface original, + public void update(final InstanceIdentifier identifier, final VpnInterface original, final VpnInterface update) { - LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}" ,update.getName(), - update.getDpnId(), original.getVpnInstanceNames(), - update.getVpnInstanceNames()); + LOG.trace("Received VpnInterface update event: original={}, update={}", original, update); + LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}", update.getName(), + update.getDpnId(), original.getVpnInstanceNames(), update.getVpnInstanceNames()); + if (original.equals(update)) { + LOG.info("update: original {} update {} are same. No update required.", original, update); + return; + } final String vpnInterfaceName = update.getName(); - final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName); - final Adjacencies origAdjs = original.getAugmentation(Adjacencies.class); - final List oldAdjs = origAdjs != null && origAdjs.getAdjacency() - != null ? origAdjs.getAdjacency() : new ArrayList<>(); - final Adjacencies updateAdjs = update.getAugmentation(Adjacencies.class); - final List newAdjs = updateAdjs != null && updateAdjs.getAdjacency() - != null ? updateAdjs.getAdjacency() : new ArrayList<>(); - + final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName); LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName); //handles switching between - if (handleVpnSwapForVpnInterface(identifier, original, update)) { - LOG.info("update: handled VPNInterface {} on dpn {} update" - + "upon VPN swap from oldVpn(s) {} to newVpn(s) {}", - original.getName(), dpnId, - VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()), - VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames())); - return; - } - for (VpnInstanceNames vpnInterfaceVpnInstance : update.getVpnInstanceNames()) { - String newVpnName = vpnInterfaceVpnInstance.getVpnName(); - List copyNewAdjs = new ArrayList<>(newAdjs); - List copyOldAdjs = new ArrayList<>(oldAdjs); - String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName); - if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) { - jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> { - WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction(); - WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction(); - InstanceIdentifier vpnInterfaceOpIdentifier = - VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName); - LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven", - update.getName(), newVpnName); - //handle both addition and removal of adjacencies - //currently, new adjacency may be an extra route - boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, newVpnName); - if (!oldAdjs.equals(newAdjs)) { - for (Adjacency adj : copyNewAdjs) { - if (copyOldAdjs.contains(adj)) { - copyOldAdjs.remove(adj); - } else { - // add new adjacency - right now only extra route will hit this path - if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker, adj)) { - addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj, - dpnId, writeOperTxn, writeConfigTxn); - } - LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to vpn " - + "interface {} on vpn {} dpnId {}", - adj.getIpAddress(), adj.getNextHopIpList(), - adj.getLabel(), adj.getSubnetId(), update.getName(), - newVpnName, dpnId); - } - } - for (Adjacency adj : copyOldAdjs) { - if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker, adj)) { - delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, - writeOperTxn, writeConfigTxn); - } - LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from" - + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(), - adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName); + jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> { + List> futures = new ArrayList<>(); + if (handleVpnInstanceUpdateForVpnInterface(identifier, original, update, futures)) { + LOG.info("update: handled Instance update for VPNInterface {} on dpn {} from oldVpn(s) {} " + + "to newVpn(s) {}", + original.getName(), dpnId, + VpnHelper.getVpnInterfaceVpnInstanceNamesString( + new ArrayList(original.getVpnInstanceNames().values())), + VpnHelper.getVpnInterfaceVpnInstanceNamesString( + new ArrayList(update.getVpnInstanceNames().values()))); + return emptyList(); + } + updateVpnInstanceAdjChange(original, update, vpnInterfaceName, futures); + return futures; + }); + } + + private boolean handleVpnInstanceUpdateForVpnInterface(InstanceIdentifier identifier, + VpnInterface original, VpnInterface update, + List> futures) { + boolean isVpnInstanceUpdate = false; + final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class); + final String interfaceName = key.getName(); + List oldVpnList = VpnUtil.getVpnListForVpnInterface(original); + List oldVpnListCopy = new ArrayList<>(); + oldVpnListCopy.addAll(oldVpnList); + List newVpnList = VpnUtil.getVpnListForVpnInterface(update); + List newVpnListCopy = new ArrayList<>(); + newVpnListCopy.addAll(newVpnList); + + oldVpnList.removeAll(newVpnList); + newVpnList.removeAll(oldVpnListCopy); + //This block will execute only on if there is a change in the VPN Instance. + if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) { + /* + * Internet BGP-VPN Instance update with single router: + * ==================================================== + * In this case single VPN Interface will be part of maximum 2 VPN Instance only. + * 1st VPN Instance : router VPN or external BGP-VPN. + * 2nd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access. + * + * VPN Instance UPDATE: + * oldVpnList = 0 and newVpnList = 1 (Internet BGP-VPN) + * oldVpnList = 1 and newVpnList = 0 (Internet BGP-VPN) + * + * External BGP-VPN Instance update with single router: + * ==================================================== + * In this case single VPN interface will be part of maximum 1 VPN Instance only. + * + * Updated VPN Instance will be always either internal router VPN to + * external BGP-VPN or external BGP-VPN to internal router VPN swap. + * + * VPN Instance UPDATE: + * oldVpnList = 1 and newVpnList = 1 (router VPN to Ext-BGPVPN) + * oldVpnList = 1 and newVpnList = 1 (Ext-BGPVPN to router VPN) + * + * Dual Router VPN Instance Update: + * ================================ + * In this case single VPN interface will be part of maximum 3 VPN Instance only. + * + * 1st VPN Instance : router VPN or external BGP-VPN-1. + * 2nd VPN Instance : router VPN or external BGP-VPN-2. + * 3rd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access. + * + * Dual Router --> Associated with common external BGP-VPN Instance. + * 1st router and 2nd router are getting associated with single External BGP-VPN + * 1) add 1st router to external bgpvpn --> oldVpnList=1, newVpnList=1; + * 2) add 2nd router to the same external bgpvpn --> oldVpnList=1, newVpnList=0 + * In this case, we need to call removeVpnInterfaceCall() followed by addVpnInterfaceCall() + * + * + */ + isVpnInstanceUpdate = true; + if (VpnUtil.isDualRouterVpnUpdate(oldVpnListCopy, newVpnListCopy)) { + if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3) + && oldVpnList.size() == 1 && newVpnList.isEmpty()) { + //Identify the external BGP-VPN Instance and pass that value as newVpnList + List externalBgpVpnList = new ArrayList<>(); + for (String newVpnName : newVpnListCopy) { + String primaryRd = vpnUtil.getPrimaryRd(newVpnName); + VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd); + if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry + .BgpvpnType.BGPVPNExternal) { + externalBgpVpnList.add(newVpnName); + break; } } - ListenableFuture operFuture = writeOperTxn.submit(); - try { - operFuture.get(); - } catch (ExecutionException e) { - LOG.error("Exception encountered while submitting operational future for update" - + " VpnInterface {} on vpn {}: {}", vpnInterfaceName, newVpnName, e); - return null; + //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall() + updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, + externalBgpVpnList, oldVpnListCopy, futures); + + } else if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3) + && oldVpnList.isEmpty() && newVpnList.size() == 1) { + //Identify the router VPN Instance and pass that value as oldVpnList + List routerVpnList = new ArrayList<>(); + for (String newVpnName : newVpnListCopy) { + String primaryRd = vpnUtil.getPrimaryRd(newVpnName); + VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd); + if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry + .BgpvpnType.VPN) { + routerVpnList.add(newVpnName); + break; + } } - List> futures = new ArrayList<>(); - futures.add(writeConfigTxn.submit()); - LOG.info("update: vpn interface updated for interface {} oldVpn(s) {} newVpn {}" - + "processed successfully", update.getName(), - VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()), newVpnName); - return futures; - }); + //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall() + updateVpnInstanceChange(identifier, interfaceName, original, update, routerVpnList, + newVpnList, oldVpnListCopy, futures); + + } else { + //Handle remaining use cases. + updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList, + oldVpnListCopy, futures); + } } else { - LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}" - + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd); + updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList, + oldVpnListCopy, futures); } } + return isVpnInstanceUpdate; } - private boolean handleVpnSwapForVpnInterface(InstanceIdentifier identifier, - VpnInterface original, VpnInterface update) { - boolean isSwap = Boolean.FALSE; - final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class); - final String interfaceName = key.getName(); - List oldVpnList = original.getVpnInstanceNames().stream() - .map(VpnInstanceNames::getVpnName).collect(Collectors.toList()); - List oldVpnListCopy = new ArrayList<>(); - oldVpnListCopy.addAll(oldVpnList); - List newVpnList = update.getVpnInstanceNames().stream() - .map(VpnInstanceNames::getVpnName).collect(Collectors.toList()); - oldVpnList.removeAll(newVpnList); - for (String oldVpnName: oldVpnList) { - isSwap = Boolean.TRUE; - LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} remove vpnName {} running" - + " config-driven swap removal", interfaceName, oldVpnName); + private void updateVpnInstanceChange(InstanceIdentifier identifier, String interfaceName, + VpnInterface original, VpnInterface update, List oldVpnList, + List newVpnList, List oldVpnListCopy, + List> futures) { + final Adjacencies origAdjs = original.augmentation(Adjacencies.class); + final List oldAdjs = origAdjs != null && origAdjs.getAdjacency() != null + ? new ArrayList(origAdjs.getAdjacency().values()) : new ArrayList<>(); + final Adjacencies updateAdjs = update.augmentation(Adjacencies.class); + final List newAdjs = updateAdjs != null && updateAdjs.getAdjacency() != null + ? new ArrayList(updateAdjs.getAdjacency().values()) : new ArrayList<>(); + + boolean isOldVpnRemoveCallExecuted = false; + for (String oldVpnName : oldVpnList) { + LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} " + + "remove from vpnName {} ", interfaceName, oldVpnName); removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName); - LOG.info("handleVpnSwapForVpnInterface: Processed Remove for update on VPNInterface {} upon VPN swap" - + "from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName, newVpnList); + LOG.info("updateVpnInstanceChange: Processed Remove for update on VPNInterface" + + " {} upon VPN update from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName, + newVpnList); + isOldVpnRemoveCallExecuted = true; } //Wait for previous interface bindings to be removed - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - //Ignore + if (isOldVpnRemoveCallExecuted && !newVpnList.isEmpty()) { + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + LOG.error("updateVpnInstanceChange: InterruptedException caught for interface {}", interfaceName, e); + } } - newVpnList.removeAll(oldVpnListCopy); - for (String newVpnName: newVpnList) { - String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName); - isSwap = Boolean.TRUE; - if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) { - LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} onto vpnName {}" - + "running config-driven swap addition", interfaceName, newVpnName); - final Adjacencies origAdjs = original.getAugmentation(Adjacencies.class); - final List oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null) - ? origAdjs.getAdjacency() : new ArrayList<>(); - final Adjacencies updateAdjs = update.getAugmentation(Adjacencies.class); - final List newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null) - ? updateAdjs.getAdjacency() : new ArrayList<>(); - + for (String newVpnName : newVpnList) { + String primaryRd = vpnUtil.getPrimaryRd(newVpnName); + if (!vpnUtil.isVpnPendingDelete(primaryRd)) { + LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} " + + "onto vpnName {} ", interfaceName, newVpnName); addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName); - LOG.info("handleVpnSwapForVpnInterface: Processed Add for update on VPNInterface {}" - + "from oldVpn(s) {} to newVpn {} upon VPN swap", - interfaceName, oldVpnListCopy, newVpnName); + LOG.info("updateVpnInstanceChange: Processed Add for update on VPNInterface {}" + + "from oldVpn(s) {} to newVpn {} ", + interfaceName, oldVpnListCopy, newVpnName); + /* This block will execute only if V6 subnet is associated with internet BGP-VPN. + * Use Case: + * In Dual stack network, first V4 subnet only attached to router and router is associated + * with internet BGP-VPN(router-gw). At this point VPN interface is having only router vpn instance. + * Later V6 subnet is added to router, at this point existing VPN interface will get updated + * with Internet BGP-VPN instance(Note: Internet BGP-VPN Instance update in vpn interface + * is applicable for only on V6 subnet is added to router). newVpnList = Contains only Internet + * BGP-VPN Instance. So we required V6 primary adjacency info needs to be populated onto + * router VPN as well as Internet BGP-VPN. + * + * addVpnInterfaceCall() --> It will create V6 Adj onto Internet BGP-VPN only. + * updateVpnInstanceAdjChange() --> This method call is needed for second primary V6 Adj + * update in existing router VPN instance. + */ + if (vpnUtil.isBgpVpnInternet(newVpnName)) { + LOG.info("updateVpnInstanceChange: VPN Interface {} with new Adjacency {} in existing " + + "VPN instance {}", interfaceName, newAdjs, original.getVpnInstanceNames()); + updateVpnInstanceAdjChange(original, update, interfaceName, futures); + } + } else { + LOG.info("updateVpnInstanceChange: failed to Add for update on VPNInterface {} from oldVpn(s) {} to " + + "newVpn {} as the new vpn does not exist in oper DS or it is in PENDING_DELETE state", + interfaceName, oldVpnListCopy, newVpnName); + } + } + } + + // TODO Clean up the exception handling + @SuppressWarnings("checkstyle:IllegalCatch") + private List> updateVpnInstanceAdjChange(VpnInterface original, VpnInterface update, + String vpnInterfaceName, + List> futures) { + final Adjacencies origAdjs = original.augmentation(Adjacencies.class); + final List oldAdjs = origAdjs != null && origAdjs.getAdjacency() + != null ? new ArrayList(origAdjs.getAdjacency().values()) : new ArrayList<>(); + final Adjacencies updateAdjs = update.augmentation(Adjacencies.class); + final List newAdjs = updateAdjs != null && updateAdjs.getAdjacency() + != null ? new ArrayList(updateAdjs.getAdjacency().values()) : new ArrayList<>(); + + final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName); + for (VpnInstanceNames vpnInterfaceVpnInstance : update.nonnullVpnInstanceNames().values()) { + String newVpnName = vpnInterfaceVpnInstance.getVpnName(); + List copyNewAdjs = new ArrayList<>(newAdjs); + List copyOldAdjs = new ArrayList<>(oldAdjs); + String primaryRd = vpnUtil.getPrimaryRd(newVpnName); + if (!vpnUtil.isVpnPendingDelete(primaryRd)) { + // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in + //set of prefix used as entry in prefix-to-interface datastore + // is prerequisite for refresh Fib to avoid race condition leading to missing remote next hop + // in bucket actions on bgp-vpn delete + Set prefixListForRefreshFib = new HashSet<>(); + ListenableFuture configTxFuture = txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, + confTx -> futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, + operTx -> { + InstanceIdentifier vpnInterfaceOpIdentifier = + VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName); + LOG.info("VPN Interface update event-intfName {} onto vpnName {} running config-driven", + update.getName(), newVpnName); + //handle both addition and removal of adjacencies + // currently, new adjacency may be an extra route + boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName); + if (!oldAdjs.equals(newAdjs)) { + for (Adjacency adj : copyNewAdjs) { + if (copyOldAdjs.contains(adj)) { + copyOldAdjs.remove(adj); + } else { + // add new adjacency + if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) { + try { + addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj, + dpnId, operTx, confTx, confTx, prefixListForRefreshFib); + } catch (RuntimeException e) { + LOG.error("Failed to add adjacency {} to vpn interface {} with" + + " dpnId {}", adj, vpnInterfaceName, dpnId, e); + } + } + LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} " + + " added to vpn interface {} on vpn {} dpnId {}", + adj.getIpAddress(), adj.getNextHopIpList(), adj.getLabel(), + adj.getSubnetId(), update.getName(), newVpnName, dpnId); + } + } + for (Adjacency adj : copyOldAdjs) { + if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) { + if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency + && !adj.isPhysNetworkFunc()) { + delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, operTx, + confTx); + //remove FIB entry + String vpnRd = vpnUtil.getVpnRd(newVpnName); + LOG.debug("update: remove prefix {} from the FIB and BGP entry " + + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd); + //remove BGP entry + fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), null, confTx); + if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) { + bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress()); + } + } else { + delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, + operTx, confTx); + } + } + LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from" + + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(), + adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName); + } + } + }))); + Futures.addCallback(configTxFuture, new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib), + MoreExecutors.directExecutor()); + futures.add(configTxFuture); + for (ListenableFuture future : futures) { + ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}", + update.getName(), update.getVpnInstanceNames()); + } + } else { + LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}" + + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd); } } - return isSwap; + return futures; } - private void updateLabelMapper(Long label, List nextHopIpList) { - Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!"); - synchronized (label.toString().intern()) { + private void updateLabelMapper(Uint32 label, List nextHopIpList) { + final String labelStr = Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!") + .toString(); + // FIXME: separate this out somehow? + final ReentrantLock lock = JvmGlobalLocks.getLockForString(labelStr); + lock.lock(); + try { InstanceIdentifier lriIid = InstanceIdentifier.builder(LabelRouteMap.class) .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build(); - Optional opResult = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid); + Optional opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker, + LogicalDatastoreType.OPERATIONAL, lriIid); if (opResult.isPresent()) { LabelRouteInfo labelRouteInfo = - new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build(); - MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid, labelRouteInfo); + new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build(); + SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid, + labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY); } + LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label, + nextHopIpList); + } catch (InterruptedException | ExecutionException e) { + LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label, + nextHopIpList); + } catch (TransactionCommitFailedException e) { + LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label, + nextHopIpList); + } finally { + lock.unlock(); } - LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label, nextHopIpList); } - public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label, - SubnetRoute route, WriteTransaction writeConfigTxn) { + public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, Uint32 label, + SubnetRoute route, String parentVpnRd, TypedWriteTransaction writeConfigTxn) { RouteOrigin origin = RouteOrigin.SELF_IMPORTED; - VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, null /* parentVpnRd */) + VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd) .addAugmentation(SubnetRoute.class, route).build(); List vrfEntryList = Collections.singletonList(vrfEntry); InstanceIdentifierBuilder idBuilder = @@ -1625,195 +1950,231 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase vrfTableId = idBuilder.build(); VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build(); if (writeConfigTxn != null) { - writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true); + writeConfigTxn.mergeParentStructureMerge(vrfTableId, vrfTableNew); } else { - VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew); + vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew); } LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}" + " and elantag {}", rd, prefix, nextHop, label, route.getElantag()); } protected void addNewAdjToVpnInterface(InstanceIdentifier identifier, String primaryRd, - Adjacency adj, BigInteger dpnId, WriteTransaction writeOperTxn, - WriteTransaction writeConfigTxn) { - - Optional optVpnInterface = VpnUtil.read(dataBroker, - LogicalDatastoreType.OPERATIONAL, identifier); - - if (optVpnInterface.isPresent()) { - VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get(); - String prefix = VpnUtil.getIpPrefix(adj.getIpAddress()); - String vpnName = currVpnIntf.getVpnInstanceName(); - VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd); - InstanceIdentifier adjPath = identifier.augmentation(AdjacenciesOp.class); - Optional optAdjacencies = VpnUtil.read(dataBroker, - LogicalDatastoreType.OPERATIONAL, adjPath); - boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni()); - VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan); - long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L : vpnInstanceOpData.getL3vni(); - VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType); - List adjacencies; - if (optAdjacencies.isPresent()) { - adjacencies = optAdjacencies.get().getAdjacency(); - } else { - // This code will be hit in case of first PNF adjacency - adjacencies = new ArrayList<>(); - } - long vpnId = VpnUtil.getVpnId(dataBroker, vpnName); - L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName) - .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd); - Adjacency operationalAdjacency = null; - if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()) { - RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL - : RouteOrigin.STATIC; - String nh = adj.getNextHopIpList().get(0); - String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix); - synchronized (vpnPrefixKey.intern()) { - java.util.Optional rdToAllocate = VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap( - dataBroker, vpnId, null, prefix, vpnName, nh, dpnId); - if (rdToAllocate.isPresent()) { - input.setRd(rdToAllocate.get()); - operationalAdjacency = populator.createOperationalAdjacency(input); - int label = operationalAdjacency.getLabel().intValue(); - vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(), - currVpnIntf.getVpnInstanceName(), l3vni, origin, - currVpnIntf.getName(), operationalAdjacency, encapType, writeConfigTxn); - LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}" - + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(), - vpnName, label, currVpnIntf.getName(), dpnId); - } else { - LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}", vpnName, - prefix); - return; - } - // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN. - // Keeping the MPLS check for now. - if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) { - final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build(); - List vpnsToImportRoute = - VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName); - vpnsToImportRoute.forEach(vpn -> { - if (vpn.getVrfId() != null) { - VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap( - dataBroker, vpn.getVpnId(), vpnId, prefix, - VpnUtil.getVpnName(dataBroker, vpn.getVpnId()), nh, dpnId) - .ifPresent( - rds -> vpnManager.addExtraRoute( - VpnUtil.getVpnName(dataBroker, vpn.getVpnId()), - adj.getIpAddress(), nh, rds, - currVpnIntf.getVpnInstanceName(), - l3vni, RouteOrigin.SELF_IMPORTED, - currVpnIntf.getName(), opAdjacency, encapType, writeConfigTxn)); - } - }); + Adjacency adj, Uint64 dpnId, + TypedWriteTransaction writeOperTxn, + TypedWriteTransaction writeConfigTxn, + TypedReadWriteTransaction writeInvTxn, + Set prefixListForRefreshFib) + throws ExecutionException, InterruptedException { + String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName(); + String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName(); + try { + Optional optVpnInterface = SingleTransactionDataBroker + .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier); + if (optVpnInterface.isPresent()) { + VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get(); + String prefix = VpnUtil.getIpPrefix(adj.getIpAddress()); + String vpnName = currVpnIntf.getVpnInstanceName(); + VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd); + InstanceIdentifier adjPath = identifier.augmentation(AdjacenciesOp.class); + Optional optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, + LogicalDatastoreType.OPERATIONAL, adjPath); + boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni()); + VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan); + Uint32 l3vni = vpnInstanceOpData.getL3vni() == null ? Uint32.ZERO : vpnInstanceOpData.getL3vni(); + VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType); + List adjacencies = new ArrayList<>(); + if (optAdjacencies.isPresent() && optAdjacencies.get().getAdjacency() != null) { + adjacencies.addAll(optAdjacencies.get().getAdjacency().values()); + } + Uint32 vpnId = vpnUtil.getVpnId(vpnName); + L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName) + .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd); + Adjacency operationalAdjacency = null; + //Handling dual stack neutron port primary adjacency + if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) { + LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix, + currVpnIntf.getName(), vpnName); + Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, + currVpnIntf.getName()); + if (interfaceState != null) { + processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd, + currVpnIntf.getName(), vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, + prefixListForRefreshFib); } } - } else if (adj.isPhysNetworkFunc()) { // PNF adjacency. - LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix, - currVpnIntf.getName(), vpnName); + if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty() + && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) { + RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.LearntIp ? RouteOrigin.DYNAMIC + : RouteOrigin.STATIC; + String nh = adj.getNextHopIpList().get(0); + String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix); + // FIXME: separate out to somehow? + final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey); + lock.lock(); + try { + java.util.Optional rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap( + vpnId, null, prefix, vpnName, nh, dpnId); + if (rdToAllocate.isPresent()) { + input.setRd(rdToAllocate.get()); + operationalAdjacency = populator.createOperationalAdjacency(input); + int label = operationalAdjacency.getLabel().intValue(); + vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(), + currVpnIntf.getVpnInstanceName(), l3vni, origin, + currVpnIntf.getName(), operationalAdjacency, encapType, + prefixListForRefreshFib, writeConfigTxn); + LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}" + + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(), + vpnName, label, currVpnIntf.getName(), dpnId); + } else { + LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}", + vpnName, prefix); + return; + } + // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN. + // Keeping the MPLS check for now. + if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) { + final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build(); + List vpnsToImportRoute = + vpnUtil.getVpnsImportingMyRoute(vpnName); + vpnsToImportRoute.forEach(vpn -> { + if (vpn.getVrfId() != null) { + vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(), + vpnId, prefix, + vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId) + .ifPresent( + rds -> vpnManager.addExtraRoute( + vpnUtil.getVpnName(vpn.getVpnId()), adj.getIpAddress(), + nh, rds, currVpnIntf.getVpnInstanceName(), l3vni, + RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(), opAdjacency, + encapType, prefixListForRefreshFib, writeConfigTxn)); + } + }); + } + } finally { + lock.unlock(); + } + } else if (adj.isPhysNetworkFunc()) { // PNF adjacency. + LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix, + currVpnIntf.getName(), vpnName); + + InstanceIdentifier vpnIfaceConfigidentifier = VpnUtil + .getVpnInterfaceIdentifier(currVpnIntf.getName()); + Optional vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker, + LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier); + Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix, + Prefixes.PrefixCue.PhysNetFunc); + if (vpnIntefaceConfig.isPresent()) { + pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix, + vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(), + vpnIntefaceConfig.get().getSegmentationId().toJava(), Prefixes.PrefixCue.PhysNetFunc); + } - String parentVpnRd = getParentVpnRdForExternalSubnet(adj); + String parentVpnRd = getParentVpnRdForExternalSubnet(adj); - writeOperTxn.merge( - LogicalDatastoreType.OPERATIONAL, - VpnUtil.getPrefixToInterfaceIdentifier(VpnUtil.getVpnId(dataBroker, - adj.getSubnetId().getValue()), prefix), - VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix, - adj.getSubnetId(), Prefixes.PrefixCue.PhysNetFunc), true); + writeOperTxn.mergeParentStructureMerge( + VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()), + prefix), pnfPrefix); - fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(), - adj.getIpAddress(), Collections.emptyList(), null /* EncapType */, 0 /* label */, 0 /*l3vni*/, - null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn); + fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(), + adj.getIpAddress(), emptyList(), null /* EncapType */, Uint32.ZERO /* label */, + Uint32.ZERO /*l3vni*/, null /* gw-mac */, parentVpnRd, + RouteOrigin.LOCAL, writeConfigTxn); - input.setRd(adj.getVrfId()); - } - if (operationalAdjacency == null) { - operationalAdjacency = populator.createOperationalAdjacency(input); + input.setRd(adj.getVrfId()); + } + if (operationalAdjacency == null) { + operationalAdjacency = populator.createOperationalAdjacency(input); + } + adjacencies.add(operationalAdjacency); + AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies); + VpnInterfaceOpDataEntry newVpnIntf = + VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(), + aug, dpnId, currVpnIntf.getLportTag().toJava(), + currVpnIntf.getGatewayMacAddress(), currVpnIntf.getGatewayIpAddress()); + writeOperTxn.mergeParentStructureMerge(identifier, newVpnIntf); } - adjacencies.add(operationalAdjacency); - AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies); - VpnInterfaceOpDataEntry newVpnIntf = - VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(), - aug, dpnId, currVpnIntf.isScheduledForRemove(), currVpnIntf.getLportTag(), - currVpnIntf.getGatewayMacAddress()); - - writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true); + } catch (InterruptedException | ExecutionException e) { + LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip " + + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress()); } } + @Nullable private String getParentVpnRdForExternalSubnet(Adjacency adj) { - Subnets subnets = VpnUtil.getExternalSubnet(dataBroker, adj.getSubnetId()); + Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId()); return subnets != null ? subnets.getExternalNetworkId().getValue() : null; } protected void delAdjFromVpnInterface(InstanceIdentifier identifier, Adjacency adj, - BigInteger dpnId, WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) { - Optional optVpnInterface = VpnUtil.read(dataBroker, - LogicalDatastoreType.OPERATIONAL, identifier); - - if (optVpnInterface.isPresent()) { - VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get(); - - InstanceIdentifier path = identifier.augmentation(AdjacenciesOp.class); - Optional optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path); - if (optAdjacencies.isPresent()) { - List adjacencies = optAdjacencies.get().getAdjacency(); - - if (!adjacencies.isEmpty()) { - LOG.trace("delAdjFromVpnInterface: Adjacencies are " + adjacencies); - Iterator adjIt = adjacencies.iterator(); - while (adjIt.hasNext()) { - Adjacency adjElem = adjIt.next(); - if (adjElem.getIpAddress().equals(adj.getIpAddress())) { - String rd = adjElem.getVrfId(); - adjIt.remove(); - - AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies); - VpnInterfaceOpDataEntry newVpnIntf = VpnUtil - .getVpnInterfaceOpDataEntry(currVpnIntf.getName(), - currVpnIntf.getVpnInstanceName(), aug, dpnId, currVpnIntf.isScheduledForRemove(), - currVpnIntf.getLportTag(), currVpnIntf.getGatewayMacAddress()); - - writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true); - if (adj.getNextHopIpList() != null) { - for (String nh : adj.getNextHopIpList()) { - deleteExtraRouteFromCurrentAndImportingVpns( - currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd, - currVpnIntf.getName(), writeConfigTxn); + Uint64 dpnId, TypedWriteTransaction writeOperTxn, + TypedWriteTransaction writeConfigTxn) { + String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName(); + String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName(); + try { + Optional optVpnInterface = SingleTransactionDataBroker.syncReadOptional( + dataBroker, LogicalDatastoreType.OPERATIONAL, identifier); + if (optVpnInterface.isPresent()) { + VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get(); + InstanceIdentifier path = identifier.augmentation(AdjacenciesOp.class); + Optional optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, + LogicalDatastoreType.OPERATIONAL, path); + if (optAdjacencies.isPresent()) { + Map keyAdjacencyMap = optAdjacencies.get().getAdjacency(); + + if (keyAdjacencyMap != null && !keyAdjacencyMap.isEmpty()) { + LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", keyAdjacencyMap); + for (Adjacency adjacency : keyAdjacencyMap.values()) { + if (Objects.equals(adjacency.getIpAddress(), adj.getIpAddress())) { + String rd = adjacency.getVrfId(); + if (adj.getNextHopIpList() != null) { + for (String nh : adj.getNextHopIpList()) { + deleteExtraRouteFromCurrentAndImportingVpns( + currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd, + currVpnIntf.getName(), writeConfigTxn, writeOperTxn); + } + } else if (adj.isPhysNetworkFunc()) { + LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}", + adj.getIpAddress(), adj.getSubnetId()); + fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(), + null, writeConfigTxn); } - } else if (adj.isPhysNetworkFunc()) { - LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet [}", - adj.getIpAddress(), adj.getSubnetId()); - fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(), - writeConfigTxn); + break; } - break; - } + } } + LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(), + dpnId, adj.getVrfId()); + } else { + LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is " + + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId()); } - LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(), - dpnId, adj.getVrfId()); - } else { - LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is " - + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId()); } + } catch (InterruptedException | ExecutionException e) { + LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}", + adj.getIpAddress(), interfaceName, dpnId, vpnName); } } private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop, - String rd, String intfName, WriteTransaction writeConfigTxn) { - vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn); - List vpnsToImportRoute = VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName); + String rd, String intfName, TypedWriteTransaction writeConfigTxn, + TypedWriteTransaction writeOperTx) { + LOG.info("removing extra-route {} for nexthop {} in VPN {} intfName {} rd {}", + destination, nextHop, vpnName, intfName, rd); + vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx); + List vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName); for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) { String vpnRd = vpn.getVrfId(); if (vpnRd != null) { - vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn); + LOG.info("deleting extra-route {} for nexthop {} in VPN {} intfName {} vpnRd {}", + destination, nextHop, vpnName, intfName, vpnRd); + vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn, + writeOperTx); } } } - InstanceIdentifier getRouterDpnId(String routerName, BigInteger dpnId) { + InstanceIdentifier getRouterDpnId(String routerName, Uint64 dpnId) { return InstanceIdentifier.builder(NeutronRouterDpns.class) .child(RouterDpnList.class, new RouterDpnListKey(routerName)) .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build(); @@ -1824,111 +2185,12 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase routerDpnListIdentifier = getRouterDpnId(routerName, dpId); - - Optional optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType - .OPERATIONAL, routerDpnListIdentifier); - RouterInterfaces routerInterface = - new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface( - vpnInterfaceName).build(); - if (optionalRouterDpnList.isPresent()) { - writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child( - RouterInterfaces.class, new RouterInterfacesKey(vpnInterfaceName)), routerInterface, true); - } else { - RouterDpnListBuilder builder = new RouterDpnListBuilder(); - builder.setRouterId(routerName); - DpnVpninterfacesListBuilder dpnVpnList = new DpnVpninterfacesListBuilder().setDpnId(dpId); - builder.setDpnVpninterfacesList(Collections.singletonList(dpnVpnList.build())); - writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, - getRouterId(routerName), - builder.build(), true); - } - } - - protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, - WriteTransaction writeOperTxn) { - BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName); - if (dpId.equals(BigInteger.ZERO)) { - LOG.error("removeFromNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}" - + " dissociation model", vpnInterfaceName, routerName); - - return; - } - InstanceIdentifier routerDpnListIdentifier = getRouterDpnId(routerName, dpId); - Optional optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType - .OPERATIONAL, routerDpnListIdentifier); - if (optionalRouterDpnList.isPresent()) { - List routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces(); - RouterInterfaces routerInterface = - new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface( - vpnInterfaceName).build(); - - if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) { - if (routerInterfaces.isEmpty()) { - if (writeOperTxn != null) { - writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier); - } else { - MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier); - } - } else { - if (writeOperTxn != null) { - writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child( - RouterInterfaces.class, - new RouterInterfacesKey(vpnInterfaceName))); - } else { - MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, - routerDpnListIdentifier.child( - RouterInterfaces.class, - new RouterInterfacesKey(vpnInterfaceName))); - } - } - } - } - } - - protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, BigInteger dpId, - WriteTransaction writeOperTxn) { - if (dpId.equals(BigInteger.ZERO)) { - LOG.error("removeFromNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}" - + " dissociation model", vpnInterfaceName, routerName); - - return; - } - InstanceIdentifier routerDpnListIdentifier = getRouterDpnId(routerName, dpId); - Optional optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType - .OPERATIONAL, routerDpnListIdentifier); - if (optionalRouterDpnList.isPresent()) { - List routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces(); - RouterInterfaces routerInterface = - new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface( - vpnInterfaceName).build(); - if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) { - if (routerInterfaces.isEmpty()) { - writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier); - } else { - writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child( - RouterInterfaces.class, - new RouterInterfacesKey(vpnInterfaceName))); - } - } - } - } - protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName, - WriteTransaction writeConfigTxn, String vpnName) { + TypedWriteTransaction writeConfigTxn, String vpnName) { if (vpnInterface == null) { return; } - List adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName); + List adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName); if (adjs == null) { LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for" + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName); @@ -1940,9 +2202,15 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase writeConfigTxn, String vpnName) { + Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class); + String rd = vpnUtil.getVpnRd(vpnName); if (adjs != null) { - List adjsList = adjs.getAdjacency(); - for (Adjacency adj : adjsList) { + Map keyAdjacencyMap = adjs.nonnullAdjacency(); + for (Adjacency adj : keyAdjacencyMap.values()) { if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) { String primaryInterfaceIp = adj.getIpAddress(); String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp); - fibManager.removeFibEntry(rd, prefix, writeConfigTxn); + fibManager.removeFibEntry(rd, prefix, null, writeConfigTxn); LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}" + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix); - return; } } } else { @@ -1981,19 +2248,12 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase vpnInterfaceOpIdentifier = VpnUtil .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName); addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null, intefaceData.identifier, vpnName); - return; } private void addToUnprocessedVpnInterfaces(InstanceIdentifier identifier, @@ -2010,17 +2270,20 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase vpnInterfaces = unprocessedVpnInterfaces.get(vpnInstanceName); if (vpnInterfaces != null) { @@ -2038,24 +2301,30 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase identifier, VpnInterface vpnInterface) { - synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) { + // FIXME: use VpnInstanceNamesKey perhaps? What about nulls? + final String firstVpnName = VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface); + final ReentrantLock lock = JvmGlobalLocks.getLockForString(firstVpnName); + lock.lock(); + try { ConcurrentLinkedQueue vpnInterfaces = - unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface)); + unprocessedVpnInterfaces.get(firstVpnName); if (vpnInterfaces != null) { if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) { LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from " - + "unprocessed list", vpnInterface.getName(), - VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface)); + + "unprocessed list", vpnInterface.getName(), firstVpnName); } } else { - LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}", - VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface)); + LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}", firstVpnName); } + } finally { + lock.unlock(); } } @@ -2116,66 +2385,83 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList(); + List vpnToDpnLists = new ArrayList(vpnInstanceOpData.getVpnToDpnList().values()); if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) { return; } LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName); - vpnToDpnLists.forEach(vpnToDpnList -> vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> { - InstanceIdentifier existingVpnInterfaceId = - VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName); - Optional vpnInterfaceOptional = VpnUtil.read(dataBroker, - LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId); - if (!vpnInterfaceOptional.isPresent()) { - return; - } - List configVpnAdjacencies = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, - vpnInterface.getInterfaceName()); - if (configVpnAdjacencies == null) { - LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface); + vpnToDpnLists.forEach(vpnToDpnList -> { + if (vpnToDpnList.getVpnInterfaces() == null) { return; } - List operationVpnAdjacencies = vpnInterfaceOptional.get() - .getAugmentation(AdjacenciesOp.class).getAdjacency(); - // Due to insufficient rds, some of the extra route wont get processed when it is added. - // The unprocessed adjacencies will be present in config vpn interface DS but will be missing - // in operational DS. These unprocessed adjacencies will be handled below. - // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in operational - // DS are retrieved which is used to call addNewAdjToVpnInterface method. - configVpnAdjacencies.stream() - .filter(adjacency -> operationVpnAdjacencies.stream() - .noneMatch(operationalAdjacency -> - operationalAdjacency.getIpAddress().equals(adjacency.getIpAddress()))) - .forEach(adjacency -> { - LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency); - jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(), - () -> { - WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction(); - WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction(); - if (VpnUtil.isAdjacencyEligibleToVpn(dataBroker, adjacency, vpnName)) { - addNewAdjToVpnInterface(existingVpnInterfaceId, primaryRd, adjacency, - vpnInterfaceOptional.get().getDpnId(), writeConfigTxn, writeOperTxn); - ListenableFuture operFuture = writeOperTxn.submit(); - try { - operFuture.get(); - } catch (ExecutionException | InterruptedException e) { - LOG.error("Exception encountered while submitting operational" - + " future for vpnInterface {}", vpnInterface, e); + vpnToDpnList.getVpnInterfaces().values().forEach(vpnInterface -> { + try { + InstanceIdentifier existingVpnInterfaceId = + VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName); + Optional vpnInterfaceOptional = SingleTransactionDataBroker + .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId); + if (!vpnInterfaceOptional.isPresent()) { + return; + } + List configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig( + vpnInterface.getInterfaceName()); + if (configVpnAdjacencies == null) { + LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface); + return; + } + List operationVpnAdjacencies = new ArrayList(vpnInterfaceOptional.get() + .augmentation(AdjacenciesOp.class).nonnullAdjacency().values()); + // Due to insufficient rds, some of the extra route wont get processed when it is added. + // The unprocessed adjacencies will be present in config vpn interface DS but will be missing + // in operational DS. These unprocessed adjacencies will be handled below. + // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in + // operational DS are retrieved which is used to call addNewAdjToVpnInterface method. + configVpnAdjacencies.stream() + .filter(adjacency -> operationVpnAdjacencies.stream() + .noneMatch(operationalAdjacency -> + Objects.equals(operationalAdjacency.getIpAddress(), adjacency.getIpAddress()))) + .forEach(adjacency -> { + LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency); + jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(), + () -> { + // TODO Deal with sequencing — the config tx must only submitted + // if the oper tx goes in + if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) { + List> futures = new ArrayList<>(); + futures.add( + txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> { + //set of prefix used, as entry in prefix-to-interface datastore + // is prerequisite for refresh Fib to avoid race condition leading + // to missing remote next hop in bucket actions on bgp-vpn delete + Set prefixListForRefreshFib = new HashSet<>(); + ListenableFuture configTxFuture = + txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, + confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId, + primaryRd, adjacency, + vpnInterfaceOptional.get().getDpnId(), + operTx, confTx, confTx, prefixListForRefreshFib)); + Futures.addCallback(configTxFuture, + new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib), + MoreExecutors.directExecutor()); + futures.add(configTxFuture); + })); + return futures; + } else { + return emptyList(); } - List> futures = new ArrayList<>(); - futures.add(writeConfigTxn.submit()); - return futures; - } else { - return Collections.emptyList(); - } - }); - }); - })); + }); + }); + } catch (InterruptedException | ExecutionException e) { + LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}", + vpnName, primaryRd); + } + }); + }); } private class PostVpnInterfaceWorker implements FutureCallback { @@ -2202,13 +2488,34 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase { + private final String primaryRd; + private final Set prefixListForRefreshFib; + + VpnInterfaceCallBackHandler(String primaryRd, Set prefixListForRefreshFib) { + this.primaryRd = primaryRd; + this.prefixListForRefreshFib = prefixListForRefreshFib; + } + + @Override + public void onSuccess(Void voidObj) { + prefixListForRefreshFib.forEach(prefix -> { + fibManager.refreshVrfEntry(primaryRd, prefix); + }); + } + + @Override + public void onFailure(Throwable throwable) { + LOG.debug("write Tx config operation failed", throwable); + } + } }