X-Git-Url: https://git.opendaylight.org/gerrit/gitweb?a=blobdiff_plain;f=vpnmanager%2Fimpl%2Fsrc%2Fmain%2Fjava%2Forg%2Fopendaylight%2Fnetvirt%2Fvpnmanager%2FVpnInterfaceManager.java;h=bc996c6075581197dfb221f453c9e2396a5a8a07;hb=cf1ea9172b94fc17e45391b62bf4ca24ac5c5fe0;hp=5ed64a5feb1a69847009563d811ac8b15079706a;hpb=2250dd11daa2cd96e304c0a7408f56f0f37f70d1;p=netvirt.git diff --git a/vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java b/vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java index 5ed64a5feb..bc996c6075 100755 --- a/vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java +++ b/vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java @@ -7,7 +7,10 @@ */ package org.opendaylight.netvirt.vpnmanager; -import com.google.common.base.Optional; +import static java.util.Collections.emptyList; +import static org.opendaylight.genius.infra.Datastore.CONFIGURATION; +import static org.opendaylight.genius.infra.Datastore.OPERATIONAL; + import com.google.common.base.Preconditions; import com.google.common.collect.Iterators; import com.google.common.util.concurrent.FutureCallback; @@ -15,39 +18,45 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.math.BigInteger; import java.util.ArrayList; import java.util.Collections; -import java.util.Iterator; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; import java.util.function.Predicate; import java.util.stream.Collectors; -import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import javax.inject.Inject; import javax.inject.Singleton; -import org.opendaylight.controller.md.sal.binding.api.DataBroker; -import org.opendaylight.controller.md.sal.binding.api.WriteTransaction; -import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType; -import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException; -import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException; -import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase; +import org.eclipse.jdt.annotation.Nullable; import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker; +import org.opendaylight.genius.infra.Datastore.Configuration; +import org.opendaylight.genius.infra.Datastore.Operational; import org.opendaylight.genius.infra.ManagedNewTransactionRunner; import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl; +import org.opendaylight.genius.infra.TypedReadWriteTransaction; +import org.opendaylight.genius.infra.TypedWriteTransaction; import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager; import org.opendaylight.genius.mdsalutil.NWUtil; import org.opendaylight.genius.mdsalutil.NwConstants; import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache; import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager; +import org.opendaylight.genius.utils.JvmGlobalLocks; import org.opendaylight.infrautils.caches.CacheProvider; import org.opendaylight.infrautils.jobcoordinator.JobCoordinator; +import org.opendaylight.infrautils.utils.concurrent.Executors; import org.opendaylight.infrautils.utils.concurrent.ListenableFutures; +import org.opendaylight.mdsal.binding.api.DataBroker; +import org.opendaylight.mdsal.common.api.LogicalDatastoreType; +import org.opendaylight.mdsal.common.api.TransactionCommitFailedException; import org.opendaylight.netvirt.bgpmanager.api.IBgpManager; import org.opendaylight.netvirt.fibmanager.api.FibHelper; import org.opendaylight.netvirt.fibmanager.api.IFibManager; @@ -59,10 +68,7 @@ import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler; import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput; import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator; import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry; -import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces; -import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface; -import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey; -import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames; +import org.opendaylight.serviceutils.tools.listener.AbstractAsyncDataTreeChangeListener; import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface; import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid; import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService; @@ -81,13 +87,9 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev15033 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType; -import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey; @@ -101,15 +103,28 @@ import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.routers.ExternalIps; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.Adjacencies; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.VpnInterfaces; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.Adjacency; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.Adjacency.AdjacencyType; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.AdjacencyBuilder; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.AdjacencyKey; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.VpnInterface; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.VpnInterfaceKey; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.vpn._interface.VpnInstanceNames; +import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType; import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort; import org.opendaylight.yangtools.yang.binding.InstanceIdentifier; import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder; +import org.opendaylight.yangtools.yang.common.Uint32; +import org.opendaylight.yangtools.yang.common.Uint64; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @Singleton -public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase { +public class VpnInterfaceManager extends AbstractAsyncDataTreeChangeListener { private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class); private static final short DJC_MAX_RETRIES = 3; @@ -149,7 +164,9 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase(VpnInstanceOpDataEntry.class, dataBroker, LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder( VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider); + start(); } public Runnable isNotifyTaskQueued(String intfName) { return vpnIntfMap.remove(intfName); } - @PostConstruct public void start() { LOG.info("{} start", getClass().getSimpleName()); - registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker); } @Override @@ -185,42 +201,39 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase getWildCardPath() { - return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class); - } - - @Override - protected VpnInterfaceManager getDataTreeChangeListener() { - return VpnInterfaceManager.this; + Executors.shutdownAndAwaitTermination(getExecutorService()); } @Override public void add(final InstanceIdentifier identifier, final VpnInterface vpnInterface) { - LOG.info("add: intfName {} onto vpnName {}", - vpnInterface.getName(), - VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames())); + LOG.trace("Received VpnInterface add event: vpnInterface={}", vpnInterface); + LOG.info("add: intfName {} onto vpnName {}", vpnInterface.getName(), + VpnHelper.getVpnInterfaceVpnInstanceNamesString( + new ArrayList(vpnInterface.getVpnInstanceNames().values()))); addVpnInterface(identifier, vpnInterface, null, null); } private boolean canHandleNewVpnInterface(final InstanceIdentifier identifier, final VpnInterface vpnInterface, String vpnName) { - synchronized (vpnName.intern()) { + // FIXME: separate this out somehow? + final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnName); + lock.lock(); + try { if (isVpnInstanceReady(vpnName)) { return true; } addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName); return false; + } finally { + lock.unlock(); } } // TODO Clean up the exception handling @SuppressWarnings("checkstyle:IllegalCatch") private void addVpnInterface(final InstanceIdentifier identifier, final VpnInterface vpnInterface, - final List oldAdjs, final List newAdjs) { - for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) { + final @Nullable List oldAdjs, final @Nullable List newAdjs) { + for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames().values()) { String vpnName = vpnInterfaceVpnInstance.getVpnName(); addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName); } @@ -228,7 +241,7 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase identifier, final VpnInterface vpnInterface, final List oldAdjs, final List newAdjs, String vpnName) { - final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class); + final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class); final String interfaceName = key.getName(); if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) { @@ -252,10 +265,10 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase vpnInterfaceOpIdentifier, - final VpnInterface vpnInterface, final List oldAdjs, - final List newAdjs, + final VpnInterface vpnInterface, final @Nullable List oldAdjs, + final @Nullable List newAdjs, final InstanceIdentifier identifier, String vpnName) { - final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class); + final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class); final String interfaceName = key.getName(); String primaryRd = vpnUtil.getPrimaryRd(vpnName); if (!vpnUtil.isVpnPendingDelete(primaryRd)) { @@ -263,94 +276,105 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase { // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in // (the inventory tx goes in last) List> futures = new ArrayList<>(); - ListenableFuture confFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit( - confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit( - operTx -> futures.add( - txRunner.callWithNewWriteOnlyTransactionAndSubmit(invTx -> { + //set of prefix used, as entry in prefix-to-interface datastore + // is prerequisite for refresh Fib to avoid race condition leading to + // missing remote next hop in bucket actions on bgp-vpn delete + Set prefixListForRefreshFib = new HashSet<>(); + ListenableFuture confFuture = + txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, + confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, + operTx -> futures.add( + txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, invTx -> { LOG.info( - "addVpnInterface: VPN Interface add event - intfName {} vpnName {}" + "addVpnInterface: VPN Interface add event - intfName {} vpnName {}" + " on dpn {}", - vpnInterface.getName(), vpnName, vpnInterface.getDpnId()); + vpnInterface.getName(), vpnName, vpnInterface.getDpnId()); processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false, - confTx, operTx, invTx, interfaceState, vpnName); + confTx, operTx, invTx, interfaceState, vpnName, + prefixListForRefreshFib); if (oldAdjs != null && !oldAdjs.equals(newAdjs)) { LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}" - + " Update for swapping VPN {} case.", interfaceName, vpnName); + + " Update for swapping VPN {} case.", interfaceName, vpnName); if (newAdjs != null) { for (Adjacency adj : newAdjs) { if (oldAdjs.contains(adj)) { oldAdjs.remove(adj); } else { if (!isBgpVpnInternetVpn - || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) { + || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) { addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, - primaryRd, adj, dpnId, operTx, confTx); + primaryRd, adj, dpnId, operTx, confTx, invTx, + prefixListForRefreshFib); } } } } for (Adjacency adj : oldAdjs) { if (!isBgpVpnInternetVpn - || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) { + || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) { delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, - operTx, confTx); + operTx, confTx); } } } }))))); + Futures.addCallback(confFuture, + new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib), + MoreExecutors.directExecutor()); futures.add(confFuture); Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"), - MoreExecutors.directExecutor()); + MoreExecutors.directExecutor()); LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}" - + " processed successfully", interfaceName, vpnName, dpnId); + + " processed successfully", interfaceName, vpnName, dpnId); return futures; }); } catch (NumberFormatException | IllegalStateException e) { LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for " - + "interface {}. Interface addition on vpn {} failed", interfaceName, - vpnName, e); + + "interface {}. Interface addition on vpn {} failed", interfaceName, + vpnName, e); return; } } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) { jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> { - ListenableFuture future = txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> { - createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName, + ListenableFuture future = + txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> { + createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName, confTx, vpnName); - LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName, + LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName, vpnName, vpnInterface.getDpnId()); - }); + }); ListenableFutures.addErrorLogging(future, LOG, - "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName); + "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName); return Collections.singletonList(future); }); } else { LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState" - + " is not available", interfaceName, vpnName); + + " is not available", interfaceName, vpnName); } } else { LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped" - + " as vpn is pending delete", interfaceName, vpnName, - vpnInterface.getDpnId()); + + " as vpn is pending delete", interfaceName, vpnName, + vpnInterface.getDpnId()); } } // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation - // see comments below. @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"}) - protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd, + protected void processVpnInterfaceUp(final Uint64 dpId, VpnInterface vpnInterface, final String primaryRd, final int lportTag, boolean isInterfaceUp, - WriteTransaction writeConfigTxn, - WriteTransaction writeOperTxn, - WriteTransaction writeInvTxn, - Interface interfaceState, - final String vpnName) { + TypedWriteTransaction writeConfigTxn, + TypedWriteTransaction writeOperTxn, + TypedReadWriteTransaction writeInvTxn, + Interface interfaceState, final String vpnName, + Set prefixListForRefreshFib) throws ExecutionException, InterruptedException { final String interfaceName = vpnInterface.getName(); Optional optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName); @@ -359,8 +383,8 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION, path); if (!optAdjacencies.isPresent()) { - LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface); + LOG.trace("No config adjacencyKeyAdjacencyMap present for vpninterface {}", vpnInterface); return; } - List adjacencies = optAdjacencies.get().getAdjacency(); - for (Adjacency adjacency : adjacencies) { + Map adjacencyKeyAdjacencyMap = optAdjacencies.get().nonnullAdjacency(); + for (Adjacency adjacency : adjacencyKeyAdjacencyMap.values()) { if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) { continue; } @@ -500,16 +524,16 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase(router + .getExternalIps().values())), router.getExtGwMacAddress(), dpId, interfaceName, lportTag); } else { vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName, - VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), + VpnUtil.getIpsListFromExternalIps(new ArrayList(router + .getExternalIps().values())), dpId, interfaceName, lportTag); } } else { @@ -557,7 +583,7 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase identifier, String vpnName, String interfaceName) { if (rd == null) { @@ -588,20 +614,20 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, path); if (adjacencies.isPresent()) { - List nextHops = adjacencies.get().getAdjacency(); - if (!nextHops.isEmpty()) { + Map nextHopsMap = adjacencies.get().getAdjacency(); + if (nextHopsMap != null && !nextHopsMap.isEmpty()) { LOG.debug("advertiseAdjacenciesForVpnToBgp: NextHops are {} for interface {} on dpn {} for vpn {}" - + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd); + + " rd {}", nextHopsMap, interfaceName, dpnId, vpnName, rd); VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd); - long l3vni = vpnInstanceOpData.getL3vni(); + Uint32 l3vni = vpnInstanceOpData.getL3vni(); VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni) ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre; - for (Adjacency nextHop : nextHops) { + for (Adjacency nextHop : nextHopsMap.values()) { if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) { continue; } String gatewayMac = null; - long label = 0; + Uint32 label = Uint32.ZERO; if (VpnUtil.isL3VpnOverVxLan(l3vni)) { final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp( vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress()); @@ -614,7 +640,7 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase identifier, - String vpnName, String interfaceName, WriteTransaction writeConfigTxn, - WriteTransaction writeOperTx) { + String vpnName, String interfaceName, TypedWriteTransaction writeConfigTxn, + TypedWriteTransaction writeOperTx) { //Read NextHops InstanceIdentifier path = identifier.augmentation(AdjacenciesOp.class); String rd = vpnUtil.getVpnRd(interfaceName); @@ -655,21 +681,21 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase adjacencies = Optional.absent(); + Optional adjacencies = Optional.empty(); try { adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, path); - } catch (ReadFailedException e) { + } catch (InterruptedException | ExecutionException e) { LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}", interfaceName, vpnName); } if (adjacencies.isPresent()) { - List nextHops = adjacencies.get().getAdjacency(); + Map nextHopsMap = adjacencies.get().getAdjacency(); - if (!nextHops.isEmpty()) { + if (nextHopsMap != null && !nextHopsMap.isEmpty()) { LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}", - nextHops, interfaceName, vpnName, rd); - for (Adjacency nextHop : nextHops) { + nextHopsMap, interfaceName, vpnName, rd); + for (Adjacency nextHop : nextHopsMap.values()) { try { if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) { LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}" @@ -679,7 +705,7 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase writeConfigTxn, + TypedWriteTransaction writeOperTxn, + TypedReadWriteTransaction writeInvTxn, + Interface interfaceState, Set prefixListForRefreshFib) + throws ExecutionException, InterruptedException { InstanceIdentifier identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName); // Read NextHops - InstanceIdentifier path = identifier.augmentation(Adjacencies.class); - Optional adjacencies = Optional.absent(); + Optional vpnInteface = Optional.empty(); try { - adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION, - path); - } catch (ReadFailedException e) { + vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker, + LogicalDatastoreType.CONFIGURATION, identifier); + } catch (InterruptedException | ExecutionException e) { LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}" + "dpn {}", interfaceName, vpnName, primaryRd, dpnId); } - if (!adjacencies.isPresent()) { - addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag, - null/*gwMac*/, writeOperTxn); - return; + Uuid intfnetworkUuid = null; + NetworkType networkType = null; + Long segmentationId = Long.valueOf(-1); + Adjacencies adjacencies = null; + if (vpnInteface.isPresent()) { + intfnetworkUuid = vpnInteface.get().getNetworkId(); + networkType = vpnInteface.get().getNetworkType(); + segmentationId = vpnInteface.get().getSegmentationId().toJava(); + adjacencies = vpnInteface.get().augmentation(Adjacencies.class); + if (adjacencies == null) { + addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag, + null/*gwMac*/, null/*gatewayIp*/, writeOperTxn); + return; + } } // Get the rd of the vpn instance String nextHopIp = null; + String gatewayIp = null; try { nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId); } catch (Exception e) { @@ -734,16 +771,17 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase gwMac = Optional.absent(); + Optional gwMac = Optional.empty(); String vpnInterfaceSubnetGwMacAddress = null; VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd); - Long l3vni = vpnInstanceOpData.getL3vni(); + Uint32 l3vni = vpnInstanceOpData.getL3vni() != null ? vpnInstanceOpData.getL3vni() : Uint32.ZERO; boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni); VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre; VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType); - List nextHops = adjacencies.get().getAdjacency(); + Map nextHopsMap = adjacencies != null ? adjacencies.getAdjacency() + : Collections.emptyMap(); List value = new ArrayList<>(); - for (Adjacency nextHop : nextHops) { + for (Adjacency nextHop : nextHopsMap.values()) { String rd = primaryRd; String nexthopIpValue = nextHop.getIpAddress().split("/")[0]; if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet @@ -758,17 +796,18 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId); if (gatewayIpOptional.isPresent()) { @@ -810,13 +849,16 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase rdToAllocate = vpnUtil .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName, nextHop.getNextHopIpList().get(0), dpnId); @@ -828,6 +870,8 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase writeOperTxn) { VpnInterfaceOpDataEntry opInterface = - VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac); + VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac, gwIp); InstanceIdentifier interfaceId = VpnUtil .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName); - writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface, - WriteTransaction.CREATE_MISSING_PARENTS); + writeOperTxn.mergeParentStructurePut(interfaceId, opInterface); LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore", interfaceName, dpnId, vpnName); } @@ -892,45 +934,45 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase writeConfigTxn, + TypedWriteTransaction writeOperTxn) { - String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue()); - BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId()); + String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue(); + Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern(); AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class); - List adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>(); - if (adjList.isEmpty()) { + Map keyAdjacencyMap = + adjacencies != null && adjacencies.getAdjacency() != null ? adjacencies.getAdjacency() + : Collections.emptyMap(); + if (keyAdjacencyMap.isEmpty()) { LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}", vpnInterface, srcDpnId); return; } String prefix = null; - long label = 0; List value = new ArrayList<>(); - boolean isNextHopAddReqd = false; + boolean isFibNextHopAddReqd = false; String vpnName = vpnInterface.getVpnInstanceName(); - long vpnId = vpnUtil.getVpnId(vpnName); + Uint32 vpnId = vpnUtil.getVpnId(vpnName); String primaryRd = vpnUtil.getPrimaryRd(vpnName); LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}", vpnInterface.getName(), vpnInterface.getDpnId(), - vpnInterface.getVpnInstanceName(), adjList); - for (Adjacency adj : adjList) { + vpnInterface.getVpnInstanceName(), keyAdjacencyMap); + for (Adjacency adj : keyAdjacencyMap.values()) { String rd = adj.getVrfId(); rd = rd != null ? rd : vpnName; prefix = adj.getIpAddress(); - label = adj.getLabel(); + Uint32 label = adj.getLabel(); List nhList = Collections.singletonList(srcTepIp); List nextHopList = adj.getNextHopIpList(); // If TEP is added , update the nexthop of primary adjacency. // Secondary adj nexthop is already pointing to primary adj IP address. - if (nextHopList != null && !nextHopList.isEmpty()) { - /* everything right already */ - } else { - isNextHopAddReqd = true; - } - if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) { value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build()); + if (nextHopList != null && !nextHopList.isEmpty()) { + /* everything right already */ + } else { + isFibNextHopAddReqd = true; + } } else { Optional vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix); if (!vrfEntryOptional.isPresent()) { @@ -939,19 +981,19 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase interfaceId = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName); - writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface, - WriteTransaction.CREATE_MISSING_PARENTS); + writeOperTxn.mergeParentStructurePut(interfaceId, opInterface); LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}", vpnInterface.getName(), srcDpnId, vpnName); @@ -1002,18 +1043,18 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase writeConfigTxn, + TypedWriteTransaction writeOperTxn) { AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class); - List adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>(); + List adjList = adjacencies != null ? new ArrayList(adjacencies.getAdjacency().values()) + : new ArrayList<>(); String prefix = null; - long label = 0; boolean isNextHopRemoveReqd = false; - String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue()); - BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId()); + String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue(); + Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern(); String vpnName = vpnInterface.getVpnInstanceName(); - long vpnId = vpnUtil.getVpnId(vpnName); + Uint32 vpnId = vpnUtil.getVpnId(vpnName); String primaryRd = vpnUtil.getVpnRd(vpnName); if (adjList != null) { List value = new ArrayList<>(); @@ -1026,7 +1067,7 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase nextHopList = adj.getNextHopIpList(); - label = adj.getLabel(); + Uint32 label = adj.getLabel(); if (nextHopList != null && !nextHopList.isEmpty()) { isNextHopRemoveReqd = true; } @@ -1054,8 +1095,8 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase interfaceId = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName); - writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface, - WriteTransaction.CREATE_MISSING_PARENTS); + writeOperTxn.mergeParentStructurePut(interfaceId, opInterface); LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}", vpnInterface.getName(), srcDpnId, vpnName); } } + @SuppressWarnings("checkstyle:IllegalCatch") private List getVpnsExportingMyRoute(final String vpnName) { List vpnsToExportRoute = new ArrayList<>(); - + final VpnInstanceOpDataEntry vpnInstanceOpDataEntry; String vpnRd = vpnUtil.getVpnRd(vpnName); - final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd); - if (vpnInstanceOpDataEntry == null) { - LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}" - + " to check for vpns exporting the routes", vpnName); + try { + VpnInstanceOpDataEntry opDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd); + if (opDataEntry == null) { + LOG.error("getVpnsExportingMyRoute: Null vpn instance op data for vpn {} rd {}" + + " when check for vpns exporting the routes", vpnName, vpnRd); + return vpnsToExportRoute; + } + vpnInstanceOpDataEntry = opDataEntry; + } catch (Exception re) { + LOG.error("getVpnsExportingMyRoute: DSexception when retrieving vpn instance op data for vpn {} rd {}" + + " to check for vpns exporting the routes", vpnName, vpnRd, re); return vpnsToExportRoute; } - Predicate excludeVpn = input -> { if (input.getVpnInstanceName() == null) { LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {} without a name", @@ -1142,46 +1189,48 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId()); if (vrfEntries != null) { - ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> { - for (VrfEntry vrfEntry : vrfEntries) { - try { - if (!FibHelper.isControllerManagedNonInterVpnLinkRoute( + ListenableFutures.addErrorLogging( + txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> { + for (VrfEntry vrfEntry : vrfEntries) { + try { + if (!FibHelper.isControllerManagedNonInterVpnLinkRoute( RouteOrigin.value(vrfEntry.getOrigin()))) { - LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}" - + " is not a controller managed non intervpn link route. Ignoring.", + LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}" + + " is not a controller managed non intervpn link route. Ignoring.", vpn.getVrfId(), vrfEntry.getDestPrefix()); - continue; - } - String prefix = vrfEntry.getDestPrefix(); - String gwMac = vrfEntry.getGatewayMacAddress(); - vrfEntry.getRoutePaths().forEach(routePath -> { - String nh = routePath.getNexthopAddress(); - int label = routePath.getLabel().intValue(); - if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value( + continue; + } + String prefix = vrfEntry.getDestPrefix(); + String gwMac = vrfEntry.getGatewayMacAddress(); + vrfEntry.nonnullRoutePaths().values().forEach(routePath -> { + String nh = routePath.getNexthopAddress(); + Uint32 label = routePath.getLabel(); + if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value( vrfEntry.getOrigin()))) { - LOG.info("handleVpnsExportingRoutesImporting: Importing fib entry rd {} prefix {}" - + " nexthop {} label {} to vpn {} vpnRd {}", + LOG.info( + "handleVpnsExportingRoutesImporting: Importing fib entry rd {}" + + " prefix {} nexthop {} label {} to vpn {} vpnRd {}", vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd); - fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix, + fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix, Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label, - 0 /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED, + Uint32.ZERO /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED, confTx); - } else { - LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry rd {} " - + "prefix {} nexthop {} label {} to vpn {} vpnRd {}", + } else { + LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry" + + " rd {} prefix {} nexthop {} label {} to vpn {} vpnRd {}", vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd); - SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class); - importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(), + SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class); + importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(), confTx); - } - }); - } catch (RuntimeException e) { - LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}" - + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(), + } + }); + } catch (RuntimeException e) { + LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}" + + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(), vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd); + } } - } - }), LOG, "Error handing VPN exporting routes"); + }), LOG, "Error handing VPN exporting routes"); } else { LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}", vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd); @@ -1191,9 +1240,10 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase identifier, VpnInterface vpnInterface) { - final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class); + LOG.trace("Received VpnInterface remove event: vpnInterface={}", vpnInterface); + final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class); final String interfaceName = key.getName(); - for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) { + for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames().values()) { String vpnName = vpnInterfaceVpnInstance.getVpnName(); removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName); } @@ -1204,100 +1254,103 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase { - ListenableFuture future = txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> { - deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName); - LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName); - }); + ListenableFuture future = + txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> { + deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName); + LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName); + }); ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}", vpnInterface.getName(), vpnName); return Collections.singletonList(future); }, DJC_MAX_RETRIES); } else { - removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName); + Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName); + removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState); } } @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE") private void removeVpnInterfaceFromVpn(final InstanceIdentifier identifier, - final VpnInterface vpnInterface, final String vpnName, - final String interfaceName) { + final VpnInterface vpnInterface, final String vpnName, + final String interfaceName, final Interface interfaceState) { LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(), vpnName, vpnInterface.getDpnId()); removeInterfaceFromUnprocessedList(identifier, vpnInterface); jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> { List> futures = new ArrayList<>(3); - ListenableFuture configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit( - writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit( - writeOperTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeInvTxn -> { - LOG.info("remove: - intfName {} onto vpnName {} running config-driven", - interfaceName, vpnName); - BigInteger dpId = BigInteger.ZERO; - int ifIndex = 0; - String gwMacAddress = null; - InstanceIdentifier interfaceId = - VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName); - Optional optVpnInterface = Optional.absent(); - try { - optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker, - LogicalDatastoreType.OPERATIONAL, interfaceId); - } catch (ReadFailedException e) { - LOG.error("remove: Failed to read data store for interface {} vpn {}", interfaceName, - vpnName); - return; - } - Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, - interfaceName); - if (interfaceState != null) { + ListenableFuture configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, + writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, + writeOperTxn -> futures.add( + txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeInvTxn -> { + LOG.info("remove: - intfName {} onto vpnName {} running config-driven", + interfaceName, vpnName); + Uint64 dpId; + int ifIndex; + String gwMacAddress; + InstanceIdentifier interfaceId = + VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName); + Optional optVpnInterface; try { - dpId = InterfaceUtils.getDpIdFromInterface(interfaceState); - } catch (NumberFormatException | IllegalStateException e) { - LOG.error("remove: Unable to retrieve dpnId from interface operational" - + " data store for interface {} on dpn {} for vpn {} Fetching" - + " from vpn interface op data store. ", interfaceName, - vpnInterface.getDpnId(), vpnName, e); - dpId = BigInteger.ZERO; + optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker, + LogicalDatastoreType.OPERATIONAL, interfaceId); + } catch (InterruptedException | ExecutionException e) { + LOG.error("remove: Failed to read data store for interface {} vpn {}", + interfaceName, vpnName); + return; } - ifIndex = interfaceState.getIfIndex(); - gwMacAddress = interfaceState.getPhysAddress().getValue(); - } else { - LOG.info("remove: Interface state not available for {}. Trying to fetch data" - + " from vpn interface op.", interfaceName); - if (optVpnInterface.isPresent()) { - VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get(); - dpId = vpnOpInterface.getDpnId(); - ifIndex = vpnOpInterface.getLportTag().intValue(); - gwMacAddress = vpnOpInterface.getGatewayMacAddress(); + if (interfaceState != null) { + try { + dpId = InterfaceUtils.getDpIdFromInterface(interfaceState); + } catch (NumberFormatException | IllegalStateException e) { + LOG.error("remove: Unable to retrieve dpnId from interface operational" + + " data store for interface {} on dpn {} for vpn {} Fetching" + + " from vpn interface op data store. ", interfaceName, + vpnInterface.getDpnId(), vpnName, e); + dpId = Uint64.ZERO; + } + ifIndex = interfaceState.getIfIndex(); + gwMacAddress = interfaceState.getPhysAddress().getValue(); } else { - LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped" - + " as interfaceState and vpn interface op is not" - + " available", interfaceName, vpnName); - return; + LOG.info("remove: Interface state not available for {}. Trying to fetch data" + + " from vpn interface op.", interfaceName); + if (optVpnInterface.isPresent()) { + VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get(); + dpId = vpnOpInterface.getDpnId(); + ifIndex = vpnOpInterface.getLportTag().intValue(); + gwMacAddress = vpnOpInterface.getGatewayMacAddress(); + } else { + LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped" + + " as interfaceState and vpn interface op is not" + + " available", interfaceName, vpnName); + return; + } } - } - processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress, - optVpnInterface.isPresent() ? optVpnInterface.get() : null, false, - writeConfigTxn, writeOperTxn, writeInvTxn); - LOG.info( - "remove: Removal of vpn interface {} on dpn {} for vpn {} processed " - + "successfully", - interfaceName, vpnInterface.getDpnId(), vpnName); - }))))); + processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress, + optVpnInterface.isPresent() ? optVpnInterface.get() : null, false, + writeConfigTxn, writeOperTxn, writeInvTxn); + LOG.info( + "remove: Removal of vpn interface {} on dpn {} for vpn {} processed " + + "successfully", + interfaceName, vpnInterface.getDpnId(), vpnName); + }))))); futures.add(configFuture); - Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config")); + Futures.addCallback(configFuture, new PostVpnInterfaceWorker( + interfaceName, false, "Config"), MoreExecutors.directExecutor()); return futures; }, DJC_MAX_RETRIES); } - protected void processVpnInterfaceDown(BigInteger dpId, + protected void processVpnInterfaceDown(Uint64 dpId, String interfaceName, int lportTag, String gwMac, VpnInterfaceOpDataEntry vpnOpInterface, boolean isInterfaceStateDown, - WriteTransaction writeConfigTxn, - WriteTransaction writeOperTxn, - WriteTransaction writeInvTxn) { + TypedWriteTransaction writeConfigTxn, + TypedWriteTransaction writeOperTxn, + TypedReadWriteTransaction writeInvTxn) + throws ExecutionException, InterruptedException { if (vpnOpInterface == null) { LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}" + " as it is not available in operational data store", interfaceName, dpId); @@ -1307,7 +1360,7 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier( interfaceName, vpnName); if (!isInterfaceStateDown) { - final long vpnId = vpnUtil.getVpnId(vpnName); + final Uint32 vpnId = vpnUtil.getVpnId(vpnName); vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName, null); final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName); removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName, @@ -1328,68 +1381,93 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase writeConfigTxn, + TypedWriteTransaction writeOperTxn, + TypedReadWriteTransaction writeInvTxn) + throws ExecutionException, InterruptedException { //Read NextHops try { InstanceIdentifier identifier = VpnUtil .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName); - InstanceIdentifier path = identifier.augmentation(AdjacenciesOp.class); - Optional adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, - LogicalDatastoreType.OPERATIONAL, path); + Optional vpnInterfaceOpDataEnteryOptional = + SingleTransactionDataBroker.syncReadOptional(dataBroker, + LogicalDatastoreType.OPERATIONAL, identifier); + boolean isNonPrimaryAdjIp = false; String primaryRd = vpnUtil.getVpnRd(vpnName); LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}", interfaceName, dpnId, vpnName, primaryRd); - if (adjacencies.isPresent() && !adjacencies.get().getAdjacency().isEmpty()) { - List nextHops = adjacencies.get().getAdjacency(); + if (!vpnInterfaceOpDataEnteryOptional.isPresent()) { + LOG.error("removeAdjacenciesFromVpn: VpnInterfaceOpDataEntry-Oper DS is absent for Interface {} " + + "on vpn {} dpn {}", interfaceName, vpnName, dpnId); + return; + } + AdjacenciesOp adjacencies = vpnInterfaceOpDataEnteryOptional.get().augmentation(AdjacenciesOp.class); + + if (adjacencies != null && !adjacencies.getAdjacency().isEmpty()) { + Map nextHopsMap = adjacencies.getAdjacency(); LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}", - interfaceName, dpnId, vpnName, nextHops); - for (Adjacency nextHop : nextHops) { + interfaceName, dpnId, vpnName, nextHopsMap); + for (Adjacency nextHop : nextHopsMap.values()) { if (nextHop.isPhysNetworkFunc()) { LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}", nextHop.getSubnetId().getValue(), nextHop.getIpAddress()); fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(), - null/*writeCfgTxn*/); + null, null/*writeCfgTxn*/); } else { String rd = nextHop.getVrfId(); List nhList; if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) { nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName); + isNonPrimaryAdjIp = Boolean.TRUE; } else { // This is a primary adjacency nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList() - : Collections.emptyList(); + : emptyList(); removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac, + vpnInterfaceOpDataEnteryOptional.get().getGatewayIpAddress(), interfaceName, writeInvTxn); + isNonPrimaryAdjIp = Boolean.FALSE; } if (!nhList.isEmpty()) { - if (rd.equals(vpnName)) { + if (Objects.equals(primaryRd, vpnName)) { //this is an internal vpn - the rd is assigned to the vpn instance name; //remove from FIB directly nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName, interfaceName, dpnId, writeConfigTxn, writeOperTxn)); } else { removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd, - interfaceName, writeConfigTxn, writeOperTxn); + interfaceName, isNonPrimaryAdjIp, writeConfigTxn, writeOperTxn); } } else { LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}" + " interface {}", nextHop.getIpAddress(), rd, nextHop.getAdjacencyType().toString(), interfaceName); bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress()); - fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn); + fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), null, writeConfigTxn); } } String ip = nextHop.getIpAddress().split("/")[0]; LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip); - if (vpnVipToPort != null) { + if (vpnVipToPort != null && vpnVipToPort.getPortName().equals(interfaceName)) { vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null); LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry" + " for Interface {} ip {} on dpn {} for vpn {}", vpnVipToPort.getPortName(), ip, dpnId, vpnName); } + // Remove the MIP-IP from VpnPortIpToPort. + if (isNonPrimaryAdjIp) { + VpnPortipToPort persistedIp = vpnUtil.getVpnPortipToPort(vpnName, ip); + if (persistedIp != null && persistedIp.isLearntIp() + && persistedIp.getPortName().equals(interfaceName)) { + VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null); + LOG.info( + "removeAdjacenciesFromVpn: Learnt-IP: {} interface {} of vpn {} removed " + + "from VpnPortipToPort", + persistedIp.getPortFixedip(), persistedIp.getPortName(), vpnName); + } + } VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip); if (vpnPortipToPort != null) { VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null); @@ -1402,24 +1480,28 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName, String interfaceName, - BigInteger dpnId, WriteTransaction writeConfigTxn, - WriteTransaction writeOperTx) { + private Consumer removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName, + String interfaceName, Uint64 dpnId, + TypedWriteTransaction writeConfigTxn, + TypedWriteTransaction writeOperTx) { return (nh) -> { String primaryRd = vpnUtil.getVpnRd(vpnName); String prefix = nextHop.getIpAddress(); String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix); LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}", nextHop, vpnName, interfaceName, dpnId); - synchronized (vpnNamePrefixKey.intern()) { + // FIXME: separate this out somehow? + final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnNamePrefixKey); + lock.lock(); + try { if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName, prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) { //If extra-route is present behind at least one VM, then do not remove or update @@ -1429,6 +1511,8 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase nhList, String vpnName, String primaryRd, - BigInteger dpnId, String rd, String interfaceName, - WriteTransaction writeConfigTxn, WriteTransaction writeOperTx) { + Uint64 dpnId, String rd, String interfaceName, boolean isNonPrimaryAdjIp, + TypedWriteTransaction writeConfigTxn, + TypedWriteTransaction writeOperTx) { List vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName); nhList.forEach((nh) -> { //IRT: remove routes from other vpns importing it - vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(), - nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx); + if (isNonPrimaryAdjIp) { + removeLearntPrefixFromBGP(rd, nextHop.getIpAddress(), nh, writeConfigTxn); + } else { + vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(), + nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx); + } for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) { String vpnRd = vpn.getVrfId(); if (vpnRd != null) { @@ -1459,9 +1548,30 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase writeConfigTxn) { + try { + if (!fibManager.checkFibEntryExist(dataBroker, rd, prefix, nextHop)) { + LOG.info("removeLearntPrefixFromBGP: IP {} with nexthop {} rd {} is already removed.Ignoring this" + + " operation", prefix, nextHop, rd); + return; + } + LOG.info("removeLearntPrefixFromBGP: VPN WITHDRAW: Removing Fib Entry rd {} prefix {} nexthop {}", + rd, prefix, nextHop); + fibManager.removeOrUpdateFibEntry(rd, prefix, nextHop, writeConfigTxn); + bgpManager.withdrawPrefix(rd, prefix); // TODO: Might be needed to include nextHop here + LOG.info("removeLearntPrefixFromBGP: VPN WITHDRAW: Removed Fib Entry rd {} prefix {} nexthop {}", + rd, prefix, nextHop); + } catch (Exception e) { + LOG.error("removeLearntPrefixFromBGP: Delete prefix {} rd {} nextHop {} failed", prefix, rd, nextHop, e); + } + } + + private void removeGwMacAndArpResponderFlows(Adjacency nextHop, Uint32 vpnId, Uint64 dpnId, + int lportTag, String gwMac, String gwIp, String interfaceName, + TypedReadWriteTransaction writeInvTxn) + throws ExecutionException, InterruptedException { final Uuid subnetId = nextHop.getSubnetId(); if (nextHop.getSubnetGatewayMacAddress() == null) { // A valid mac-address was not available for this subnet-gateway-ip @@ -1469,11 +1579,11 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId, + private List getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, Uint64 dpnId, String interfaceName) { // This is either an extra-route (or) a learned IP via subnet-route List nhList = null; @@ -1483,7 +1593,7 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase identifier, final VpnInterface original, + public void update(final InstanceIdentifier identifier, final VpnInterface original, final VpnInterface update) { - LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}" ,update.getName(), - update.getDpnId(), original.getVpnInstanceNames(), - update.getVpnInstanceNames()); + LOG.trace("Received VpnInterface update event: original={}, update={}", original, update); + LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}", update.getName(), + update.getDpnId(), original.getVpnInstanceNames(), update.getVpnInstanceNames()); + if (original.equals(update)) { + LOG.info("update: original {} update {} are same. No update required.", original, update); + return; + } final String vpnInterfaceName = update.getName(); - final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName); - final Adjacencies origAdjs = original.augmentation(Adjacencies.class); - final List oldAdjs = origAdjs != null && origAdjs.getAdjacency() - != null ? origAdjs.getAdjacency() : new ArrayList<>(); - final Adjacencies updateAdjs = update.augmentation(Adjacencies.class); - final List newAdjs = updateAdjs != null && updateAdjs.getAdjacency() - != null ? updateAdjs.getAdjacency() : new ArrayList<>(); - + final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName); LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName); //handles switching between jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> { - if (handleVpnSwapForVpnInterface(identifier, original, update)) { - LOG.info("update: handled VPNInterface {} on dpn {} update" - + "upon VPN swap from oldVpn(s) {} to newVpn(s) {}", + List> futures = new ArrayList<>(); + if (handleVpnInstanceUpdateForVpnInterface(identifier, original, update, futures)) { + LOG.info("update: handled Instance update for VPNInterface {} on dpn {} from oldVpn(s) {} " + + "to newVpn(s) {}", original.getName(), dpnId, - VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()), - VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames())); - return Collections.emptyList(); + VpnHelper.getVpnInterfaceVpnInstanceNamesString( + new ArrayList(original.getVpnInstanceNames().values())), + VpnHelper.getVpnInterfaceVpnInstanceNamesString( + new ArrayList(update.getVpnInstanceNames().values()))); + return emptyList(); } - for (VpnInstanceNames vpnInterfaceVpnInstance : update.getVpnInstanceNames()) { - String newVpnName = vpnInterfaceVpnInstance.getVpnName(); - List copyNewAdjs = new ArrayList<>(newAdjs); - List copyOldAdjs = new ArrayList<>(oldAdjs); - String primaryRd = vpnUtil.getPrimaryRd(newVpnName); - if (!vpnUtil.isVpnPendingDelete(primaryRd)) { - // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in - List> futures = new ArrayList<>(); - futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> { - futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(operTx -> { + updateVpnInstanceAdjChange(original, update, vpnInterfaceName, futures); + return futures; + }); + } + + private boolean handleVpnInstanceUpdateForVpnInterface(InstanceIdentifier identifier, + VpnInterface original, VpnInterface update, + List> futures) { + boolean isVpnInstanceUpdate = false; + final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class); + final String interfaceName = key.getName(); + List oldVpnList = VpnUtil.getVpnListForVpnInterface(original); + List oldVpnListCopy = new ArrayList<>(); + oldVpnListCopy.addAll(oldVpnList); + List newVpnList = VpnUtil.getVpnListForVpnInterface(update); + List newVpnListCopy = new ArrayList<>(); + newVpnListCopy.addAll(newVpnList); + + oldVpnList.removeAll(newVpnList); + newVpnList.removeAll(oldVpnListCopy); + //This block will execute only on if there is a change in the VPN Instance. + if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) { + /* + * Internet BGP-VPN Instance update with single router: + * ==================================================== + * In this case single VPN Interface will be part of maximum 2 VPN Instance only. + * 1st VPN Instance : router VPN or external BGP-VPN. + * 2nd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access. + * + * VPN Instance UPDATE: + * oldVpnList = 0 and newVpnList = 1 (Internet BGP-VPN) + * oldVpnList = 1 and newVpnList = 0 (Internet BGP-VPN) + * + * External BGP-VPN Instance update with single router: + * ==================================================== + * In this case single VPN interface will be part of maximum 1 VPN Instance only. + * + * Updated VPN Instance will be always either internal router VPN to + * external BGP-VPN or external BGP-VPN to internal router VPN swap. + * + * VPN Instance UPDATE: + * oldVpnList = 1 and newVpnList = 1 (router VPN to Ext-BGPVPN) + * oldVpnList = 1 and newVpnList = 1 (Ext-BGPVPN to router VPN) + * + * Dual Router VPN Instance Update: + * ================================ + * In this case single VPN interface will be part of maximum 3 VPN Instance only. + * + * 1st VPN Instance : router VPN or external BGP-VPN-1. + * 2nd VPN Instance : router VPN or external BGP-VPN-2. + * 3rd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access. + * + * Dual Router --> Associated with common external BGP-VPN Instance. + * 1st router and 2nd router are getting associated with single External BGP-VPN + * 1) add 1st router to external bgpvpn --> oldVpnList=1, newVpnList=1; + * 2) add 2nd router to the same external bgpvpn --> oldVpnList=1, newVpnList=0 + * In this case, we need to call removeVpnInterfaceCall() followed by addVpnInterfaceCall() + * + * + */ + isVpnInstanceUpdate = true; + if (VpnUtil.isDualRouterVpnUpdate(oldVpnListCopy, newVpnListCopy)) { + if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3) + && oldVpnList.size() == 1 && newVpnList.isEmpty()) { + //Identify the external BGP-VPN Instance and pass that value as newVpnList + List externalBgpVpnList = new ArrayList<>(); + for (String newVpnName : newVpnListCopy) { + String primaryRd = vpnUtil.getPrimaryRd(newVpnName); + VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd); + if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry + .BgpvpnType.BGPVPNExternal) { + externalBgpVpnList.add(newVpnName); + break; + } + } + //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall() + updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, + externalBgpVpnList, oldVpnListCopy, futures); + + } else if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3) + && oldVpnList.isEmpty() && newVpnList.size() == 1) { + //Identify the router VPN Instance and pass that value as oldVpnList + List routerVpnList = new ArrayList<>(); + for (String newVpnName : newVpnListCopy) { + String primaryRd = vpnUtil.getPrimaryRd(newVpnName); + VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd); + if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry + .BgpvpnType.VPN) { + routerVpnList.add(newVpnName); + break; + } + } + //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall() + updateVpnInstanceChange(identifier, interfaceName, original, update, routerVpnList, + newVpnList, oldVpnListCopy, futures); + + } else { + //Handle remaining use cases. + updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList, + oldVpnListCopy, futures); + } + } else { + updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList, + oldVpnListCopy, futures); + } + } + return isVpnInstanceUpdate; + } + + private void updateVpnInstanceChange(InstanceIdentifier identifier, String interfaceName, + VpnInterface original, VpnInterface update, List oldVpnList, + List newVpnList, List oldVpnListCopy, + List> futures) { + final Adjacencies origAdjs = original.augmentation(Adjacencies.class); + final List oldAdjs = origAdjs != null && origAdjs.getAdjacency() != null + ? new ArrayList(origAdjs.getAdjacency().values()) : new ArrayList<>(); + final Adjacencies updateAdjs = update.augmentation(Adjacencies.class); + final List newAdjs = updateAdjs != null && updateAdjs.getAdjacency() != null + ? new ArrayList(updateAdjs.getAdjacency().values()) : new ArrayList<>(); + + boolean isOldVpnRemoveCallExecuted = false; + for (String oldVpnName : oldVpnList) { + LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} " + + "remove from vpnName {} ", interfaceName, oldVpnName); + removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName); + LOG.info("updateVpnInstanceChange: Processed Remove for update on VPNInterface" + + " {} upon VPN update from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName, + newVpnList); + isOldVpnRemoveCallExecuted = true; + } + //Wait for previous interface bindings to be removed + if (isOldVpnRemoveCallExecuted && !newVpnList.isEmpty()) { + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + LOG.error("updateVpnInstanceChange: InterruptedException caught for interface {}", interfaceName, e); + } + } + for (String newVpnName : newVpnList) { + String primaryRd = vpnUtil.getPrimaryRd(newVpnName); + if (!vpnUtil.isVpnPendingDelete(primaryRd)) { + LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} " + + "onto vpnName {} ", interfaceName, newVpnName); + addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName); + LOG.info("updateVpnInstanceChange: Processed Add for update on VPNInterface {}" + + "from oldVpn(s) {} to newVpn {} ", + interfaceName, oldVpnListCopy, newVpnName); + /* This block will execute only if V6 subnet is associated with internet BGP-VPN. + * Use Case: + * In Dual stack network, first V4 subnet only attached to router and router is associated + * with internet BGP-VPN(router-gw). At this point VPN interface is having only router vpn instance. + * Later V6 subnet is added to router, at this point existing VPN interface will get updated + * with Internet BGP-VPN instance(Note: Internet BGP-VPN Instance update in vpn interface + * is applicable for only on V6 subnet is added to router). newVpnList = Contains only Internet + * BGP-VPN Instance. So we required V6 primary adjacency info needs to be populated onto + * router VPN as well as Internet BGP-VPN. + * + * addVpnInterfaceCall() --> It will create V6 Adj onto Internet BGP-VPN only. + * updateVpnInstanceAdjChange() --> This method call is needed for second primary V6 Adj + * update in existing router VPN instance. + */ + if (vpnUtil.isBgpVpnInternet(newVpnName)) { + LOG.info("updateVpnInstanceChange: VPN Interface {} with new Adjacency {} in existing " + + "VPN instance {}", interfaceName, newAdjs, original.getVpnInstanceNames()); + updateVpnInstanceAdjChange(original, update, interfaceName, futures); + } + } else { + LOG.info("updateVpnInstanceChange: failed to Add for update on VPNInterface {} from oldVpn(s) {} to " + + "newVpn {} as the new vpn does not exist in oper DS or it is in PENDING_DELETE state", + interfaceName, oldVpnListCopy, newVpnName); + } + } + } + + // TODO Clean up the exception handling + @SuppressWarnings("checkstyle:IllegalCatch") + private List> updateVpnInstanceAdjChange(VpnInterface original, VpnInterface update, + String vpnInterfaceName, + List> futures) { + final Adjacencies origAdjs = original.augmentation(Adjacencies.class); + final List oldAdjs = origAdjs != null && origAdjs.getAdjacency() + != null ? new ArrayList(origAdjs.getAdjacency().values()) : new ArrayList<>(); + final Adjacencies updateAdjs = update.augmentation(Adjacencies.class); + final List newAdjs = updateAdjs != null && updateAdjs.getAdjacency() + != null ? new ArrayList(updateAdjs.getAdjacency().values()) : new ArrayList<>(); + + final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName); + for (VpnInstanceNames vpnInterfaceVpnInstance : update.nonnullVpnInstanceNames().values()) { + String newVpnName = vpnInterfaceVpnInstance.getVpnName(); + List copyNewAdjs = new ArrayList<>(newAdjs); + List copyOldAdjs = new ArrayList<>(oldAdjs); + String primaryRd = vpnUtil.getPrimaryRd(newVpnName); + if (!vpnUtil.isVpnPendingDelete(primaryRd)) { + // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in + //set of prefix used as entry in prefix-to-interface datastore + // is prerequisite for refresh Fib to avoid race condition leading to missing remote next hop + // in bucket actions on bgp-vpn delete + Set prefixListForRefreshFib = new HashSet<>(); + ListenableFuture configTxFuture = txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, + confTx -> futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, + operTx -> { InstanceIdentifier vpnInterfaceOpIdentifier = - VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName); - LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven", + VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName); + LOG.info("VPN Interface update event-intfName {} onto vpnName {} running config-driven", update.getName(), newVpnName); //handle both addition and removal of adjacencies - //currently, new adjacency may be an extra route + // currently, new adjacency may be an extra route boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName); if (!oldAdjs.equals(newAdjs)) { for (Adjacency adj : copyNewAdjs) { if (copyOldAdjs.contains(adj)) { copyOldAdjs.remove(adj); } else { - // add new adjacency - right now only extra route will hit this path + // add new adjacency if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) { - addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj, - dpnId, operTx, confTx); + try { + addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj, + dpnId, operTx, confTx, confTx, prefixListForRefreshFib); + } catch (RuntimeException e) { + LOG.error("Failed to add adjacency {} to vpn interface {} with" + + " dpnId {}", adj, vpnInterfaceName, dpnId, e); + } } - LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to" - + " vpn interface {} on vpn {} dpnId {}", - adj.getIpAddress(), adj.getNextHopIpList(), - adj.getLabel(), adj.getSubnetId(), update.getName(), - newVpnName, dpnId); + LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} " + + " added to vpn interface {} on vpn {} dpnId {}", + adj.getIpAddress(), adj.getNextHopIpList(), adj.getLabel(), + adj.getSubnetId(), update.getName(), newVpnName, dpnId); } } for (Adjacency adj : copyOldAdjs) { if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) { if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency - && !adj.isPhysNetworkFunc()) { - delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, - operTx, confTx); - Optional optVpnInterface = operTx.read( - LogicalDatastoreType.OPERATIONAL, vpnInterfaceOpIdentifier).get(); - if (optVpnInterface.isPresent()) { - VpnInterfaceOpDataEntry vpnInterfaceOpDataEntry = optVpnInterface.get(); - long vpnId = vpnUtil.getVpnId(newVpnName); - vpnUtil.removePrefixToInterfaceAdj(adj, vpnId, - vpnInterfaceOpDataEntry, operTx); - } else { - LOG.info("update: Vpninterface {} not present in Operational", - vpnInterfaceName); - } + && !adj.isPhysNetworkFunc()) { + delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, operTx, + confTx); //remove FIB entry String vpnRd = vpnUtil.getVpnRd(newVpnName); LOG.debug("update: remove prefix {} from the FIB and BGP entry " - + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd); + + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd); //remove BGP entry - fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx); + fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), null, confTx); if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) { bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress()); } } else { delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, - operTx, confTx); + operTx, confTx); } } LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from" - + " vpn interface {} on vpn {}", adj.getIpAddress(), adj - .getNextHopIpList(), - adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName); + + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(), + adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName); } } - })); - })); - for (ListenableFuture future : futures) { - ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}", - update.getName(), update.getVpnInstanceNames()); - } - return futures; - } else { - LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}" - + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd); - } - } - return Collections.emptyList(); - }); - } - - private boolean handleVpnSwapForVpnInterface(InstanceIdentifier identifier, - VpnInterface original, VpnInterface update) { - boolean isSwap = Boolean.FALSE; - final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class); - final String interfaceName = key.getName(); - List oldVpnList = original.getVpnInstanceNames().stream() - .map(VpnInstanceNames::getVpnName).collect(Collectors.toList()); - List oldVpnListCopy = new ArrayList<>(); - oldVpnListCopy.addAll(oldVpnList); - List newVpnList = update.getVpnInstanceNames().stream() - .map(VpnInstanceNames::getVpnName).collect(Collectors.toList()); - oldVpnList.removeAll(newVpnList); - newVpnList.removeAll(oldVpnListCopy); - if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) { - for (String oldVpnName: oldVpnList) { - isSwap = Boolean.TRUE; - LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} remove vpnName {}" - + " running config-driven swap removal", interfaceName, oldVpnName); - removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName); - LOG.info("handleVpnSwapForVpnInterface: Processed Remove for update on VPNInterface {} upon VPN swap" - + "from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName, newVpnList); - } - //Wait for previous interface bindings to be removed - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - //Ignore - } - - final Adjacencies origAdjs = original.augmentation(Adjacencies.class); - final List oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null) - ? origAdjs.getAdjacency() : new ArrayList<>(); - final Adjacencies updateAdjs = update.augmentation(Adjacencies.class); - final List newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null) - ? updateAdjs.getAdjacency() : new ArrayList<>(); - - for (String newVpnName: newVpnList) { - String primaryRd = vpnUtil.getPrimaryRd(newVpnName); - isSwap = Boolean.TRUE; - if (!vpnUtil.isVpnPendingDelete(primaryRd)) { - LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} onto vpnName {}" - + "running config-driven swap addition", interfaceName, newVpnName); - addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName); - LOG.info("handleVpnSwapForVpnInterface: Processed Add for update on VPNInterface {}" - + "from oldVpn(s) {} to newVpn {} upon VPN swap", - interfaceName, oldVpnListCopy, newVpnName); + }))); + Futures.addCallback(configTxFuture, new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib), + MoreExecutors.directExecutor()); + futures.add(configTxFuture); + for (ListenableFuture future : futures) { + ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}", + update.getName(), update.getVpnInstanceNames()); } + } else { + LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}" + + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd); } } - return isSwap; + return futures; } - private void updateLabelMapper(Long label, List nextHopIpList) { + private void updateLabelMapper(Uint32 label, List nextHopIpList) { + final String labelStr = Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!") + .toString(); + // FIXME: separate this out somehow? + final ReentrantLock lock = JvmGlobalLocks.getLockForString(labelStr); + lock.lock(); try { - Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!"); - synchronized (label.toString().intern()) { - InstanceIdentifier lriIid = InstanceIdentifier.builder(LabelRouteMap.class) - .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build(); - Optional opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker, - LogicalDatastoreType.OPERATIONAL, lriIid); - if (opResult.isPresent()) { - LabelRouteInfo labelRouteInfo = - new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build(); - SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid, - labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY); - } + InstanceIdentifier lriIid = InstanceIdentifier.builder(LabelRouteMap.class) + .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build(); + Optional opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker, + LogicalDatastoreType.OPERATIONAL, lriIid); + if (opResult.isPresent()) { + LabelRouteInfo labelRouteInfo = + new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build(); + SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid, + labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY); } LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label, nextHopIpList); - } catch (ReadFailedException e) { + } catch (InterruptedException | ExecutionException e) { LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label, nextHopIpList); } catch (TransactionCommitFailedException e) { LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label, nextHopIpList); + } finally { + lock.unlock(); } } - public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label, - SubnetRoute route, String parentVpnRd, WriteTransaction writeConfigTxn) { + public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, Uint32 label, + SubnetRoute route, String parentVpnRd, TypedWriteTransaction writeConfigTxn) { RouteOrigin origin = RouteOrigin.SELF_IMPORTED; VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd) @@ -1705,7 +1950,7 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase vrfTableId = idBuilder.build(); VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build(); if (writeConfigTxn != null) { - writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true); + writeConfigTxn.mergeParentStructureMerge(vrfTableId, vrfTableNew); } else { vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew); } @@ -1714,8 +1959,12 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase identifier, String primaryRd, - Adjacency adj, BigInteger dpnId, WriteTransaction writeOperTxn, - WriteTransaction writeConfigTxn) { + Adjacency adj, Uint64 dpnId, + TypedWriteTransaction writeOperTxn, + TypedWriteTransaction writeConfigTxn, + TypedReadWriteTransaction writeInvTxn, + Set prefixListForRefreshFib) + throws ExecutionException, InterruptedException { String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName(); String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName(); try { @@ -1731,16 +1980,13 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase adjacencies; - if (optAdjacencies.isPresent()) { - adjacencies = optAdjacencies.get().getAdjacency(); - } else { - // This code will be hit in case of first PNF adjacency - adjacencies = new ArrayList<>(); + List adjacencies = new ArrayList<>(); + if (optAdjacencies.isPresent() && optAdjacencies.get().getAdjacency() != null) { + adjacencies.addAll(optAdjacencies.get().getAdjacency().values()); } - long vpnId = vpnUtil.getVpnId(vpnName); + Uint32 vpnId = vpnUtil.getVpnId(vpnName); L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName) .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd); Adjacency operationalAdjacency = null; @@ -1752,17 +1998,20 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap( vpnId, null, prefix, vpnName, nh, dpnId); if (rdToAllocate.isPresent()) { @@ -1771,7 +2020,8 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase { if (vpn.getVrfId() != null) { - vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(), vpnId, prefix, + vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(), + vpnId, prefix, vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId) .ifPresent( rds -> vpnManager.addExtraRoute( - vpnUtil.getVpnName(vpn.getVpnId()), - adj.getIpAddress(), nh, rds, - currVpnIntf.getVpnInstanceName(), l3vni, - RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(), - opAdjacency, encapType, writeConfigTxn)); + vpnUtil.getVpnName(vpn.getVpnId()), adj.getIpAddress(), + nh, rds, currVpnIntf.getVpnInstanceName(), l3vni, + RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(), opAdjacency, + encapType, prefixListForRefreshFib, writeConfigTxn)); } }); } + } finally { + lock.unlock(); } } else if (adj.isPhysNetworkFunc()) { // PNF adjacency. LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix, currVpnIntf.getName(), vpnName); + InstanceIdentifier vpnIfaceConfigidentifier = VpnUtil + .getVpnInterfaceIdentifier(currVpnIntf.getName()); + Optional vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker, + LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier); + Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix, + Prefixes.PrefixCue.PhysNetFunc); + if (vpnIntefaceConfig.isPresent()) { + pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix, + vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(), + vpnIntefaceConfig.get().getSegmentationId().toJava(), Prefixes.PrefixCue.PhysNetFunc); + } + String parentVpnRd = getParentVpnRdForExternalSubnet(adj); - writeOperTxn.merge( - LogicalDatastoreType.OPERATIONAL, + writeOperTxn.mergeParentStructureMerge( VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()), - prefix), VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), - prefix, adj.getSubnetId(), Prefixes.PrefixCue.PhysNetFunc), true); + prefix), pnfPrefix); fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(), - adj.getIpAddress(), Collections.emptyList(), null /* EncapType */, 0 /* label */, - 0 /*l3vni*/, null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn); + adj.getIpAddress(), emptyList(), null /* EncapType */, Uint32.ZERO /* label */, + Uint32.ZERO /*l3vni*/, null /* gw-mac */, parentVpnRd, + RouteOrigin.LOCAL, writeConfigTxn); input.setRd(adj.getVrfId()); } @@ -1826,24 +2089,25 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase identifier, Adjacency adj, - BigInteger dpnId, WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) { + Uint64 dpnId, TypedWriteTransaction writeOperTxn, + TypedWriteTransaction writeConfigTxn) { String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName(); String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName(); try { @@ -1855,15 +2119,13 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, path); if (optAdjacencies.isPresent()) { - List adjacencies = optAdjacencies.get().getAdjacency(); - - if (!adjacencies.isEmpty()) { - LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies); - Iterator adjIt = adjacencies.iterator(); - while (adjIt.hasNext()) { - Adjacency adjElem = adjIt.next(); - if (adjElem.getIpAddress().equals(adj.getIpAddress())) { - String rd = adjElem.getVrfId(); + Map keyAdjacencyMap = optAdjacencies.get().getAdjacency(); + + if (keyAdjacencyMap != null && !keyAdjacencyMap.isEmpty()) { + LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", keyAdjacencyMap); + for (Adjacency adjacency : keyAdjacencyMap.values()) { + if (Objects.equals(adjacency.getIpAddress(), adj.getIpAddress())) { + String rd = adjacency.getVrfId(); if (adj.getNextHopIpList() != null) { for (String nh : adj.getNextHopIpList()) { deleteExtraRouteFromCurrentAndImportingVpns( @@ -1874,7 +2136,7 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase writeConfigTxn, + TypedWriteTransaction writeOperTx) { + LOG.info("removing extra-route {} for nexthop {} in VPN {} intfName {} rd {}", + destination, nextHop, vpnName, intfName, rd); vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx); List vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName); for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) { String vpnRd = vpn.getVrfId(); if (vpnRd != null) { + LOG.info("deleting extra-route {} for nexthop {} in VPN {} intfName {} vpnRd {}", + destination, nextHop, vpnName, intfName, vpnRd); vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn, writeOperTx); } } } - InstanceIdentifier getRouterDpnId(String routerName, BigInteger dpnId) { + InstanceIdentifier getRouterDpnId(String routerName, Uint64 dpnId) { return InstanceIdentifier.builder(NeutronRouterDpns.class) .child(RouterDpnList.class, new RouterDpnListKey(routerName)) .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build(); @@ -1919,7 +2186,7 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase writeConfigTxn, String vpnName) { if (vpnInterface == null) { return; } @@ -1935,9 +2202,15 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase writeConfigTxn, String vpnName) { Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class); String rd = vpnUtil.getVpnRd(vpnName); if (adjs != null) { - List adjsList = adjs.getAdjacency(); - for (Adjacency adj : adjsList) { + Map keyAdjacencyMap = adjs.nonnullAdjacency(); + for (Adjacency adj : keyAdjacencyMap.values()) { if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) { String primaryInterfaceIp = adj.getIpAddress(); String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp); - fibManager.removeFibEntry(rd, prefix, writeConfigTxn); + fibManager.removeFibEntry(rd, prefix, null, writeConfigTxn); LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}" + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix); - return; } } } else { @@ -1976,19 +2248,12 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase vpnInterfaceOpIdentifier = VpnUtil .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName); addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null, intefaceData.identifier, vpnName); - return; } private void addToUnprocessedVpnInterfaces(InstanceIdentifier identifier, @@ -2015,7 +2280,10 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase vpnInterfaces = unprocessedVpnInterfaces.get(vpnInstanceName); if (vpnInterfaces != null) { @@ -2033,24 +2301,30 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase identifier, VpnInterface vpnInterface) { - synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) { + // FIXME: use VpnInstanceNamesKey perhaps? What about nulls? + final String firstVpnName = VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface); + final ReentrantLock lock = JvmGlobalLocks.getLockForString(firstVpnName); + lock.lock(); + try { ConcurrentLinkedQueue vpnInterfaces = - unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface)); + unprocessedVpnInterfaces.get(firstVpnName); if (vpnInterfaces != null) { if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) { LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from " - + "unprocessed list", vpnInterface.getName(), - VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface)); + + "unprocessed list", vpnInterface.getName(), firstVpnName); } } else { - LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}", - VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface)); + LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}", firstVpnName); } + } finally { + lock.unlock(); } } @@ -2116,7 +2390,7 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList(); + List vpnToDpnLists = new ArrayList(vpnInstanceOpData.getVpnToDpnList().values()); if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) { return; } @@ -2125,7 +2399,7 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase { + vpnToDpnList.getVpnInterfaces().values().forEach(vpnInterface -> { try { InstanceIdentifier existingVpnInterfaceId = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName); @@ -2140,8 +2414,8 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase operationVpnAdjacencies = vpnInterfaceOptional.get() - .augmentation(AdjacenciesOp.class).getAdjacency(); + List operationVpnAdjacencies = new ArrayList(vpnInterfaceOptional.get() + .augmentation(AdjacenciesOp.class).nonnullAdjacency().values()); // Due to insufficient rds, some of the extra route wont get processed when it is added. // The unprocessed adjacencies will be present in config vpn interface DS but will be missing // in operational DS. These unprocessed adjacencies will be handled below. @@ -2150,7 +2424,7 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase operationVpnAdjacencies.stream() .noneMatch(operationalAdjacency -> - operationalAdjacency.getIpAddress().equals(adjacency.getIpAddress()))) + Objects.equals(operationalAdjacency.getIpAddress(), adjacency.getIpAddress()))) .forEach(adjacency -> { LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency); jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(), @@ -2159,18 +2433,30 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase> futures = new ArrayList<>(); - futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(operTx -> - futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit( - confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId, - primaryRd, adjacency, vpnInterfaceOptional.get() - .getDpnId(), confTx, operTx))))); + futures.add( + txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> { + //set of prefix used, as entry in prefix-to-interface datastore + // is prerequisite for refresh Fib to avoid race condition leading + // to missing remote next hop in bucket actions on bgp-vpn delete + Set prefixListForRefreshFib = new HashSet<>(); + ListenableFuture configTxFuture = + txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, + confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId, + primaryRd, adjacency, + vpnInterfaceOptional.get().getDpnId(), + operTx, confTx, confTx, prefixListForRefreshFib)); + Futures.addCallback(configTxFuture, + new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib), + MoreExecutors.directExecutor()); + futures.add(configTxFuture); + })); return futures; } else { - return Collections.emptyList(); + return emptyList(); } }); }); - } catch (ReadFailedException e) { + } catch (InterruptedException | ExecutionException e) { LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}", vpnName, primaryRd); } @@ -2210,4 +2496,26 @@ public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase { + private final String primaryRd; + private final Set prefixListForRefreshFib; + + VpnInterfaceCallBackHandler(String primaryRd, Set prefixListForRefreshFib) { + this.primaryRd = primaryRd; + this.prefixListForRefreshFib = prefixListForRefreshFib; + } + + @Override + public void onSuccess(Void voidObj) { + prefixListForRefreshFib.forEach(prefix -> { + fibManager.refreshVrfEntry(primaryRd, prefix); + }); + } + + @Override + public void onFailure(Throwable throwable) { + LOG.debug("write Tx config operation failed", throwable); + } + } }