vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14
  15 import com.google.common.base.Optional;
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.collect.Iterators;
  18 import com.google.common.util.concurrent.FutureCallback;
  19 import com.google.common.util.concurrent.Futures;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import com.google.common.util.concurrent.MoreExecutors;
  22 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  23 import java.util.ArrayList;
  24 import java.util.Collections;
  25 import java.util.HashSet;
  26 import java.util.List;
  27 import java.util.Map;
  28 import java.util.Objects;
  29 import java.util.Set;
  30 import java.util.concurrent.ConcurrentHashMap;
  31 import java.util.concurrent.ConcurrentLinkedQueue;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.locks.ReentrantLock;
  34 import java.util.function.Consumer;
  35 import java.util.function.Predicate;
  36 import java.util.stream.Collectors;
  37 import javax.annotation.PostConstruct;
  38 import javax.annotation.PreDestroy;
  39 import javax.inject.Inject;
  40 import javax.inject.Singleton;
  41 import org.eclipse.jdt.annotation.Nullable;
  42 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  43 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  44 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  45 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  46 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  47 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  48 import org.opendaylight.genius.infra.Datastore.Configuration;
  49 import org.opendaylight.genius.infra.Datastore.Operational;
  50 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  51 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  52 import org.opendaylight.genius.infra.TransactionAdapter;
  53 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  54 import org.opendaylight.genius.infra.TypedWriteTransaction;
  55 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  56 import org.opendaylight.genius.mdsalutil.NWUtil;
  57 import org.opendaylight.genius.mdsalutil.NwConstants;
  58 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  59 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  60 import org.opendaylight.genius.utils.JvmGlobalLocks;
  61 import org.opendaylight.infrautils.caches.CacheProvider;
  62 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  63 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  64 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  65 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  66 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  67 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  68 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  69 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  70 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  71 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  72 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  73 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  74 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  75 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  76 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.Adjacencies;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.VpnInterfaces;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.Adjacency;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.Adjacency.AdjacencyType;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.adjacency.list.AdjacencyBuilder;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.VpnInterface;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.VpnInterfaceKey;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.l3vpn.rev200204.vpn.interfaces.vpn._interface.VpnInstanceNames;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 120 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 121 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 122 import org.opendaylight.yangtools.yang.common.Uint32;
 123 import org.opendaylight.yangtools.yang.common.Uint64;
 124 import org.slf4j.Logger;
 125 import org.slf4j.LoggerFactory;
 126
 127 @Singleton
 128 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 129
 130     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 131     private static final short DJC_MAX_RETRIES = 3;
 132
 133     private final DataBroker dataBroker;
 134     private final ManagedNewTransactionRunner txRunner;
 135     private final IBgpManager bgpManager;
 136     private final IFibManager fibManager;
 137     private final IMdsalApiManager mdsalManager;
 138     private final IdManagerService idManager;
 139     private final OdlInterfaceRpcService ifaceMgrRpcService;
 140     private final VpnFootprintService vpnFootprintService;
 141     private final IInterfaceManager interfaceManager;
 142     private final IVpnManager vpnManager;
 143     private final ArpResponderHandler arpResponderHandler;
 144     private final JobCoordinator jobCoordinator;
 145     private final VpnUtil vpnUtil;
 146
 147     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 148
 149     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 150             new ConcurrentHashMap<>();
 151
 152     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 153
 154     @Inject
 155     public VpnInterfaceManager(final DataBroker dataBroker,
 156                                final IBgpManager bgpManager,
 157                                final IdManagerService idManager,
 158                                final IMdsalApiManager mdsalManager,
 159                                final IFibManager fibManager,
 160                                final OdlInterfaceRpcService ifaceMgrRpcService,
 161                                final VpnFootprintService vpnFootprintService,
 162                                final IInterfaceManager interfaceManager,
 163                                final IVpnManager vpnManager,
 164                                final ArpResponderHandler arpResponderHandler,
 165                                final JobCoordinator jobCoordinator,
 166                                final CacheProvider cacheProvider,
 167                                final VpnUtil vpnUtil) {
 168         super(VpnInterface.class, VpnInterfaceManager.class);
 169
 170         this.dataBroker = dataBroker;
 171         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 172         this.bgpManager = bgpManager;
 173         this.idManager = idManager;
 174         this.mdsalManager = mdsalManager;
 175         this.fibManager = fibManager;
 176         this.ifaceMgrRpcService = ifaceMgrRpcService;
 177         this.vpnFootprintService = vpnFootprintService;
 178         this.interfaceManager = interfaceManager;
 179         this.vpnManager = vpnManager;
 180         this.arpResponderHandler = arpResponderHandler;
 181         this.jobCoordinator = jobCoordinator;
 182         this.vpnUtil = vpnUtil;
 183
 184         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 185                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 186                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 187     }
 188
 189     public Runnable isNotifyTaskQueued(String intfName) {
 190         return vpnIntfMap.remove(intfName);
 191     }
 192
 193     @PostConstruct
 194     public void start() {
 195         LOG.info("{} start", getClass().getSimpleName());
 196         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 197     }
 198
 199     @Override
 200     @PreDestroy
 201     public void close() {
 202         super.close();
 203         vpnInstanceOpDataEntryCache.close();
 204     }
 205
 206     @Override
 207     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 208         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 209     }
 210
 211     @Override
 212     protected VpnInterfaceManager getDataTreeChangeListener() {
 213         return VpnInterfaceManager.this;
 214     }
 215
 216     @Override
 217     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 218         LOG.trace("Received VpnInterface add event: vpnInterface={}", vpnInterface);
 219         LOG.info("add: intfName {} onto vpnName {}", vpnInterface.getName(),
 220                 VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 221         addVpnInterface(identifier, vpnInterface, null, null);
 222     }
 223
 224     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 225                           final VpnInterface vpnInterface, String vpnName) {
 226         // FIXME: separate this out somehow?
 227         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnName);
 228         lock.lock();
 229         try {
 230             if (isVpnInstanceReady(vpnName)) {
 231                 return true;
 232             }
 233             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 234             return false;
 235         } finally {
 236             lock.unlock();
 237         }
 238     }
 239
 240     // TODO Clean up the exception handling
 241     @SuppressWarnings("checkstyle:IllegalCatch")
 242     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 243                              final @Nullable List<Adjacency> oldAdjs, final @Nullable List<Adjacency> newAdjs) {
 244         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
 245             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 246             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 247         }
 248     }
 249
 250     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 251                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 252         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 253         final String interfaceName = key.getName();
 254
 255         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 256             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 257                   vpnName, vpnInterface.getName());
 258             return;
 259         }
 260         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 261              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 262         List<Adjacency> copyOldAdjs = null;
 263         if (oldAdjs != null) {
 264             copyOldAdjs = new ArrayList<>();
 265             copyOldAdjs.addAll(oldAdjs);
 266         }
 267         List<Adjacency> copyNewAdjs = null;
 268         if (newAdjs != null) {
 269             copyNewAdjs = new ArrayList<>();
 270             copyNewAdjs.addAll(newAdjs);
 271         }
 272         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 273     }
 274
 275     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 276                     final VpnInterface vpnInterface, final @Nullable List<Adjacency> oldAdjs,
 277                     final @Nullable List<Adjacency> newAdjs,
 278                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 279         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 280         final String interfaceName = key.getName();
 281         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 282         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 283             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 284             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 285             if (interfaceState != null) {
 286                 try {
 287                     final Uint64 dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 288                     final int ifIndex = interfaceState.getIfIndex();
 289                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 290                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 291                         // (the inventory tx goes in last)
 292                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 293                         //set of prefix used, as entry in prefix-to-interface datastore
 294                         // is prerequisite for refresh Fib to avoid race condition leading to
 295                         // missing remote next hop in bucket actions on bgp-vpn delete
 296                         Set<String> prefixListForRefreshFib = new HashSet<>();
 297                         ListenableFuture<Void> confFuture =
 298                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 299                                 confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
 300                                     operTx -> futures.add(
 301                                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, invTx -> {
 302                                             LOG.info(
 303                                                 "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 304                                                     + " on dpn {}",
 305                                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 306                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 307                                                 confTx, operTx, invTx, interfaceState, vpnName,
 308                                                 prefixListForRefreshFib);
 309                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 310                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 311                                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 312                                                 if (newAdjs != null) {
 313                                                     for (Adjacency adj : newAdjs) {
 314                                                         if (oldAdjs.contains(adj)) {
 315                                                             oldAdjs.remove(adj);
 316                                                         } else {
 317                                                             if (!isBgpVpnInternetVpn
 318                                                                 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 319                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 320                                                                     primaryRd, adj, dpnId, operTx, confTx, invTx,
 321                                                                     prefixListForRefreshFib);
 322                                                             }
 323                                                         }
 324                                                     }
 325                                                 }
 326                                                 for (Adjacency adj : oldAdjs) {
 327                                                     if (!isBgpVpnInternetVpn
 328                                                         || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 329                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 330                                                             operTx, confTx);
 331                                                     }
 332                                                 }
 333                                             }
 334                                         })))));
 335                         Futures.addCallback(confFuture,
 336                             new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
 337                             MoreExecutors.directExecutor());
 338                         futures.add(confFuture);
 339                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 340                             MoreExecutors.directExecutor());
 341                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 342                             + " processed successfully", interfaceName, vpnName, dpnId);
 343                         return futures;
 344                     });
 345                 } catch (NumberFormatException | IllegalStateException e) {
 346                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 347                             + "interface {}. Interface addition on vpn {} failed", interfaceName,
 348                         vpnName, e);
 349                     return;
 350                 }
 351             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 352                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 353                     () -> {
 354                         ListenableFuture<Void> future =
 355                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
 356                                 createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 357                                     confTx, vpnName);
 358                                 LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 359                                     vpnName, vpnInterface.getDpnId());
 360                             });
 361                         ListenableFutures.addErrorLogging(future, LOG,
 362                             "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 363                         return Collections.singletonList(future);
 364                     });
 365             } else {
 366                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 367                     + " is not available", interfaceName, vpnName);
 368             }
 369         } else {
 370             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 371                     + " as vpn is pending delete", interfaceName, vpnName,
 372                 vpnInterface.getDpnId());
 373         }
 374     }
 375
 376     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 377     // see comments below.
 378     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 379     protected void processVpnInterfaceUp(final Uint64 dpId, VpnInterface vpnInterface, final String primaryRd,
 380             final int lportTag, boolean isInterfaceUp,
 381             TypedWriteTransaction<Configuration> writeConfigTxn,
 382             TypedWriteTransaction<Operational> writeOperTxn,
 383             TypedReadWriteTransaction<Configuration> writeInvTxn,
 384             Interface interfaceState, final String vpnName,
 385             Set<String> prefixListForRefreshFib) throws ExecutionException, InterruptedException {
 386         final String interfaceName = vpnInterface.getName();
 387         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 388                 vpnName);
 389         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 390         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 391         if (!isInterfaceUp) {
 392             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 393                      interfaceName, dpId, vpnName);
 394             Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 395             if (VpnConstants.INVALID_ID.equals(vpnId)) {
 396                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 397                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 398                         dpId);
 399                 return;
 400             }
 401
 402             boolean waitForVpnInterfaceOpRemoval = false;
 403             if (opVpnInterface != null) {
 404                 String opVpnName = opVpnInterface.getVpnInstanceName();
 405                 String primaryInterfaceIp = null;
 406                 if (Objects.equals(opVpnName, vpnName)) {
 407                     // Please check if the primary VRF Entry does not exist for VPNInterface
 408                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 409                     // back to back
 410                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 411                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 412                     if (adjs == null) {
 413                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 414                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 415                                 vpnName);
 416                         return;
 417                     }
 418                     for (Adjacency adj : adjs) {
 419                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 420                             primaryInterfaceIp = adj.getIpAddress();
 421                             break;
 422                         }
 423                     }
 424                     if (primaryInterfaceIp == null) {
 425                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 426                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 427                                 dpId, vpnName);
 428                         return;
 429                     }
 430                     // Get the rd of the vpn instance
 431                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 432                     if (vrf != null) {
 433                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 434                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 435                         return;
 436                     }
 437                     waitForVpnInterfaceOpRemoval = true;
 438                 } else {
 439                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 440                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 441                 }
 442             }
 443             if (!waitForVpnInterfaceOpRemoval) {
 444                 // Add the VPNInterface and quit
 445                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 446                         null/*ipAddressSourceValuePair*/,
 447                         true /* add */);
 448                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 449                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 450                 if (!isBgpVpnInternetVpn) {
 451                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 452                 }
 453                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 454                         dpId, vpnName);
 455                 if (interfaceManager.isExternalInterface(interfaceName)) {
 456                     processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
 457                         NwConstants.ADD_FLOW);
 458                 }
 459                 return;
 460             }
 461
 462             // FIB didn't get a chance yet to clean up this VPNInterface
 463             // Let us give it a chance here !
 464             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 465                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 466             try {
 467                 Runnable notifyTask = new VpnNotifyTask();
 468                 synchronized (notifyTask) {
 469                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 470                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 471                     // published here while holding the lock on it so this path will hit the wait before notify can be
 472                     // invoked.
 473                     vpnIntfMap.put(interfaceName, notifyTask);
 474                     try {
 475                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 476                     } catch (InterruptedException e) {
 477                         // Ignored
 478                     }
 479                 }
 480             } finally {
 481                 vpnIntfMap.remove(interfaceName);
 482             }
 483
 484             if (opVpnInterface != null) {
 485                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 486                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 487                         dpId, vpnName);
 488                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 489                 return;
 490             }
 491             // VPNInterface got removed, proceed with Add
 492             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 493                     interfaceName, dpId, vpnName);
 494             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 495                     null/*ipAddressSourceValuePair*/,
 496                     true /* add */);
 497             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 498                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 499             if (!isBgpVpnInternetVpn) {
 500                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 501             }
 502             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 503                     + " FIB to clean up", interfaceName, dpId, vpnName);
 504             if (interfaceManager.isExternalInterface(interfaceName)) {
 505                 processExternalVpnInterface(interfaceName, vpnName, dpId,
 506                                lportTag, NwConstants.ADD_FLOW);
 507             }
 508         } else {
 509             try {
 510                 // Interface is retained in the DPN, but its Link Up.
 511                 // Advertise prefixes again for this interface to BGP
 512                 InstanceIdentifier<VpnInterface> identifier =
 513                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 514                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 515                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 516                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 517                 // Perform similar operation as interface add event for extraroutes.
 518                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 519                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 520                         LogicalDatastoreType.CONFIGURATION, path);
 521                 if (!optAdjacencies.isPresent()) {
 522                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 523                     return;
 524                 }
 525                 List<Adjacency> adjacencies = optAdjacencies.get().nonnullAdjacency();
 526                 for (Adjacency adjacency : adjacencies) {
 527                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 528                         continue;
 529                     }
 530                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 531                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 532                         continue;
 533                     }
 534                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 535                             dpId, writeOperTxn, writeConfigTxn, writeInvTxn, prefixListForRefreshFib);
 536                 }
 537             } catch (ReadFailedException e) {
 538                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 539                         interfaceName, vpnName, primaryRd, dpId);
 540             }
 541         }
 542     }
 543
 544     private void processExternalVpnInterface(String interfaceName, String vpnName, Uint64 dpId,
 545         int lportTag, int addOrRemove) {
 546         Uuid extNetworkId;
 547         try {
 548             // vpn instance of ext-net interface is the network-id
 549             extNetworkId = new Uuid(vpnName);
 550         } catch (IllegalArgumentException e) {
 551             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 552                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 553             return;
 554         }
 555
 556         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 557         if (routerIds == null || routerIds.isEmpty()) {
 558             LOG.info("processExternalVpnInterface: No router is associated with {}."
 559                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 560                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 561             return;
 562         }
 563
 564         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 565                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 566         for (Uuid routerId : routerIds) {
 567             String routerName = routerId.getValue();
 568             Uint64 primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 569             if (Objects.equals(primarySwitch, dpId)) {
 570                 Routers router = vpnUtil.getExternalRouter(routerName);
 571                 if (router != null) {
 572                     if (addOrRemove == NwConstants.ADD_FLOW) {
 573                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 574                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 575                                 dpId, interfaceName, lportTag);
 576                     } else {
 577                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 578                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 579                                 dpId, interfaceName, lportTag);
 580                     }
 581                 } else {
 582                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 583                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 584                             interfaceName, dpId, vpnName);
 585                 }
 586             }
 587         }
 588     }
 589
 590     // TODO Clean up the exception handling
 591     @SuppressWarnings("checkstyle:IllegalCatch")
 592     private void advertiseAdjacenciesForVpnToBgp(final String rd, Uint64 dpnId,
 593                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 594                                                   String vpnName, String interfaceName) {
 595         if (rd == null) {
 596             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 597                   interfaceName, dpnId, vpnName);
 598             return;
 599         } else {
 600             if (rd.equals(vpnName)) {
 601                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 602                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 603                 return;
 604             }
 605         }
 606         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 607                 interfaceName, dpnId, vpnName, rd);
 608
 609         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 610         if (nextHopIp == null) {
 611             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 612                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 613                     rd, vpnName);
 614             return;
 615         }
 616
 617         try {
 618             //Read NextHops
 619             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 620             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 621                     LogicalDatastoreType.OPERATIONAL, path);
 622             if (adjacencies.isPresent()) {
 623                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 624                 if (nextHops != null && !nextHops.isEmpty()) {
 625                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 626                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 627                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 628                     Uint32 l3vni = vpnInstanceOpData.getL3vni();
 629                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 630                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 631                     for (Adjacency nextHop : nextHops) {
 632                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 633                             continue;
 634                         }
 635                         String gatewayMac = null;
 636                         Uint32 label = Uint32.ZERO;
 637                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 638                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 639                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 640                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 641                                     .get();
 642                         } else {
 643                             label = nextHop.getLabel();
 644                         }
 645                         try {
 646                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 647                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 648                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 649                                     encapType, label, l3vni, Uint32.ZERO /*l2vni*/,
 650                                     gatewayMac);
 651                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 652                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 653                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 654                         } catch (Exception e) {
 655                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 656                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 657                                     interfaceName, dpnId, e);
 658                         }
 659                     }
 660                 }
 661             }
 662         } catch (ReadFailedException e) {
 663             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 664                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 665         }
 666     }
 667
 668     // TODO Clean up the exception handling
 669     @SuppressWarnings("checkstyle:IllegalCatch")
 670     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 671                         String vpnName, String interfaceName, TypedWriteTransaction<Configuration> writeConfigTxn,
 672                         TypedWriteTransaction<Operational> writeOperTx) {
 673         //Read NextHops
 674         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 675         String rd = vpnUtil.getVpnRd(interfaceName);
 676         if (rd == null) {
 677             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 678                 interfaceName, vpnName);
 679             return;
 680         } else {
 681             if (rd.equals(vpnName)) {
 682                 LOG.info(
 683                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 684                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 685                 return;
 686             }
 687         }
 688         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 689                vpnName, rd);
 690         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 691         try {
 692             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 693                     path);
 694         } catch (ReadFailedException e) {
 695             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 696                     interfaceName, vpnName);
 697         }
 698         if (adjacencies.isPresent()) {
 699             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 700
 701             if (nextHops != null && !nextHops.isEmpty()) {
 702                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 703                         nextHops, interfaceName, vpnName, rd);
 704                 for (Adjacency nextHop : nextHops) {
 705                     try {
 706                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 707                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 708                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 709                                     interfaceName, vpnName);
 710                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 711                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 712                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 713                                     interfaceName, vpnName);
 714                         } else if (nextHop.getNextHopIpList() != null) {
 715                             // Perform similar operation as interface delete event for extraroutes.
 716                             String allocatedRd = nextHop.getVrfId();
 717                             for (String nh : nextHop.getNextHopIpList()) {
 718                                 deleteExtraRouteFromCurrentAndImportingVpns(
 719                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
 720                                     writeOperTx);
 721                             }
 722                         }
 723                     } catch (Exception e) {
 724                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 725                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 726                     }
 727                 }
 728             }
 729         }
 730     }
 731
 732     @SuppressWarnings("checkstyle:IllegalCatch")
 733     protected void processVpnInterfaceAdjacencies(Uint64 dpnId, final int lportTag, String vpnName,
 734                                                   String primaryRd, String interfaceName, final Uint32 vpnId,
 735                                                   TypedWriteTransaction<Configuration> writeConfigTxn,
 736                                                   TypedWriteTransaction<Operational> writeOperTxn,
 737                                                   TypedReadWriteTransaction<Configuration> writeInvTxn,
 738                                                   Interface interfaceState, Set<String> prefixListForRefreshFib)
 739             throws ExecutionException, InterruptedException {
 740         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 741         // Read NextHops
 742         Optional<VpnInterface> vpnInteface = Optional.absent();
 743         try {
 744             vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 745             LogicalDatastoreType.CONFIGURATION, identifier);
 746         } catch (ReadFailedException e) {
 747             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 748                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 749         }
 750         Uuid intfnetworkUuid = null;
 751         NetworkType networkType = null;
 752         Long segmentationId = Long.valueOf(-1);
 753         Adjacencies adjacencies = null;
 754         if (vpnInteface.isPresent()) {
 755             intfnetworkUuid = vpnInteface.get().getNetworkId();
 756             networkType = vpnInteface.get().getNetworkType();
 757             segmentationId = vpnInteface.get().getSegmentationId().toJava();
 758             adjacencies = vpnInteface.get().augmentation(Adjacencies.class);
 759             if (adjacencies == null) {
 760                 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 761                         null/*gwMac*/,  null/*gatewayIp*/, writeOperTxn);
 762                 return;
 763             }
 764         }
 765         // Get the rd of the vpn instance
 766         String nextHopIp = null;
 767         String gatewayIp = null;
 768         try {
 769             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 770         } catch (Exception e) {
 771             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 772                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 773         }
 774         List<String> nhList = new ArrayList<>();
 775         if (nextHopIp != null) {
 776             nhList.add(nextHopIp);
 777             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 778                     interfaceName, dpnId, vpnName, nhList);
 779         }
 780         Optional<String> gwMac = Optional.absent();
 781         String vpnInterfaceSubnetGwMacAddress = null;
 782         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 783         Uint32 l3vni = vpnInstanceOpData.getL3vni() != null ? vpnInstanceOpData.getL3vni() : Uint32.ZERO;
 784         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 785         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 786         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 787         List<Adjacency> nextHops = adjacencies != null ? adjacencies.getAdjacency() : emptyList();
 788         List<Adjacency> value = new ArrayList<>();
 789         for (Adjacency nextHop : nextHops) {
 790             String rd = primaryRd;
 791             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 792             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 793                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 794                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 795                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 796                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 797                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 798                 continue;
 799             }
 800             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 801                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 802                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 803                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 804                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 805                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 806
 807                 Prefixes prefixes = intfnetworkUuid != null
 808                     ? VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, intfnetworkUuid ,networkType,
 809                             segmentationId, prefixCue) :
 810                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, prefixCue);
 811                 writeOperTxn.merge(VpnUtil.getPrefixToInterfaceIdentifier(
 812                         vpnUtil.getVpnId(vpnName), prefix), prefixes, true);
 813                 final Uuid subnetId = nextHop.getSubnetId();
 814
 815                 gatewayIp = nextHop.getSubnetGatewayIp();
 816                 if (gatewayIp == null) {
 817                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 818                     if (gatewayIpOptional.isPresent()) {
 819                         gatewayIp = gatewayIpOptional.get();
 820                     }
 821                 }
 822
 823                 if (gatewayIp != null) {
 824                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 825                     if (gwMac.isPresent()) {
 826                         // A valid mac-address is available for this subnet-gateway-ip
 827                         // Use this for programming ARP_RESPONDER table here.  And save this
 828                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 829                         // to populate L3_GW_MAC_TABLE there.
 830                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 831                                 gatewayIp, gwMac.get());
 832                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 833                     } else {
 834                         // A valid mac-address is not available for this subnet-gateway-ip
 835                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 836                         // Save this connected-mac-address as gateway-mac-address for the
 837                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 838                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 839                         if (gwMac.isPresent()) {
 840                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 841                                     NwConstants.ADD_FLOW, gwMac.get());
 842                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 843                                     gatewayIp, gwMac.get());
 844                         } else {
 845                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 846                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 847                                 + "gwIp {}",
 848                                 subnetId, interfaceName, vpnName, gatewayIp);
 849                         }
 850                     }
 851                 } else {
 852                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 853                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 854                         subnetId, interfaceName, vpnName);
 855                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 856                 }
 857                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 858                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 859             } else {
 860                 //Extra route adjacency
 861                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 862                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 863                 // FIXME: separate this out somehow?
 864                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
 865                 lock.lock();
 866                 try {
 867                     java.util.Optional<String> rdToAllocate = vpnUtil
 868                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 869                                     nextHop.getNextHopIpList().get(0), dpnId);
 870                     if (rdToAllocate.isPresent()) {
 871                         rd = rdToAllocate.get();
 872                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 873                             rd, prefix);
 874                     } else {
 875                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 876                         continue;
 877                     }
 878                 } finally {
 879                     lock.unlock();
 880                 }
 881                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 882                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 883                         interfaceName, dpnId);
 884             }
 885             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 886             // can be different from the gateway-mac-address within the VRFEntry as the
 887             // gateway-mac-address is a superset.
 888             RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 889             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 890                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 891                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 892             Adjacency operationalAdjacency = null;
 893             try {
 894                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 895             } catch (NullPointerException e) {
 896                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 897                     input, e.getMessage());
 898                 return;
 899             }
 900             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 901                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 902                     vpnName, l3vni, origin, interfaceName, operationalAdjacency, encapType, prefixListForRefreshFib,
 903                     writeConfigTxn);
 904             }
 905             value.add(operationalAdjacency);
 906         }
 907
 908         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 909         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 910                 gwMac.isPresent() ? gwMac.get() : null, gatewayIp, writeOperTxn);
 911
 912         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni.longValue()).setPrimaryRd(primaryRd)
 913                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 914                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 915
 916         for (Adjacency nextHop : aug.getAdjacency()) {
 917             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 918             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 919                 RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 920                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 921                 registeredPopulator.populateFib(input, writeConfigTxn);
 922             }
 923         }
 924     }
 925
 926     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, Uint64 dpnId, AdjacenciesOp aug,
 927                                               long lportTag, String gwMac, String gwIp,
 928                                               TypedWriteTransaction<Operational> writeOperTxn) {
 929         VpnInterfaceOpDataEntry opInterface =
 930               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac, gwIp);
 931         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 932             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 933         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
 934         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 935                 interfaceName, dpnId, vpnName);
 936     }
 937
 938     // TODO Clean up the exception handling
 939     @SuppressWarnings("checkstyle:IllegalCatch")
 940     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 941                                            StateTunnelList stateTunnelList,
 942                                            TypedWriteTransaction<Configuration> writeConfigTxn,
 943                                            TypedWriteTransaction<Operational> writeOperTxn) {
 944
 945         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
 946         Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern();
 947         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 948         List<Adjacency> adjList =
 949             adjacencies != null && adjacencies.getAdjacency() != null ? adjacencies.getAdjacency() : emptyList();
 950         if (adjList.isEmpty()) {
 951             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 952                     vpnInterface, srcDpnId);
 953             return;
 954         }
 955         String prefix = null;
 956         List<Adjacency> value = new ArrayList<>();
 957         boolean isFibNextHopAddReqd = false;
 958         String vpnName = vpnInterface.getVpnInstanceName();
 959         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 960         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 961         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 962                 vpnInterface.getName(), vpnInterface.getDpnId(),
 963                 vpnInterface.getVpnInstanceName(), adjList);
 964         for (Adjacency adj : adjList) {
 965             String rd = adj.getVrfId();
 966             rd = rd != null ? rd : vpnName;
 967             prefix = adj.getIpAddress();
 968             Uint32 label = adj.getLabel();
 969             List<String> nhList = Collections.singletonList(srcTepIp);
 970             List<String> nextHopList = adj.getNextHopIpList();
 971             // If TEP is added , update the nexthop of primary adjacency.
 972             // Secondary adj nexthop is already pointing to primary adj IP address.
 973             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 974                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 975                 if (nextHopList != null && !nextHopList.isEmpty()) {
 976                     /* everything right already */
 977                 } else {
 978                     isFibNextHopAddReqd = true;
 979                 }
 980             } else {
 981                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 982                 if (!vrfEntryOptional.isPresent()) {
 983                     continue;
 984                 }
 985                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 986                 if (!nhList.contains(srcTepIp)) {
 987                     nhList.add(srcTepIp);
 988                     isFibNextHopAddReqd = true;
 989                 }
 990                 value.add(adj);
 991             }
 992
 993             if (isFibNextHopAddReqd) {
 994                 updateLabelMapper(label, nhList);
 995                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 996                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 997                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 998                 // Update the VRF entry with nextHop
 999                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1000                         label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1001
1002                 //Get the list of VPN's importing this route(prefix) .
1003                 // Then update the VRF entry with nhList
1004                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1005                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1006                     String vpnRd = vpn.getVrfId();
1007                     if (vpnRd != null) {
1008                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1009                             srcTepIp, label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1010                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
1011                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
1012                             vpnInterface.getName(), srcDpnId, vpnName,
1013                             vpn.getVpnInstanceName(), vpnRd);
1014                     }
1015                 }
1016                 // Advertise the prefix to BGP only for external vpn
1017                 // since there is a nexthop change.
1018                 try {
1019                     if (!rd.equalsIgnoreCase(vpnName)) {
1020                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
1021                                 VrfEntry.EncapType.Mplsgre, label, Uint32.ZERO /*evi*/, Uint32.ZERO /*l2vni*/,
1022                                 null /*gatewayMacAddress*/);
1023                     }
1024                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
1025                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
1026                             srcDpnId, vpnName);
1027                 } catch (Exception ex) {
1028                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
1029                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1030                             vpnInterface.getName(), srcDpnId, vpnName, ex);
1031                 }
1032             }
1033         }
1034         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1035         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1036                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1037                 .addAugmentation(AdjacenciesOp.class, aug).build();
1038         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1039                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1040         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1041         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
1042                 vpnInterface.getName(), srcDpnId, vpnName);
1043
1044     }
1045
1046     // TODO Clean up the exception handling
1047     @SuppressWarnings("checkstyle:IllegalCatch")
1048     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1049                                               StateTunnelList stateTunnelList,
1050                                               TypedWriteTransaction<Configuration> writeConfigTxn,
1051                                               TypedWriteTransaction<Operational> writeOperTxn) {
1052
1053         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1054         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1055         String prefix = null;
1056         boolean isNextHopRemoveReqd = false;
1057         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
1058         Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern();
1059         String vpnName = vpnInterface.getVpnInstanceName();
1060         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1061         String primaryRd = vpnUtil.getVpnRd(vpnName);
1062         if (adjList != null) {
1063             List<Adjacency> value = new ArrayList<>();
1064             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1065                     vpnInterface.getName(), vpnInterface.getDpnId(),
1066                     vpnInterface.getVpnInstanceName(), adjList);
1067             for (Adjacency adj : adjList) {
1068                 List<String> nhList = new ArrayList<>();
1069                 String rd = adj.getVrfId();
1070                 rd = rd != null ? rd : vpnName;
1071                 prefix = adj.getIpAddress();
1072                 List<String> nextHopList = adj.getNextHopIpList();
1073                 Uint32 label = adj.getLabel();
1074                 if (nextHopList != null && !nextHopList.isEmpty()) {
1075                     isNextHopRemoveReqd = true;
1076                 }
1077                 // If TEP is deleted , remove the nexthop from primary adjacency.
1078                 // Secondary adj nexthop will continue to point to primary adj IP address.
1079                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1080                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1081                 } else {
1082                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1083                     if (!vrfEntryOptional.isPresent()) {
1084                         continue;
1085                     }
1086                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1087                     if (nhList.contains(srcTepIp)) {
1088                         nhList.remove(srcTepIp);
1089                         isNextHopRemoveReqd = true;
1090                     }
1091                     value.add(adj);
1092                 }
1093
1094                 if (isNextHopRemoveReqd) {
1095                     updateLabelMapper(label, nhList);
1096                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1097                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1098                             prefix, nhList, vpnName,
1099                             vpnId, rd, vpnInterface.getName());
1100                     // Update the VRF entry with removed nextHop
1101                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1102                             label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1103
1104                     //Get the list of VPN's importing this route(prefix) .
1105                     // Then update the VRF entry with nhList
1106                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1107                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1108                         String vpnRd = vpn.getVrfId();
1109                         if (vpnRd != null) {
1110                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1111                                 srcTepIp, label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1112                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1113                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1114                                     nhList, label, vpnInterface.getName(), srcDpnId,
1115                                     vpnName,
1116                                     vpn.getVpnInstanceName(), vpnRd);
1117                         }
1118                     }
1119
1120                     // Withdraw prefix from BGP only for external vpn.
1121                     try {
1122                         if (!rd.equalsIgnoreCase(vpnName)) {
1123                             bgpManager.withdrawPrefix(rd, prefix);
1124                         }
1125                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1126                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1127                                 vpnInterface.getName(), srcDpnId,
1128                                 vpnName);
1129                     } catch (Exception ex) {
1130                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1131                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1132                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1133                     }
1134                 }
1135             }
1136             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1137             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1138                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1139                     .addAugmentation(AdjacenciesOp.class, aug).build();
1140             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1141                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1142             writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1143             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1144                          vpnInterface.getName(), srcDpnId, vpnName);
1145         }
1146     }
1147
1148     @SuppressWarnings("checkstyle:IllegalCatch")
1149     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1150         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1151         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry;
1152         String vpnRd = vpnUtil.getVpnRd(vpnName);
1153         try {
1154             VpnInstanceOpDataEntry opDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1155             if (opDataEntry == null) {
1156                 LOG.error("getVpnsExportingMyRoute: Null vpn instance op data for vpn {} rd {}"
1157                         + " when check for vpns exporting the routes", vpnName, vpnRd);
1158                 return vpnsToExportRoute;
1159             }
1160             vpnInstanceOpDataEntry = opDataEntry;
1161         } catch (Exception re) {
1162             LOG.error("getVpnsExportingMyRoute: DSexception when retrieving vpn instance op data for vpn {} rd {}"
1163                     + " to check for vpns exporting the routes", vpnName, vpnRd, re);
1164             return vpnsToExportRoute;
1165         }
1166         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1167             if (input.getVpnInstanceName() == null) {
1168                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1169                         input.getVrfId());
1170                 return false;
1171             }
1172             return !input.getVpnInstanceName().equals(vpnName);
1173         };
1174
1175         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1176             Iterable<String> commonRTs =
1177                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1178                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1179             return Iterators.size(commonRTs.iterator()) > 0;
1180         };
1181
1182         vpnsToExportRoute =
1183                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1184                         Collectors.toList());
1185         return vpnsToExportRoute;
1186     }
1187
1188     // TODO Clean up the exception handling
1189     @SuppressWarnings("checkstyle:IllegalCatch")
1190     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1191         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1192         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1193             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1194             if (vrfEntries != null) {
1195                 ListenableFutures.addErrorLogging(
1196                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1197                         for (VrfEntry vrfEntry : vrfEntries) {
1198                             try {
1199                                 if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1200                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1201                                     LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1202                                             + " is not a controller managed non intervpn link route. Ignoring.",
1203                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1204                                     continue;
1205                                 }
1206                                 String prefix = vrfEntry.getDestPrefix();
1207                                 String gwMac = vrfEntry.getGatewayMacAddress();
1208                                 vrfEntry.nonnullRoutePaths().forEach(routePath -> {
1209                                     String nh = routePath.getNexthopAddress();
1210                                     Uint32 label = routePath.getLabel();
1211                                     if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1212                                         vrfEntry.getOrigin()))) {
1213                                         LOG.info(
1214                                             "handleVpnsExportingRoutesImporting: Importing fib entry rd {}"
1215                                                 + " prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1216                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1217                                         fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1218                                             Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1219                                                 Uint32.ZERO /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1220                                             confTx);
1221                                     } else {
1222                                         LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry"
1223                                                 + " rd {} prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1224                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1225                                         SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1226                                         importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1227                                             confTx);
1228                                     }
1229                                 });
1230                             } catch (RuntimeException e) {
1231                                 LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1232                                         + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1233                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1234                             }
1235                         }
1236                     }), LOG, "Error handing VPN exporting routes");
1237             } else {
1238                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1239                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1240             }
1241         }
1242     }
1243
1244     @Override
1245     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1246         LOG.trace("Received VpnInterface remove event: vpnInterface={}", vpnInterface);
1247         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1248         final String interfaceName = key.getName();
1249         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
1250             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1251             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1252         }
1253     }
1254
1255     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1256                                 final VpnInterface vpnInterface, final String vpnName,
1257                                 final String interfaceName) {
1258         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1259             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1260                 ListenableFuture<Void> future =
1261                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1262                         deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1263                         LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1264                     });
1265                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1266                         vpnInterface.getName(), vpnName);
1267                 return Collections.singletonList(future);
1268             }, DJC_MAX_RETRIES);
1269         } else {
1270             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1271             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1272         }
1273     }
1274
1275     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1276     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1277                                            final VpnInterface vpnInterface, final String vpnName,
1278                                            final String interfaceName, final Interface interfaceState) {
1279         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1280                 vpnName, vpnInterface.getDpnId());
1281         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1282         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1283             () -> {
1284                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1285                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1286                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1287                         writeOperTxn -> futures.add(
1288                             txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeInvTxn -> {
1289                                 LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1290                                         interfaceName, vpnName);
1291                                 Uint64 dpId;
1292                                 int ifIndex;
1293                                 String gwMacAddress;
1294                                 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1295                                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1296                                 Optional<VpnInterfaceOpDataEntry> optVpnInterface;
1297                                 try {
1298                                     optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1299                                             LogicalDatastoreType.OPERATIONAL, interfaceId);
1300                                 } catch (ReadFailedException e) {
1301                                     LOG.error("remove: Failed to read data store for interface {} vpn {}",
1302                                             interfaceName, vpnName);
1303                                     return;
1304                                 }
1305                                 if (interfaceState != null) {
1306                                     try {
1307                                         dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1308                                     } catch (NumberFormatException | IllegalStateException e) {
1309                                         LOG.error("remove: Unable to retrieve dpnId from interface operational"
1310                                                         + " data store for interface {} on dpn {} for vpn {} Fetching"
1311                                                         + " from vpn interface op data store. ", interfaceName,
1312                                                 vpnInterface.getDpnId(), vpnName, e);
1313                                         dpId = Uint64.ZERO;
1314                                     }
1315                                     ifIndex = interfaceState.getIfIndex();
1316                                     gwMacAddress = interfaceState.getPhysAddress().getValue();
1317                                 } else {
1318                                     LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1319                                             + " from vpn interface op.", interfaceName);
1320                                     if (optVpnInterface.isPresent()) {
1321                                         VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1322                                         dpId = vpnOpInterface.getDpnId();
1323                                         ifIndex = vpnOpInterface.getLportTag().intValue();
1324                                         gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1325                                     } else {
1326                                         LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1327                                                 + " as interfaceState and vpn interface op is not"
1328                                                 + " available", interfaceName, vpnName);
1329                                         return;
1330                                     }
1331                                 }
1332                                 processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1333                                         optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1334                                         writeConfigTxn, writeOperTxn, writeInvTxn);
1335                                 LOG.info(
1336                                         "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1337                                                 + "successfully",
1338                                         interfaceName, vpnInterface.getDpnId(), vpnName);
1339                             })))));
1340                 futures.add(configFuture);
1341                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(
1342                         interfaceName, false, "Config"), MoreExecutors.directExecutor());
1343                 return futures;
1344             }, DJC_MAX_RETRIES);
1345     }
1346
1347     protected void processVpnInterfaceDown(Uint64 dpId,
1348                                            String interfaceName,
1349                                            int lportTag,
1350                                            String gwMac,
1351                                            VpnInterfaceOpDataEntry vpnOpInterface,
1352                                            boolean isInterfaceStateDown,
1353                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1354                                            TypedWriteTransaction<Operational> writeOperTxn,
1355                                            TypedReadWriteTransaction<Configuration> writeInvTxn)
1356             throws ExecutionException, InterruptedException {
1357         if (vpnOpInterface == null) {
1358             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1359                     + " as it is not available in operational data store", interfaceName, dpId);
1360             return;
1361         }
1362         final String vpnName = vpnOpInterface.getVpnInstanceName();
1363         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1364                                                     interfaceName, vpnName);
1365         if (!isInterfaceStateDown) {
1366             final Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1367             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName,  null);
1368             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1369             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1370                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1371             if (interfaceManager.isExternalInterface(interfaceName)) {
1372                 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1373                     NwConstants.DEL_FLOW);
1374             }
1375             if (!isBgpVpnInternetVpn) {
1376                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1377             }
1378             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1379                     + " successful", interfaceName, dpId, vpnName);
1380         } else {
1381             // Interface is retained in the DPN, but its Link Down.
1382             // Only withdraw the prefixes for this interface from BGP
1383             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1384         }
1385     }
1386
1387     private void removeAdjacenciesFromVpn(final Uint64 dpnId, final int lportTag, final String interfaceName,
1388                                           final String vpnName, final Uint32 vpnId, String gwMac,
1389                                           TypedWriteTransaction<Configuration> writeConfigTxn,
1390                                           TypedWriteTransaction<Operational> writeOperTxn,
1391                                           TypedReadWriteTransaction<Configuration> writeInvTxn)
1392             throws ExecutionException, InterruptedException {
1393         //Read NextHops
1394         try {
1395             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1396                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1397             Optional<VpnInterfaceOpDataEntry> vpnInterfaceOpDataEnteryOptional =
1398                     SingleTransactionDataBroker.syncReadOptional(dataBroker,
1399                             LogicalDatastoreType.OPERATIONAL, identifier);
1400             boolean isNonPrimaryAdjIp = false;
1401             String primaryRd = vpnUtil.getVpnRd(vpnName);
1402             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1403                     interfaceName, dpnId, vpnName, primaryRd);
1404             if (!vpnInterfaceOpDataEnteryOptional.isPresent()) {
1405                 LOG.error("removeAdjacenciesFromVpn: VpnInterfaceOpDataEntry-Oper DS is absent for Interface {} "
1406                         + "on vpn {} dpn {}", interfaceName, vpnName, dpnId);
1407                 return;
1408             }
1409             AdjacenciesOp adjacencies = vpnInterfaceOpDataEnteryOptional.get().augmentation(AdjacenciesOp.class);
1410
1411             if (adjacencies != null && !adjacencies.getAdjacency().isEmpty()) {
1412                 List<Adjacency> nextHops = adjacencies.getAdjacency();
1413                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1414                         interfaceName, dpnId, vpnName, nextHops);
1415                 for (Adjacency nextHop : nextHops) {
1416                     if (nextHop.isPhysNetworkFunc()) {
1417                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1418                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1419                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1420                                 null, null/*writeCfgTxn*/);
1421                     } else {
1422                         String rd = nextHop.getVrfId();
1423                         List<String> nhList;
1424                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1425                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1426                             isNonPrimaryAdjIp = Boolean.TRUE;
1427                         } else {
1428                             // This is a primary adjacency
1429                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1430                                     : emptyList();
1431                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1432                                     vpnInterfaceOpDataEnteryOptional.get().getGatewayIpAddress(),
1433                                     interfaceName, writeInvTxn);
1434                             isNonPrimaryAdjIp = Boolean.FALSE;
1435                         }
1436                         if (!nhList.isEmpty()) {
1437                             if (Objects.equals(primaryRd, vpnName)) {
1438                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1439                                 //remove from FIB directly
1440                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1441                                         interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1442                             } else {
1443                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1444                                         interfaceName, isNonPrimaryAdjIp, writeConfigTxn, writeOperTxn);
1445                             }
1446                         } else {
1447                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1448                                             + " interface {}", nextHop.getIpAddress(), rd,
1449                                     nextHop.getAdjacencyType().toString(), interfaceName);
1450                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1451                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), null, writeConfigTxn);
1452                         }
1453                     }
1454                     String ip = nextHop.getIpAddress().split("/")[0];
1455                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1456                     if (vpnVipToPort != null && vpnVipToPort.getPortName().equals(interfaceName)) {
1457                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1458                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1459                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1460                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1461                     }
1462                     // Remove the MIP-IP from VpnPortIpToPort.
1463                     if (isNonPrimaryAdjIp) {
1464                         VpnPortipToPort persistedIp = vpnUtil.getVpnPortipToPort(vpnName, ip);
1465                         if (persistedIp != null && persistedIp.isLearntIp()
1466                                 && persistedIp.getPortName().equals(interfaceName)) {
1467                             VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1468                             LOG.info(
1469                                     "removeAdjacenciesFromVpn: Learnt-IP: {} interface {} of vpn {} removed "
1470                                             + "from VpnPortipToPort",
1471                                     persistedIp.getPortFixedip(), persistedIp.getPortName(), vpnName);
1472                         }
1473                     }
1474                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1475                     if (vpnPortipToPort != null) {
1476                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1477                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1478                                  + "Interface {} ip {} on dpn {} for vpn {}",
1479                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1480                     }
1481                 }
1482             } else {
1483                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1484                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1485                         + " Removing it.", interfaceName, vpnName, dpnId);
1486                 writeOperTxn.delete(identifier);
1487             }
1488         } catch (ReadFailedException e) {
1489             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1490                     interfaceName, dpnId, vpnName);
1491         }
1492     }
1493
1494     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1495                                                             String interfaceName, Uint64 dpnId,
1496                                                             TypedWriteTransaction<Configuration> writeConfigTxn,
1497                                                             TypedWriteTransaction<Operational> writeOperTx) {
1498         return (nh) -> {
1499             String primaryRd = vpnUtil.getVpnRd(vpnName);
1500             String prefix = nextHop.getIpAddress();
1501             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1502             LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1503                     nextHop, vpnName, interfaceName, dpnId);
1504             // FIXME: separate this out somehow?
1505             final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnNamePrefixKey);
1506             lock.lock();
1507             try {
1508                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1509                         prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1510                     //If extra-route is present behind at least one VM, then do not remove or update
1511                     //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1512                     //prefixtointerface DS
1513                     return;
1514                 }
1515                 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1516                         writeConfigTxn);
1517             } finally {
1518                 lock.unlock();
1519             }
1520             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1521                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1522                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1523         };
1524     }
1525
1526     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1527                                            Uint64 dpnId, String rd, String interfaceName, boolean isNonPrimaryAdjIp,
1528                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1529                                            TypedWriteTransaction<Operational> writeOperTx) {
1530         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1531                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1532         nhList.forEach((nh) -> {
1533             //IRT: remove routes from other vpns importing it
1534             if (isNonPrimaryAdjIp) {
1535                 removeLearntPrefixFromBGP(rd, nextHop.getIpAddress(), nh, writeConfigTxn);
1536             } else {
1537                 vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1538                         nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1539             }
1540             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1541                 String vpnRd = vpn.getVrfId();
1542                 if (vpnRd != null) {
1543                     fibManager.removeOrUpdateFibEntry(vpnRd,
1544                             nextHop.getIpAddress(), nh, writeConfigTxn);
1545                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1546                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1547                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1548                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1549                 }
1550             }
1551         });
1552     }
1553
1554     @SuppressWarnings("checkstyle:IllegalCatch")
1555     private void removeLearntPrefixFromBGP(String rd, String prefix, String nextHop,
1556                                            TypedWriteTransaction<Configuration> writeConfigTxn) {
1557         try {
1558             if (!fibManager.checkFibEntryExist(dataBroker, rd, prefix, nextHop)) {
1559                 LOG.info("removeLearntPrefixFromBGP: IP {} with nexthop {} rd {} is already removed.Ignoring this"
1560                         + " operation", prefix, nextHop, rd);
1561                 return;
1562             }
1563             LOG.info("removeLearntPrefixFromBGP: VPN WITHDRAW: Removing Fib Entry rd {} prefix {} nexthop {}",
1564                     rd, prefix, nextHop);
1565             fibManager.removeOrUpdateFibEntry(rd, prefix, nextHop, writeConfigTxn);
1566             bgpManager.withdrawPrefix(rd, prefix); // TODO: Might be needed to include nextHop here
1567             LOG.info("removeLearntPrefixFromBGP: VPN WITHDRAW: Removed Fib Entry rd {} prefix {} nexthop {}",
1568                     rd, prefix, nextHop);
1569         } catch (Exception e) {
1570             LOG.error("removeLearntPrefixFromBGP: Delete prefix {} rd {} nextHop {} failed", prefix, rd, nextHop, e);
1571         }
1572     }
1573
1574     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, Uint32 vpnId, Uint64 dpnId,
1575                                                  int lportTag, String gwMac, String gwIp, String interfaceName,
1576                                                  TypedReadWriteTransaction<Configuration> writeInvTxn)
1577             throws ExecutionException, InterruptedException {
1578         final Uuid subnetId = nextHop.getSubnetId();
1579         if (nextHop.getSubnetGatewayMacAddress() == null) {
1580             // A valid mac-address was not available for this subnet-gateway-ip
1581             // So a connected-mac-address was used for this subnet and we need
1582             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1583             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1584         }
1585         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, gwIp,
1586                 subnetId);
1587     }
1588
1589     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, Uint64 dpnId,
1590                                                   String interfaceName) {
1591         // This is either an extra-route (or) a learned IP via subnet-route
1592         List<String> nhList = null;
1593         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1594         if (nextHopIp == null || nextHopIp.isEmpty()) {
1595             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1596                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1597                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1598                     vpnName);
1599             nhList = emptyList();
1600         } else {
1601             nhList = Collections.singletonList(nextHopIp);
1602         }
1603         return nhList;
1604     }
1605
1606     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1607         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1608         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1609         // else use connected interface
1610         if (gwPort != null && gwPort.isSubnetIp()) {
1611             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1612                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1613             return Optional.of(gwPort.getMacAddress());
1614         }
1615         return Optional.absent();
1616     }
1617
1618     @Override
1619     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1620         final VpnInterface update) {
1621         LOG.trace("Received VpnInterface update event: original={}, update={}", original, update);
1622         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}", update.getName(),
1623                 update.getDpnId(), original.getVpnInstanceNames(), update.getVpnInstanceNames());
1624         if (original.equals(update)) {
1625             LOG.info("update: original {} update {} are same. No update required.", original, update);
1626             return;
1627         }
1628         final String vpnInterfaceName = update.getName();
1629         final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1630         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1631         //handles switching between <internal VPN - external VPN>
1632         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1633             List<ListenableFuture<Void>> futures = new ArrayList<>();
1634             if (handleVpnInstanceUpdateForVpnInterface(identifier, original, update, futures)) {
1635                 LOG.info("update: handled Instance update for VPNInterface {} on dpn {} from oldVpn(s) {} "
1636                                 + "to newVpn(s) {}",
1637                         original.getName(), dpnId,
1638                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1639                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1640                 return emptyList();
1641             }
1642             updateVpnInstanceAdjChange(original, update, vpnInterfaceName, futures);
1643             return futures;
1644         });
1645     }
1646
1647     private boolean handleVpnInstanceUpdateForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1648                                                            VpnInterface original, VpnInterface update,
1649                                                            List<ListenableFuture<Void>> futures) {
1650         boolean isVpnInstanceUpdate = false;
1651         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1652         final String interfaceName = key.getName();
1653         List<String> oldVpnList = VpnUtil.getVpnListForVpnInterface(original);
1654         List<String> oldVpnListCopy = new ArrayList<>();
1655         oldVpnListCopy.addAll(oldVpnList);
1656         List<String> newVpnList = VpnUtil.getVpnListForVpnInterface(update);
1657         List<String> newVpnListCopy = new ArrayList<>();
1658         newVpnListCopy.addAll(newVpnList);
1659
1660         oldVpnList.removeAll(newVpnList);
1661         newVpnList.removeAll(oldVpnListCopy);
1662         //This block will execute only on if there is a change in the VPN Instance.
1663         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1664             /*
1665              * Internet BGP-VPN Instance update with single router:
1666              * ====================================================
1667              * In this case single VPN Interface will be part of maximum 2 VPN Instance only.
1668              *     1st VPN Instance : router VPN or external BGP-VPN.
1669              *     2nd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1670              *
1671              * VPN Instance UPDATE:
1672              * oldVpnList = 0 and newVpnList = 1 (Internet BGP-VPN)
1673              * oldVpnList = 1 and newVpnList = 0 (Internet BGP-VPN)
1674              *
1675              * External BGP-VPN Instance update with single router:
1676              * ====================================================
1677              * In this case single VPN interface will be part of maximum 1 VPN Instance only.
1678              *
1679              * Updated VPN Instance will be always either internal router VPN to
1680              * external BGP-VPN or external BGP-VPN to internal router VPN swap.
1681              *
1682              * VPN Instance UPDATE:
1683              * oldVpnList = 1 and newVpnList = 1 (router VPN to Ext-BGPVPN)
1684              * oldVpnList = 1 and newVpnList = 1 (Ext-BGPVPN to router VPN)
1685              *
1686              * Dual Router VPN Instance Update:
1687              * ================================
1688              * In this case single VPN interface will be part of maximum 3 VPN Instance only.
1689              *
1690              * 1st VPN Instance : router VPN or external BGP-VPN-1.
1691              * 2nd VPN Instance : router VPN or external BGP-VPN-2.
1692              * 3rd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1693              *
1694              * Dual Router --> Associated with common external BGP-VPN Instance.
1695              * 1st router and 2nd router are getting associated with single External BGP-VPN
1696              * 1) add 1st router to external bgpvpn --> oldVpnList=1, newVpnList=1;
1697              * 2) add 2nd router to the same external bgpvpn --> oldVpnList=1, newVpnList=0
1698              * In this case, we need to call removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1699              *
1700              *
1701              */
1702             isVpnInstanceUpdate = true;
1703             if (VpnUtil.isDualRouterVpnUpdate(oldVpnListCopy, newVpnListCopy)) {
1704                 if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1705                         && oldVpnList.size() == 1 && newVpnList.isEmpty()) {
1706                     //Identify the external BGP-VPN Instance and pass that value as newVpnList
1707                     List<String> externalBgpVpnList = new ArrayList<>();
1708                     for (String newVpnName : newVpnListCopy) {
1709                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1710                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1711                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1712                                 .BgpvpnType.BGPVPNExternal) {
1713                             externalBgpVpnList.add(newVpnName);
1714                             break;
1715                         }
1716                     }
1717                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1718                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList,
1719                             externalBgpVpnList, oldVpnListCopy, futures);
1720
1721                 } else if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1722                         && oldVpnList.isEmpty() && newVpnList.size() == 1) {
1723                     //Identify the router VPN Instance and pass that value as oldVpnList
1724                     List<String> routerVpnList = new ArrayList<>();
1725                     for (String newVpnName : newVpnListCopy) {
1726                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1727                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1728                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1729                                 .BgpvpnType.VPN) {
1730                             routerVpnList.add(newVpnName);
1731                             break;
1732                         }
1733                     }
1734                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1735                     updateVpnInstanceChange(identifier, interfaceName, original, update, routerVpnList,
1736                             newVpnList, oldVpnListCopy, futures);
1737
1738                 } else {
1739                     //Handle remaining use cases.
1740                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1741                             oldVpnListCopy, futures);
1742                 }
1743             } else {
1744                 updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1745                         oldVpnListCopy, futures);
1746             }
1747         }
1748         return isVpnInstanceUpdate;
1749     }
1750
1751     private void updateVpnInstanceChange(InstanceIdentifier<VpnInterface> identifier, String interfaceName,
1752                                          VpnInterface original, VpnInterface update, List<String> oldVpnList,
1753                                          List<String> newVpnList, List<String> oldVpnListCopy,
1754                                          List<ListenableFuture<Void>> futures) {
1755         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1756         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency() != null
1757                 ? origAdjs.getAdjacency() : new ArrayList<>();
1758         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1759         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency() != null
1760                 ? updateAdjs.getAdjacency() : new ArrayList<>();
1761
1762         boolean isOldVpnRemoveCallExecuted = false;
1763         for (String oldVpnName : oldVpnList) {
1764             LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1765                     + "remove from vpnName {} ", interfaceName, oldVpnName);
1766             removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1767             LOG.info("updateVpnInstanceChange: Processed Remove for update on VPNInterface"
1768                             + " {} upon VPN update from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName,
1769                     newVpnList);
1770             isOldVpnRemoveCallExecuted = true;
1771         }
1772         //Wait for previous interface bindings to be removed
1773         if (isOldVpnRemoveCallExecuted && !newVpnList.isEmpty()) {
1774             try {
1775                 Thread.sleep(2000);
1776             } catch (InterruptedException e) {
1777                 LOG.error("updateVpnInstanceChange: InterruptedException caught for interface {}", interfaceName, e);
1778             }
1779         }
1780         for (String newVpnName : newVpnList) {
1781             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1782             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1783                 LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1784                         + "onto vpnName {} ", interfaceName, newVpnName);
1785                 addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1786                 LOG.info("updateVpnInstanceChange: Processed Add for update on VPNInterface {}"
1787                                 + "from oldVpn(s) {} to newVpn {} ",
1788                         interfaceName, oldVpnListCopy, newVpnName);
1789                 /* This block will execute only if V6 subnet is associated with internet BGP-VPN.
1790                  * Use Case:
1791                  *     In Dual stack network, first V4 subnet only attached to router and router is associated
1792                  *     with internet BGP-VPN(router-gw). At this point VPN interface is having only router vpn instance.
1793                  *     Later V6 subnet is added to router, at this point existing VPN interface will get updated
1794                  *     with Internet BGP-VPN instance(Note: Internet BGP-VPN Instance update in vpn interface
1795                  *     is applicable for only on V6 subnet is added to router). newVpnList = Contains only Internet
1796                  *     BGP-VPN Instance. So we required V6 primary adjacency info needs to be populated onto
1797                  *     router VPN as well as Internet BGP-VPN.
1798                  *
1799                  *     addVpnInterfaceCall() --> It will create V6 Adj onto Internet BGP-VPN only.
1800                  *     updateVpnInstanceAdjChange() --> This method call is needed for second primary V6 Adj
1801                  *                                       update in existing router VPN instance.
1802                  */
1803                 if (vpnUtil.isBgpVpnInternet(newVpnName)) {
1804                     LOG.info("updateVpnInstanceChange: VPN Interface {} with new Adjacency {} in existing "
1805                             + "VPN instance {}", interfaceName, newAdjs, original.getVpnInstanceNames());
1806                     updateVpnInstanceAdjChange(original, update, interfaceName, futures);
1807                 }
1808             } else {
1809                 LOG.info("updateVpnInstanceChange: failed to Add for update on VPNInterface {} from oldVpn(s) {} to "
1810                                 + "newVpn {} as the new vpn does not exist in oper DS or it is in PENDING_DELETE state",
1811                         interfaceName, oldVpnListCopy, newVpnName);
1812             }
1813         }
1814     }
1815
1816     // TODO Clean up the exception handling
1817     @SuppressWarnings("checkstyle:IllegalCatch")
1818     private List<ListenableFuture<Void>> updateVpnInstanceAdjChange(VpnInterface original, VpnInterface update,
1819                                                                     String vpnInterfaceName,
1820                                                                     List<ListenableFuture<Void>> futures) {
1821         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1822         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1823                 != null ? origAdjs.getAdjacency() : new ArrayList<>();
1824         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1825         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1826                 != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1827
1828         final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1829         for (VpnInstanceNames vpnInterfaceVpnInstance : update.nonnullVpnInstanceNames()) {
1830             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1831             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1832             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1833             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1834             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1835                 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1836                 //set of prefix used as entry in prefix-to-interface datastore
1837                 // is prerequisite for refresh Fib to avoid race condition leading to missing remote next hop
1838                 // in bucket actions on bgp-vpn delete
1839                 Set<String> prefixListForRefreshFib = new HashSet<>();
1840                 ListenableFuture<Void> configTxFuture = txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
1841                     confTx -> futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL,
1842                         operTx -> {
1843                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1844                                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1845                             LOG.info("VPN Interface update event-intfName {} onto vpnName {} running config-driven",
1846                                     update.getName(), newVpnName);
1847                             //handle both addition and removal of adjacencies
1848                             // currently, new adjacency may be an extra route
1849                             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1850                             if (!oldAdjs.equals(newAdjs)) {
1851                                 for (Adjacency adj : copyNewAdjs) {
1852                                     if (copyOldAdjs.contains(adj)) {
1853                                         copyOldAdjs.remove(adj);
1854                                     } else {
1855                                         // add new adjacency
1856                                         if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1857                                             try {
1858                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1859                                                     dpnId, operTx, confTx, confTx, prefixListForRefreshFib);
1860                                             } catch (RuntimeException e) {
1861                                                 LOG.error("Failed to add adjacency {} to vpn interface {} with"
1862                                                         + " dpnId {}", adj, vpnInterfaceName, dpnId, e);
1863                                             }
1864                                         }
1865                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} "
1866                                             + " added to vpn interface {} on vpn {} dpnId {}",
1867                                             adj.getIpAddress(), adj.getNextHopIpList(), adj.getLabel(),
1868                                             adj.getSubnetId(), update.getName(), newVpnName, dpnId);
1869                                     }
1870                                 }
1871                                 for (Adjacency adj : copyOldAdjs) {
1872                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1873                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1874                                             && !adj.isPhysNetworkFunc()) {
1875                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, operTx,
1876                                                 confTx);
1877                                             //remove FIB entry
1878                                             String vpnRd = vpnUtil.getVpnRd(newVpnName);
1879                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1880                                                 + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1881                                             //remove BGP entry
1882                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), null, confTx);
1883                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1884                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1885                                             }
1886                                         } else {
1887                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1888                                                 operTx, confTx);
1889                                         }
1890                                     }
1891                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1892                                         + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1893                                         adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1894                                 }
1895                             }
1896                         })));
1897                 Futures.addCallback(configTxFuture, new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
1898                     MoreExecutors.directExecutor());
1899                 futures.add(configTxFuture);
1900                 for (ListenableFuture<Void> future : futures) {
1901                     ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1902                             update.getName(), update.getVpnInstanceNames());
1903                 }
1904             } else {
1905                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1906                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1907             }
1908         }
1909         return futures;
1910     }
1911
1912     private void updateLabelMapper(Uint32 label, List<String> nextHopIpList) {
1913         final String labelStr = Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!")
1914                 .toString();
1915         // FIXME: separate this out somehow?
1916         final ReentrantLock lock = JvmGlobalLocks.getLockForString(labelStr);
1917         lock.lock();
1918         try {
1919             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1920                     .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1921             Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1922                 LogicalDatastoreType.OPERATIONAL, lriIid);
1923             if (opResult.isPresent()) {
1924                 LabelRouteInfo labelRouteInfo =
1925                         new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1926                 SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1927                     labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1928             }
1929             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1930                     nextHopIpList);
1931         } catch (ReadFailedException e) {
1932             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1933                     nextHopIpList);
1934         } catch (TransactionCommitFailedException e) {
1935             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1936                     nextHopIpList);
1937         } finally {
1938             lock.unlock();
1939         }
1940     }
1941
1942     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, Uint32 label,
1943         SubnetRoute route, String parentVpnRd, TypedWriteTransaction<Configuration> writeConfigTxn) {
1944
1945         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1946         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1947                 .addAugmentation(SubnetRoute.class, route).build();
1948         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1949         InstanceIdentifierBuilder<VrfTables> idBuilder =
1950             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1951         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1952         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1953         if (writeConfigTxn != null) {
1954             writeConfigTxn.merge(vrfTableId, vrfTableNew, CREATE_MISSING_PARENTS);
1955         } else {
1956             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1957         }
1958         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1959                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1960     }
1961
1962     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1963                                            Adjacency adj, Uint64 dpnId,
1964                                            TypedWriteTransaction<Operational> writeOperTxn,
1965                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1966                                            TypedReadWriteTransaction<Configuration> writeInvTxn,
1967                                            Set<String> prefixListForRefreshFib)
1968             throws ExecutionException, InterruptedException {
1969         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1970         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1971         try {
1972             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1973                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1974             if (optVpnInterface.isPresent()) {
1975                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1976                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1977                 String vpnName = currVpnIntf.getVpnInstanceName();
1978                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1979                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1980                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1981                         LogicalDatastoreType.OPERATIONAL, adjPath);
1982                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1983                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1984                 Uint32 l3vni = vpnInstanceOpData.getL3vni() == null ? Uint32.ZERO : vpnInstanceOpData.getL3vni();
1985                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1986                 List<Adjacency> adjacencies = new ArrayList<>();
1987                 if (optAdjacencies.isPresent() && optAdjacencies.get().getAdjacency() != null) {
1988                     adjacencies.addAll(optAdjacencies.get().getAdjacency());
1989                 }
1990                 Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1991                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1992                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1993                 Adjacency operationalAdjacency = null;
1994                 //Handling dual stack neutron port primary adjacency
1995                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1996                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1997                             currVpnIntf.getName(), vpnName);
1998                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1999                             currVpnIntf.getName());
2000                     if (interfaceState != null) {
2001                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
2002                             currVpnIntf.getName(), vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState,
2003                             prefixListForRefreshFib);
2004                     }
2005                 }
2006                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
2007                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
2008                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.LearntIp ? RouteOrigin.DYNAMIC
2009                             : RouteOrigin.STATIC;
2010                     String nh = adj.getNextHopIpList().get(0);
2011                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
2012                     // FIXME: separate out to somehow?
2013                     final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
2014                     lock.lock();
2015                     try {
2016                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
2017                                 vpnId, null, prefix, vpnName, nh, dpnId);
2018                         if (rdToAllocate.isPresent()) {
2019                             input.setRd(rdToAllocate.get());
2020                             operationalAdjacency = populator.createOperationalAdjacency(input);
2021                             int label = operationalAdjacency.getLabel().intValue();
2022                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
2023                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
2024                                     currVpnIntf.getName(), operationalAdjacency, encapType,
2025                                     prefixListForRefreshFib, writeConfigTxn);
2026                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
2027                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
2028                                     vpnName, label, currVpnIntf.getName(), dpnId);
2029                         } else {
2030                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
2031                                     vpnName, prefix);
2032                             return;
2033                         }
2034                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
2035                         // Keeping the MPLS check for now.
2036                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
2037                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
2038                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
2039                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
2040                             vpnsToImportRoute.forEach(vpn -> {
2041                                 if (vpn.getVrfId() != null) {
2042                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(),
2043                                             vpnId, prefix,
2044                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
2045                                             .ifPresent(
2046                                                 rds -> vpnManager.addExtraRoute(
2047                                                         vpnUtil.getVpnName(vpn.getVpnId()), adj.getIpAddress(),
2048                                                         nh, rds, currVpnIntf.getVpnInstanceName(), l3vni,
2049                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(), opAdjacency,
2050                                                         encapType, prefixListForRefreshFib, writeConfigTxn));
2051                                 }
2052                             });
2053                         }
2054                     } finally {
2055                         lock.unlock();
2056                     }
2057                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
2058                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
2059                             currVpnIntf.getName(), vpnName);
2060
2061                     InstanceIdentifier<VpnInterface> vpnIfaceConfigidentifier = VpnUtil
2062                             .getVpnInterfaceIdentifier(currVpnIntf.getName());
2063                     Optional<VpnInterface> vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2064                             LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier);
2065                     Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix,
2066                             Prefixes.PrefixCue.PhysNetFunc);
2067                     if (vpnIntefaceConfig.isPresent()) {
2068                         pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix,
2069                                 vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(),
2070                                 vpnIntefaceConfig.get().getSegmentationId().toJava(), Prefixes.PrefixCue.PhysNetFunc);
2071                     }
2072
2073                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
2074
2075                     writeOperTxn.merge(
2076                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
2077                                     prefix), pnfPrefix, true);
2078
2079                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
2080                             adj.getIpAddress(), emptyList(), null /* EncapType */, Uint32.ZERO /* label */,
2081                             Uint32.ZERO /*l3vni*/, null /* gw-mac */, parentVpnRd,
2082                             RouteOrigin.LOCAL, writeConfigTxn);
2083
2084                     input.setRd(adj.getVrfId());
2085                 }
2086                 if (operationalAdjacency == null) {
2087                     operationalAdjacency = populator.createOperationalAdjacency(input);
2088                 }
2089                 adjacencies.add(operationalAdjacency);
2090                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
2091                 VpnInterfaceOpDataEntry newVpnIntf =
2092                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
2093                                 aug, dpnId, currVpnIntf.getLportTag().toJava(),
2094                                 currVpnIntf.getGatewayMacAddress(), currVpnIntf.getGatewayIpAddress());
2095                 writeOperTxn.merge(identifier, newVpnIntf, CREATE_MISSING_PARENTS);
2096             }
2097         } catch (ReadFailedException e) {
2098             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
2099                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
2100         }
2101     }
2102
2103     @Nullable
2104     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
2105         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
2106         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
2107     }
2108
2109     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
2110                                             Uint64 dpnId, TypedWriteTransaction<Operational> writeOperTxn,
2111                                             TypedWriteTransaction<Configuration> writeConfigTxn) {
2112         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
2113         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
2114         try {
2115             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
2116                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
2117             if (optVpnInterface.isPresent()) {
2118                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
2119                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
2120                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2121                         LogicalDatastoreType.OPERATIONAL, path);
2122                 if (optAdjacencies.isPresent()) {
2123                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
2124
2125                     if (adjacencies != null && !adjacencies.isEmpty()) {
2126                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
2127                         for (Adjacency adjacency : adjacencies) {
2128                             if (Objects.equals(adjacency.getIpAddress(), adj.getIpAddress())) {
2129                                 String rd = adjacency.getVrfId();
2130                                 if (adj.getNextHopIpList() != null) {
2131                                     for (String nh : adj.getNextHopIpList()) {
2132                                         deleteExtraRouteFromCurrentAndImportingVpns(
2133                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
2134                                                 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
2135                                     }
2136                                 } else if (adj.isPhysNetworkFunc()) {
2137                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
2138                                             adj.getIpAddress(), adj.getSubnetId());
2139                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
2140                                             null, writeConfigTxn);
2141                                 }
2142                                 break;
2143                             }
2144
2145                         }
2146                     }
2147                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
2148                             dpnId, adj.getVrfId());
2149                 } else {
2150                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
2151                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
2152                 }
2153             }
2154         } catch (ReadFailedException e) {
2155             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
2156                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
2157         }
2158     }
2159
2160     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
2161                                     String rd, String intfName, TypedWriteTransaction<Configuration> writeConfigTxn,
2162                                     TypedWriteTransaction<Operational> writeOperTx) {
2163         LOG.info("removing extra-route {} for nexthop {} in VPN {} intfName {} rd {}",
2164                 destination, nextHop, vpnName, intfName, rd);
2165         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
2166         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
2167         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
2168             String vpnRd = vpn.getVrfId();
2169             if (vpnRd != null) {
2170                 LOG.info("deleting extra-route {} for nexthop {} in VPN {} intfName {} vpnRd {}",
2171                         destination, nextHop, vpnName, intfName, vpnRd);
2172                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
2173                         writeOperTx);
2174             }
2175         }
2176     }
2177
2178     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, Uint64 dpnId) {
2179         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2180             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
2181             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
2182     }
2183
2184     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
2185         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2186             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
2187     }
2188
2189     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2190                                                 TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2191         if (vpnInterface == null) {
2192             return;
2193         }
2194         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
2195         if (adjs == null) {
2196             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2197                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2198             return;
2199         }
2200         for (Adjacency adj : adjs) {
2201             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2202                 String primaryInterfaceIp = adj.getIpAddress();
2203                 String macAddress = adj.getMacAddress();
2204                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2205
2206                 Uint32 label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
2207                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2208                 if (label.longValue() == VpnConstants.INVALID_LABEL) {
2209                     LOG.error(
2210                             "createFibEntryForRouterInterface: Unable to retrieve label for vpn pool {}, "
2211                                     + "vpninterface {}, vpn {}, rd {}",
2212                             VpnConstants.VPN_IDPOOL_NAME, interfaceName, vpnName, primaryRd);
2213                     return;
2214                 }
2215                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2216                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2217                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
2218                         routerInt, label, writeConfigTxn);
2219                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2220                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2221                         macAddress);
2222             } else {
2223                 LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2224                                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}",
2225                         interfaceName, primaryRd, vpnName);
2226             }
2227         }
2228     }
2229
2230     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
2231             TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2232         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
2233         String rd = vpnUtil.getVpnRd(vpnName);
2234         if (adjs != null) {
2235             List<Adjacency> adjsList = adjs.nonnullAdjacency();
2236             for (Adjacency adj : adjsList) {
2237                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2238                     String primaryInterfaceIp = adj.getIpAddress();
2239                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2240                     fibManager.removeFibEntry(rd, prefix, null, writeConfigTxn);
2241                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2242                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2243                 }
2244             }
2245         } else {
2246             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2247                     vpnInterface.getName(), rd);
2248         }
2249     }
2250
2251     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2252         final VpnInterfaceKey key = intefaceData.identifier.firstKeyOf(VpnInterface.class);
2253         final String interfaceName = key.getName();
2254         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2255                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2256         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2257                   intefaceData.identifier, vpnName);
2258     }
2259
2260     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2261                                               VpnInterface vpnInterface, String vpnName) {
2262         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2263                .get(vpnName);
2264         if (vpnInterfaces == null) {
2265             vpnInterfaces = new ConcurrentLinkedQueue<>();
2266         }
2267         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2268         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2269         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2270                 vpnInterface.getName(), vpnName);
2271     }
2272
2273     public boolean isVpnInstanceReady(String vpnInstanceName) {
2274         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2275         if (vpnRd == null) {
2276             return false;
2277         }
2278         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2279
2280         return vpnInstanceOpDataEntry != null;
2281     }
2282
2283     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2284         // FIXME: separate out to somehow?
2285         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnInstanceName);
2286         lock.lock();
2287         try {
2288             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2289                     unprocessedVpnInterfaces.get(vpnInstanceName);
2290             if (vpnInterfaces != null) {
2291                 while (!vpnInterfaces.isEmpty()) {
2292                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2293                     if (hasVpnInstanceCreatedSuccessfully) {
2294                         processSavedInterface(savedInterface, vpnInstanceName);
2295                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2296                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2297                     } else {
2298                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2299                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2300                     }
2301                 }
2302             } else {
2303                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2304             }
2305         } finally {
2306             lock.unlock();
2307         }
2308     }
2309
2310     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2311             VpnInterface vpnInterface) {
2312         // FIXME: use VpnInstanceNamesKey perhaps? What about nulls?
2313         final String firstVpnName = VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface);
2314         final ReentrantLock lock = JvmGlobalLocks.getLockForString(firstVpnName);
2315         lock.lock();
2316         try {
2317             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2318                     unprocessedVpnInterfaces.get(firstVpnName);
2319             if (vpnInterfaces != null) {
2320                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2321                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2322                             + "unprocessed list", vpnInterface.getName(), firstVpnName);
2323                 }
2324             } else {
2325                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}", firstVpnName);
2326             }
2327         } finally {
2328             lock.unlock();
2329         }
2330     }
2331
2332     public void vpnInstanceIsReady(String vpnInstanceName) {
2333         processSavedInterfaces(vpnInstanceName, true);
2334     }
2335
2336     public void vpnInstanceFailed(String vpnInstanceName) {
2337         processSavedInterfaces(vpnInstanceName, false);
2338     }
2339
2340     private static class UnprocessedVpnInterfaceData {
2341         InstanceIdentifier<VpnInterface> identifier;
2342         VpnInterface vpnInterface;
2343
2344         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2345             this.identifier = identifier;
2346             this.vpnInterface = vpnInterface;
2347         }
2348
2349         @Override
2350         public int hashCode() {
2351             final int prime = 31;
2352             int result = 1;
2353             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2354             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2355             return result;
2356         }
2357
2358         @Override
2359         public boolean equals(Object obj) {
2360             if (this == obj) {
2361                 return true;
2362             }
2363             if (obj == null) {
2364                 return false;
2365             }
2366             if (getClass() != obj.getClass()) {
2367                 return false;
2368             }
2369             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2370             if (identifier == null) {
2371                 if (other.identifier != null) {
2372                     return false;
2373                 }
2374             } else if (!identifier.equals(other.identifier)) {
2375                 return false;
2376             }
2377             if (vpnInterface == null) {
2378                 if (other.vpnInterface != null) {
2379                     return false;
2380                 }
2381             } else if (!vpnInterface.equals(other.vpnInterface)) {
2382                 return false;
2383             }
2384             return true;
2385         }
2386     }
2387
2388     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2389         String primaryRd = vpnUtil.getVpnRd(vpnName);
2390         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2391         if (vpnInstanceOpData == null) {
2392             return;
2393         }
2394         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2395         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2396             return;
2397         }
2398         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2399         vpnToDpnLists.forEach(vpnToDpnList -> {
2400             if (vpnToDpnList.getVpnInterfaces() == null) {
2401                 return;
2402             }
2403             vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2404                 try {
2405                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2406                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2407                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2408                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2409                     if (!vpnInterfaceOptional.isPresent()) {
2410                         return;
2411                     }
2412                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2413                             vpnInterface.getInterfaceName());
2414                     if (configVpnAdjacencies == null) {
2415                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2416                         return;
2417                     }
2418                     List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2419                             .augmentation(AdjacenciesOp.class).nonnullAdjacency();
2420                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2421                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2422                     // in operational DS. These unprocessed adjacencies will be handled below.
2423                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2424                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2425                     configVpnAdjacencies.stream()
2426                         .filter(adjacency -> operationVpnAdjacencies.stream()
2427                                 .noneMatch(operationalAdjacency ->
2428                                     Objects.equals(operationalAdjacency.getIpAddress(), adjacency.getIpAddress())))
2429                         .forEach(adjacency -> {
2430                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2431                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2432                                 () -> {
2433                                     // TODO Deal with sequencing — the config tx must only submitted
2434                                     // if the oper tx goes in
2435                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2436                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2437                                         futures.add(
2438                                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
2439                                                 //set of prefix used, as entry in prefix-to-interface datastore
2440                                                 // is prerequisite for refresh Fib to avoid race condition leading
2441                                                 // to missing remote next hop in bucket actions on bgp-vpn delete
2442                                                 Set<String> prefixListForRefreshFib = new HashSet<>();
2443                                                 ListenableFuture<Void> configTxFuture =
2444                                                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
2445                                                         confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2446                                                             primaryRd, adjacency,
2447                                                             vpnInterfaceOptional.get().getDpnId(),
2448                                                                 operTx, confTx, confTx, prefixListForRefreshFib));
2449                                                 Futures.addCallback(configTxFuture,
2450                                                     new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
2451                                                     MoreExecutors.directExecutor());
2452                                                 futures.add(configTxFuture);
2453                                             }));
2454                                         return futures;
2455                                     } else {
2456                                         return emptyList();
2457                                     }
2458                                 });
2459                         });
2460                 } catch (ReadFailedException e) {
2461                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2462                             vpnName, primaryRd);
2463                 }
2464             });
2465         });
2466     }
2467
2468     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2469         private final String interfaceName;
2470         private final boolean add;
2471         private final String txnDestination;
2472
2473         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2474             this.interfaceName = interfaceName;
2475             this.add = add;
2476             this.txnDestination = transactionDest;
2477         }
2478
2479         @Override
2480         public void onSuccess(Void voidObj) {
2481             if (add) {
2482                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2483                         interfaceName, txnDestination);
2484             } else {
2485                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2486             }
2487         }
2488
2489         @Override
2490         public void onFailure(Throwable throwable) {
2491             if (add) {
2492                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2493                         interfaceName, txnDestination, throwable);
2494             } else {
2495                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2496                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2497             }
2498         }
2499     }
2500
2501     private class VpnInterfaceCallBackHandler implements FutureCallback<Void> {
2502         private final String primaryRd;
2503         private final Set<String> prefixListForRefreshFib;
2504
2505         VpnInterfaceCallBackHandler(String primaryRd, Set<String> prefixListForRefreshFib) {
2506             this.primaryRd = primaryRd;
2507             this.prefixListForRefreshFib = prefixListForRefreshFib;
2508         }
2509
2510         @Override
2511         public void onSuccess(Void voidObj) {
2512             prefixListForRefreshFib.forEach(prefix -> {
2513                 fibManager.refreshVrfEntry(primaryRd, prefix);
2514             });
2515         }
2516
2517         @Override
2518         public void onFailure(Throwable throwable) {
2519             LOG.debug("write Tx config operation failed", throwable);
2520         }
2521     }
2522 }