vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14
  15 import com.google.common.base.Optional;
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.collect.Iterators;
  18 import com.google.common.util.concurrent.FutureCallback;
  19 import com.google.common.util.concurrent.Futures;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import com.google.common.util.concurrent.MoreExecutors;
  22 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  23 import java.util.ArrayList;
  24 import java.util.Collections;
  25 import java.util.HashSet;
  26 import java.util.List;
  27 import java.util.Map;
  28 import java.util.Objects;
  29 import java.util.Set;
  30 import java.util.concurrent.ConcurrentHashMap;
  31 import java.util.concurrent.ConcurrentLinkedQueue;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.locks.ReentrantLock;
  34 import java.util.function.Consumer;
  35 import java.util.function.Predicate;
  36 import java.util.stream.Collectors;
  37 import javax.annotation.PostConstruct;
  38 import javax.annotation.PreDestroy;
  39 import javax.inject.Inject;
  40 import javax.inject.Singleton;
  41 import org.eclipse.jdt.annotation.Nullable;
  42 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  43 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  44 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  45 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  46 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  47 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  48 import org.opendaylight.genius.infra.Datastore.Configuration;
  49 import org.opendaylight.genius.infra.Datastore.Operational;
  50 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  51 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  52 import org.opendaylight.genius.infra.TransactionAdapter;
  53 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  54 import org.opendaylight.genius.infra.TypedWriteTransaction;
  55 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  56 import org.opendaylight.genius.mdsalutil.NWUtil;
  57 import org.opendaylight.genius.mdsalutil.NwConstants;
  58 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  59 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  60 import org.opendaylight.genius.utils.JvmGlobalLocks;
  61 import org.opendaylight.infrautils.caches.CacheProvider;
  62 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  63 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  64 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  65 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  66 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  67 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  68 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  69 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  70 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  71 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  72 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  73 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  74 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  75 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  76 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  77 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  78 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  79 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  80 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 120 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 121 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 122 import org.opendaylight.yangtools.yang.common.Uint32;
 123 import org.opendaylight.yangtools.yang.common.Uint64;
 124 import org.slf4j.Logger;
 125 import org.slf4j.LoggerFactory;
 126
 127 @Singleton
 128 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 129
 130     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 131     private static final short DJC_MAX_RETRIES = 3;
 132
 133     private final DataBroker dataBroker;
 134     private final ManagedNewTransactionRunner txRunner;
 135     private final IBgpManager bgpManager;
 136     private final IFibManager fibManager;
 137     private final IMdsalApiManager mdsalManager;
 138     private final IdManagerService idManager;
 139     private final OdlInterfaceRpcService ifaceMgrRpcService;
 140     private final VpnFootprintService vpnFootprintService;
 141     private final IInterfaceManager interfaceManager;
 142     private final IVpnManager vpnManager;
 143     private final ArpResponderHandler arpResponderHandler;
 144     private final JobCoordinator jobCoordinator;
 145     private final VpnUtil vpnUtil;
 146
 147     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 148
 149     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 150             new ConcurrentHashMap<>();
 151
 152     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 153
 154     @Inject
 155     public VpnInterfaceManager(final DataBroker dataBroker,
 156                                final IBgpManager bgpManager,
 157                                final IdManagerService idManager,
 158                                final IMdsalApiManager mdsalManager,
 159                                final IFibManager fibManager,
 160                                final OdlInterfaceRpcService ifaceMgrRpcService,
 161                                final VpnFootprintService vpnFootprintService,
 162                                final IInterfaceManager interfaceManager,
 163                                final IVpnManager vpnManager,
 164                                final ArpResponderHandler arpResponderHandler,
 165                                final JobCoordinator jobCoordinator,
 166                                final CacheProvider cacheProvider,
 167                                final VpnUtil vpnUtil) {
 168         super(VpnInterface.class, VpnInterfaceManager.class);
 169
 170         this.dataBroker = dataBroker;
 171         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 172         this.bgpManager = bgpManager;
 173         this.idManager = idManager;
 174         this.mdsalManager = mdsalManager;
 175         this.fibManager = fibManager;
 176         this.ifaceMgrRpcService = ifaceMgrRpcService;
 177         this.vpnFootprintService = vpnFootprintService;
 178         this.interfaceManager = interfaceManager;
 179         this.vpnManager = vpnManager;
 180         this.arpResponderHandler = arpResponderHandler;
 181         this.jobCoordinator = jobCoordinator;
 182         this.vpnUtil = vpnUtil;
 183
 184         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 185                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 186                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 187     }
 188
 189     public Runnable isNotifyTaskQueued(String intfName) {
 190         return vpnIntfMap.remove(intfName);
 191     }
 192
 193     @PostConstruct
 194     public void start() {
 195         LOG.info("{} start", getClass().getSimpleName());
 196         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 197     }
 198
 199     @Override
 200     @PreDestroy
 201     public void close() {
 202         super.close();
 203         vpnInstanceOpDataEntryCache.close();
 204     }
 205
 206     @Override
 207     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 208         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 209     }
 210
 211     @Override
 212     protected VpnInterfaceManager getDataTreeChangeListener() {
 213         return VpnInterfaceManager.this;
 214     }
 215
 216     @Override
 217     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 218         LOG.trace("Received VpnInterface add event: vpnInterface={}", vpnInterface);
 219         LOG.info("add: intfName {} onto vpnName {}", vpnInterface.getName(),
 220                 VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 221         addVpnInterface(identifier, vpnInterface, null, null);
 222     }
 223
 224     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 225                           final VpnInterface vpnInterface, String vpnName) {
 226         // FIXME: separate this out somehow?
 227         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnName);
 228         lock.lock();
 229         try {
 230             if (isVpnInstanceReady(vpnName)) {
 231                 return true;
 232             }
 233             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 234             return false;
 235         } finally {
 236             lock.unlock();
 237         }
 238     }
 239
 240     // TODO Clean up the exception handling
 241     @SuppressWarnings("checkstyle:IllegalCatch")
 242     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 243                              final @Nullable List<Adjacency> oldAdjs, final @Nullable List<Adjacency> newAdjs) {
 244         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
 245             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 246             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 247         }
 248     }
 249
 250     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 251                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 252         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 253         final String interfaceName = key.getName();
 254
 255         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 256             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 257                   vpnName, vpnInterface.getName());
 258             return;
 259         }
 260         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 261              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 262         List<Adjacency> copyOldAdjs = null;
 263         if (oldAdjs != null) {
 264             copyOldAdjs = new ArrayList<>();
 265             copyOldAdjs.addAll(oldAdjs);
 266         }
 267         List<Adjacency> copyNewAdjs = null;
 268         if (newAdjs != null) {
 269             copyNewAdjs = new ArrayList<>();
 270             copyNewAdjs.addAll(newAdjs);
 271         }
 272         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 273     }
 274
 275     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 276                     final VpnInterface vpnInterface, final @Nullable List<Adjacency> oldAdjs,
 277                     final @Nullable List<Adjacency> newAdjs,
 278                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 279         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 280         final String interfaceName = key.getName();
 281         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 282         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 283             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 284             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 285             if (interfaceState != null) {
 286                 try {
 287                     final Uint64 dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 288                     final int ifIndex = interfaceState.getIfIndex();
 289                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 290                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 291                         // (the inventory tx goes in last)
 292                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 293                         //set of prefix used, as entry in prefix-to-interface datastore
 294                         // is prerequisite for refresh Fib to avoid race condition leading to
 295                         // missing remote next hop in bucket actions on bgp-vpn delete
 296                         Set<String> prefixListForRefreshFib = new HashSet<>();
 297                         ListenableFuture<Void> confFuture =
 298                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 299                                 confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
 300                                     operTx -> futures.add(
 301                                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, invTx -> {
 302                                             LOG.info(
 303                                                 "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 304                                                     + " on dpn {}",
 305                                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 306                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 307                                                 confTx, operTx, invTx, interfaceState, vpnName,
 308                                                 prefixListForRefreshFib);
 309                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 310                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 311                                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 312                                                 if (newAdjs != null) {
 313                                                     for (Adjacency adj : newAdjs) {
 314                                                         if (oldAdjs.contains(adj)) {
 315                                                             oldAdjs.remove(adj);
 316                                                         } else {
 317                                                             if (!isBgpVpnInternetVpn
 318                                                                 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 319                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 320                                                                     primaryRd, adj, dpnId, operTx, confTx, invTx,
 321                                                                     prefixListForRefreshFib);
 322                                                             }
 323                                                         }
 324                                                     }
 325                                                 }
 326                                                 for (Adjacency adj : oldAdjs) {
 327                                                     if (!isBgpVpnInternetVpn
 328                                                         || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 329                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 330                                                             operTx, confTx);
 331                                                     }
 332                                                 }
 333                                             }
 334                                         })))));
 335                         Futures.addCallback(confFuture,
 336                             new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
 337                             MoreExecutors.directExecutor());
 338                         futures.add(confFuture);
 339                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 340                             MoreExecutors.directExecutor());
 341                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 342                             + " processed successfully", interfaceName, vpnName, dpnId);
 343                         return futures;
 344                     });
 345                 } catch (NumberFormatException | IllegalStateException e) {
 346                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 347                             + "interface {}. Interface addition on vpn {} failed", interfaceName,
 348                         vpnName, e);
 349                     return;
 350                 }
 351             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 352                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 353                     () -> {
 354                         ListenableFuture<Void> future =
 355                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
 356                                 createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 357                                     confTx, vpnName);
 358                                 LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 359                                     vpnName, vpnInterface.getDpnId());
 360                             });
 361                         ListenableFutures.addErrorLogging(future, LOG,
 362                             "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 363                         return Collections.singletonList(future);
 364                     });
 365             } else {
 366                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 367                     + " is not available", interfaceName, vpnName);
 368             }
 369         } else {
 370             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 371                     + " as vpn is pending delete", interfaceName, vpnName,
 372                 vpnInterface.getDpnId());
 373         }
 374     }
 375
 376     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 377     // see comments below.
 378     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 379     protected void processVpnInterfaceUp(final Uint64 dpId, VpnInterface vpnInterface, final String primaryRd,
 380             final int lportTag, boolean isInterfaceUp,
 381             TypedWriteTransaction<Configuration> writeConfigTxn,
 382             TypedWriteTransaction<Operational> writeOperTxn,
 383             TypedReadWriteTransaction<Configuration> writeInvTxn,
 384             Interface interfaceState, final String vpnName,
 385             Set<String> prefixListForRefreshFib) throws ExecutionException, InterruptedException {
 386         final String interfaceName = vpnInterface.getName();
 387         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 388                 vpnName);
 389         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 390         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 391         if (!isInterfaceUp) {
 392             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 393                      interfaceName, dpId, vpnName);
 394             Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 395             if (VpnConstants.INVALID_ID.equals(vpnId)) {
 396                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 397                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 398                         dpId);
 399                 return;
 400             }
 401
 402             boolean waitForVpnInterfaceOpRemoval = false;
 403             if (opVpnInterface != null) {
 404                 String opVpnName = opVpnInterface.getVpnInstanceName();
 405                 String primaryInterfaceIp = null;
 406                 if (Objects.equals(opVpnName, vpnName)) {
 407                     // Please check if the primary VRF Entry does not exist for VPNInterface
 408                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 409                     // back to back
 410                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 411                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 412                     if (adjs == null) {
 413                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 414                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 415                                 vpnName);
 416                         return;
 417                     }
 418                     for (Adjacency adj : adjs) {
 419                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 420                             primaryInterfaceIp = adj.getIpAddress();
 421                             break;
 422                         }
 423                     }
 424                     if (primaryInterfaceIp == null) {
 425                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 426                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 427                                 dpId, vpnName);
 428                         return;
 429                     }
 430                     // Get the rd of the vpn instance
 431                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 432                     if (vrf != null) {
 433                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 434                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 435                         return;
 436                     }
 437                     waitForVpnInterfaceOpRemoval = true;
 438                 } else {
 439                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 440                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 441                 }
 442             }
 443             if (!waitForVpnInterfaceOpRemoval) {
 444                 // Add the VPNInterface and quit
 445                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 446                         null/*ipAddressSourceValuePair*/,
 447                         true /* add */);
 448                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 449                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 450                 if (!isBgpVpnInternetVpn) {
 451                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 452                 }
 453                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 454                         dpId, vpnName);
 455                 if (interfaceManager.isExternalInterface(interfaceName)) {
 456                     processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
 457                         NwConstants.ADD_FLOW);
 458                 }
 459                 return;
 460             }
 461
 462             // FIB didn't get a chance yet to clean up this VPNInterface
 463             // Let us give it a chance here !
 464             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 465                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 466             try {
 467                 Runnable notifyTask = new VpnNotifyTask();
 468                 synchronized (notifyTask) {
 469                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 470                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 471                     // published here while holding the lock on it so this path will hit the wait before notify can be
 472                     // invoked.
 473                     vpnIntfMap.put(interfaceName, notifyTask);
 474                     try {
 475                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 476                     } catch (InterruptedException e) {
 477                         // Ignored
 478                     }
 479                 }
 480             } finally {
 481                 vpnIntfMap.remove(interfaceName);
 482             }
 483
 484             if (opVpnInterface != null) {
 485                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 486                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 487                         dpId, vpnName);
 488                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 489                 return;
 490             }
 491             // VPNInterface got removed, proceed with Add
 492             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 493                     interfaceName, dpId, vpnName);
 494             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 495                     null/*ipAddressSourceValuePair*/,
 496                     true /* add */);
 497             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 498                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 499             if (!isBgpVpnInternetVpn) {
 500                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 501             }
 502             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 503                     + " FIB to clean up", interfaceName, dpId, vpnName);
 504             if (interfaceManager.isExternalInterface(interfaceName)) {
 505                 processExternalVpnInterface(interfaceName, vpnName, dpId,
 506                                lportTag, NwConstants.ADD_FLOW);
 507             }
 508         } else {
 509             try {
 510                 // Interface is retained in the DPN, but its Link Up.
 511                 // Advertise prefixes again for this interface to BGP
 512                 InstanceIdentifier<VpnInterface> identifier =
 513                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 514                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 515                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 516                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 517                 // Perform similar operation as interface add event for extraroutes.
 518                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 519                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 520                         LogicalDatastoreType.CONFIGURATION, path);
 521                 if (!optAdjacencies.isPresent()) {
 522                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 523                     return;
 524                 }
 525                 List<Adjacency> adjacencies = optAdjacencies.get().nonnullAdjacency();
 526                 for (Adjacency adjacency : adjacencies) {
 527                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 528                         continue;
 529                     }
 530                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 531                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 532                         continue;
 533                     }
 534                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 535                             dpId, writeOperTxn, writeConfigTxn, writeInvTxn, prefixListForRefreshFib);
 536                 }
 537             } catch (ReadFailedException e) {
 538                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 539                         interfaceName, vpnName, primaryRd, dpId);
 540             }
 541         }
 542     }
 543
 544     private void processExternalVpnInterface(String interfaceName, String vpnName, Uint64 dpId,
 545         int lportTag, int addOrRemove) {
 546         Uuid extNetworkId;
 547         try {
 548             // vpn instance of ext-net interface is the network-id
 549             extNetworkId = new Uuid(vpnName);
 550         } catch (IllegalArgumentException e) {
 551             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 552                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 553             return;
 554         }
 555
 556         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 557         if (routerIds == null || routerIds.isEmpty()) {
 558             LOG.info("processExternalVpnInterface: No router is associated with {}."
 559                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 560                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 561             return;
 562         }
 563
 564         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 565                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 566         for (Uuid routerId : routerIds) {
 567             String routerName = routerId.getValue();
 568             Uint64 primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 569             if (Objects.equals(primarySwitch, dpId)) {
 570                 Routers router = vpnUtil.getExternalRouter(routerName);
 571                 if (router != null) {
 572                     if (addOrRemove == NwConstants.ADD_FLOW) {
 573                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 574                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 575                                 dpId, interfaceName, lportTag);
 576                     } else {
 577                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 578                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 579                                 dpId, interfaceName, lportTag);
 580                     }
 581                 } else {
 582                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 583                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 584                             interfaceName, dpId, vpnName);
 585                 }
 586             }
 587         }
 588     }
 589
 590     // TODO Clean up the exception handling
 591     @SuppressWarnings("checkstyle:IllegalCatch")
 592     private void advertiseAdjacenciesForVpnToBgp(final String rd, Uint64 dpnId,
 593                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 594                                                   String vpnName, String interfaceName) {
 595         if (rd == null) {
 596             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 597                   interfaceName, dpnId, vpnName);
 598             return;
 599         } else {
 600             if (rd.equals(vpnName)) {
 601                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 602                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 603                 return;
 604             }
 605         }
 606         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 607                 interfaceName, dpnId, vpnName, rd);
 608
 609         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 610         if (nextHopIp == null) {
 611             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 612                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 613                     rd, vpnName);
 614             return;
 615         }
 616
 617         try {
 618             //Read NextHops
 619             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 620             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 621                     LogicalDatastoreType.OPERATIONAL, path);
 622             if (adjacencies.isPresent()) {
 623                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 624                 if (nextHops != null && !nextHops.isEmpty()) {
 625                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 626                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 627                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 628                     Uint32 l3vni = vpnInstanceOpData.getL3vni();
 629                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 630                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 631                     for (Adjacency nextHop : nextHops) {
 632                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 633                             continue;
 634                         }
 635                         String gatewayMac = null;
 636                         Uint32 label = Uint32.ZERO;
 637                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 638                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 639                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 640                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 641                                     .get();
 642                         } else {
 643                             label = nextHop.getLabel();
 644                         }
 645                         try {
 646                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 647                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 648                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 649                                     encapType, label, l3vni, Uint32.ZERO /*l2vni*/,
 650                                     gatewayMac);
 651                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 652                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 653                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 654                         } catch (Exception e) {
 655                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 656                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 657                                     interfaceName, dpnId, e);
 658                         }
 659                     }
 660                 }
 661             }
 662         } catch (ReadFailedException e) {
 663             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 664                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 665         }
 666     }
 667
 668     // TODO Clean up the exception handling
 669     @SuppressWarnings("checkstyle:IllegalCatch")
 670     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 671                         String vpnName, String interfaceName, TypedWriteTransaction<Configuration> writeConfigTxn,
 672                         TypedWriteTransaction<Operational> writeOperTx) {
 673         //Read NextHops
 674         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 675         String rd = vpnUtil.getVpnRd(interfaceName);
 676         if (rd == null) {
 677             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 678                 interfaceName, vpnName);
 679             return;
 680         } else {
 681             if (rd.equals(vpnName)) {
 682                 LOG.info(
 683                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 684                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 685                 return;
 686             }
 687         }
 688         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 689                vpnName, rd);
 690         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 691         try {
 692             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 693                     path);
 694         } catch (ReadFailedException e) {
 695             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 696                     interfaceName, vpnName);
 697         }
 698         if (adjacencies.isPresent()) {
 699             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 700
 701             if (nextHops != null && !nextHops.isEmpty()) {
 702                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 703                         nextHops, interfaceName, vpnName, rd);
 704                 for (Adjacency nextHop : nextHops) {
 705                     try {
 706                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 707                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 708                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 709                                     interfaceName, vpnName);
 710                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 711                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 712                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 713                                     interfaceName, vpnName);
 714                         } else if (nextHop.getNextHopIpList() != null) {
 715                             // Perform similar operation as interface delete event for extraroutes.
 716                             String allocatedRd = nextHop.getVrfId();
 717                             for (String nh : nextHop.getNextHopIpList()) {
 718                                 deleteExtraRouteFromCurrentAndImportingVpns(
 719                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
 720                                     writeOperTx);
 721                             }
 722                         }
 723                     } catch (Exception e) {
 724                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 725                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 726                     }
 727                 }
 728             }
 729         }
 730     }
 731
 732     @SuppressWarnings("checkstyle:IllegalCatch")
 733     protected void processVpnInterfaceAdjacencies(Uint64 dpnId, final int lportTag, String vpnName,
 734                                                   String primaryRd, String interfaceName, final Uint32 vpnId,
 735                                                   TypedWriteTransaction<Configuration> writeConfigTxn,
 736                                                   TypedWriteTransaction<Operational> writeOperTxn,
 737                                                   TypedReadWriteTransaction<Configuration> writeInvTxn,
 738                                                   Interface interfaceState, Set<String> prefixListForRefreshFib)
 739             throws ExecutionException, InterruptedException {
 740         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 741         // Read NextHops
 742         Optional<VpnInterface> vpnInteface = Optional.absent();
 743         try {
 744             vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 745             LogicalDatastoreType.CONFIGURATION, identifier);
 746         } catch (ReadFailedException e) {
 747             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 748                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 749         }
 750         Uuid intfnetworkUuid = null;
 751         NetworkType networkType = null;
 752         Long segmentationId = Long.valueOf(-1);
 753         Adjacencies adjacencies = null;
 754         if (vpnInteface.isPresent()) {
 755             intfnetworkUuid = vpnInteface.get().getNetworkId();
 756             networkType = vpnInteface.get().getNetworkType();
 757             segmentationId = vpnInteface.get().getSegmentationId().toJava();
 758             adjacencies = vpnInteface.get().augmentation(Adjacencies.class);
 759             if (adjacencies == null) {
 760                 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 761                         null/*gwMac*/, writeOperTxn);
 762                 return;
 763             }
 764         }
 765         // Get the rd of the vpn instance
 766         String nextHopIp = null;
 767         try {
 768             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 769         } catch (Exception e) {
 770             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 771                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 772         }
 773         List<String> nhList = new ArrayList<>();
 774         if (nextHopIp != null) {
 775             nhList.add(nextHopIp);
 776             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 777                     interfaceName, dpnId, vpnName, nhList);
 778         }
 779         Optional<String> gwMac = Optional.absent();
 780         String vpnInterfaceSubnetGwMacAddress = null;
 781         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 782         Uint32 l3vni = vpnInstanceOpData.getL3vni() != null ? vpnInstanceOpData.getL3vni() : Uint32.ZERO;
 783         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 784         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 785         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 786         List<Adjacency> nextHops = adjacencies != null ? adjacencies.getAdjacency() : emptyList();
 787         List<Adjacency> value = new ArrayList<>();
 788         for (Adjacency nextHop : nextHops) {
 789             String rd = primaryRd;
 790             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 791             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 792                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 793                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 794                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 795                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 796                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 797                 continue;
 798             }
 799             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 800                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 801                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 802                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 803                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 804                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 805
 806                 Prefixes prefixes = intfnetworkUuid != null
 807                     ? VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, intfnetworkUuid ,networkType,
 808                             segmentationId, prefixCue) :
 809                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, prefixCue);
 810                 writeOperTxn.merge(VpnUtil.getPrefixToInterfaceIdentifier(
 811                         vpnUtil.getVpnId(vpnName), prefix), prefixes, true);
 812                 final Uuid subnetId = nextHop.getSubnetId();
 813
 814                 String gatewayIp = nextHop.getSubnetGatewayIp();
 815                 if (gatewayIp == null) {
 816                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 817                     if (gatewayIpOptional.isPresent()) {
 818                         gatewayIp = gatewayIpOptional.get();
 819                     }
 820                 }
 821
 822                 if (gatewayIp != null) {
 823                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 824                     if (gwMac.isPresent()) {
 825                         // A valid mac-address is available for this subnet-gateway-ip
 826                         // Use this for programming ARP_RESPONDER table here.  And save this
 827                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 828                         // to populate L3_GW_MAC_TABLE there.
 829                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 830                                 gatewayIp, gwMac.get());
 831                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 832                     } else {
 833                         // A valid mac-address is not available for this subnet-gateway-ip
 834                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 835                         // Save this connected-mac-address as gateway-mac-address for the
 836                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 837                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 838                         if (gwMac.isPresent()) {
 839                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 840                                     NwConstants.ADD_FLOW, gwMac.get());
 841                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 842                                     gatewayIp, gwMac.get());
 843                         } else {
 844                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 845                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 846                                 + "gwIp {}",
 847                                 subnetId, interfaceName, vpnName, gatewayIp);
 848                         }
 849                     }
 850                 } else {
 851                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 852                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 853                         subnetId, interfaceName, vpnName);
 854                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 855                 }
 856                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 857                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 858             } else {
 859                 //Extra route adjacency
 860                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 861                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 862                 // FIXME: separate this out somehow?
 863                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
 864                 lock.lock();
 865                 try {
 866                     java.util.Optional<String> rdToAllocate = vpnUtil
 867                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 868                                     nextHop.getNextHopIpList().get(0), dpnId);
 869                     if (rdToAllocate.isPresent()) {
 870                         rd = rdToAllocate.get();
 871                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 872                             rd, prefix);
 873                     } else {
 874                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 875                         continue;
 876                     }
 877                 } finally {
 878                     lock.unlock();
 879                 }
 880                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 881                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 882                         interfaceName, dpnId);
 883             }
 884             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 885             // can be different from the gateway-mac-address within the VRFEntry as the
 886             // gateway-mac-address is a superset.
 887             RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 888             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 889                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 890                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 891             Adjacency operationalAdjacency = null;
 892             try {
 893                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 894             } catch (NullPointerException e) {
 895                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 896                     input, e.getMessage());
 897                 return;
 898             }
 899             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 900                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 901                     vpnName, l3vni, origin, interfaceName, operationalAdjacency, encapType, prefixListForRefreshFib,
 902                     writeConfigTxn);
 903             }
 904             value.add(operationalAdjacency);
 905         }
 906
 907         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 908         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 909                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 910
 911         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni.longValue()).setPrimaryRd(primaryRd)
 912                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 913                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 914
 915         for (Adjacency nextHop : aug.getAdjacency()) {
 916             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 917             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 918                 RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 919                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 920                 registeredPopulator.populateFib(input, writeConfigTxn);
 921             }
 922         }
 923     }
 924
 925     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, Uint64 dpnId, AdjacenciesOp aug,
 926                                               long lportTag, String gwMac,
 927                                               TypedWriteTransaction<Operational> writeOperTxn) {
 928         VpnInterfaceOpDataEntry opInterface =
 929               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac);
 930         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 931             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 932         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
 933         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 934                 interfaceName, dpnId, vpnName);
 935     }
 936
 937     // TODO Clean up the exception handling
 938     @SuppressWarnings("checkstyle:IllegalCatch")
 939     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 940                                            StateTunnelList stateTunnelList,
 941                                            TypedWriteTransaction<Configuration> writeConfigTxn,
 942                                            TypedWriteTransaction<Operational> writeOperTxn) {
 943
 944         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
 945         Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern();
 946         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 947         List<Adjacency> adjList =
 948             adjacencies != null && adjacencies.getAdjacency() != null ? adjacencies.getAdjacency() : emptyList();
 949         if (adjList.isEmpty()) {
 950             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 951                     vpnInterface, srcDpnId);
 952             return;
 953         }
 954         String prefix = null;
 955         List<Adjacency> value = new ArrayList<>();
 956         boolean isNextHopAddReqd = false;
 957         String vpnName = vpnInterface.getVpnInstanceName();
 958         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 959         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 960         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 961                 vpnInterface.getName(), vpnInterface.getDpnId(),
 962                 vpnInterface.getVpnInstanceName(), adjList);
 963         for (Adjacency adj : adjList) {
 964             String rd = adj.getVrfId();
 965             rd = rd != null ? rd : vpnName;
 966             prefix = adj.getIpAddress();
 967             Uint32 label = adj.getLabel();
 968             List<String> nhList = Collections.singletonList(srcTepIp);
 969             List<String> nextHopList = adj.getNextHopIpList();
 970             // If TEP is added , update the nexthop of primary adjacency.
 971             // Secondary adj nexthop is already pointing to primary adj IP address.
 972             if (nextHopList == null || nextHopList.isEmpty()) {
 973                 isNextHopAddReqd = true;
 974             }
 975
 976             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 977                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 978             } else {
 979                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 980                 if (!vrfEntryOptional.isPresent()) {
 981                     continue;
 982                 }
 983                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 984                 if (!nhList.contains(srcTepIp)) {
 985                     nhList.add(srcTepIp);
 986                     isNextHopAddReqd = true;
 987                 }
 988                 value.add(adj);
 989             }
 990
 991             if (isNextHopAddReqd) {
 992                 updateLabelMapper(label, nhList);
 993                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 994                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 995                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 996                 // Update the VRF entry with nextHop
 997                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 998                         label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
 999
1000                 //Get the list of VPN's importing this route(prefix) .
1001                 // Then update the VRF entry with nhList
1002                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1003                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1004                     String vpnRd = vpn.getVrfId();
1005                     if (vpnRd != null) {
1006                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1007                             srcTepIp, label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1008                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
1009                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
1010                             vpnInterface.getName(), srcDpnId, vpnName,
1011                             vpn.getVpnInstanceName(), vpnRd);
1012                     }
1013                 }
1014                 // Advertise the prefix to BGP only for external vpn
1015                 // since there is a nexthop change.
1016                 try {
1017                     if (!rd.equalsIgnoreCase(vpnName)) {
1018                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
1019                                 VrfEntry.EncapType.Mplsgre, label, Uint32.ZERO /*evi*/, Uint32.ZERO /*l2vni*/,
1020                                 null /*gatewayMacAddress*/);
1021                     }
1022                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
1023                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
1024                             srcDpnId, vpnName);
1025                 } catch (Exception ex) {
1026                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
1027                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1028                             vpnInterface.getName(), srcDpnId, vpnName, ex);
1029                 }
1030             }
1031         }
1032         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1033         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1034                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1035                 .addAugmentation(AdjacenciesOp.class, aug).build();
1036         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1037                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1038         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1039         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
1040                 vpnInterface.getName(), srcDpnId, vpnName);
1041
1042     }
1043
1044     // TODO Clean up the exception handling
1045     @SuppressWarnings("checkstyle:IllegalCatch")
1046     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1047                                               StateTunnelList stateTunnelList,
1048                                               TypedWriteTransaction<Configuration> writeConfigTxn,
1049                                               TypedWriteTransaction<Operational> writeOperTxn) {
1050
1051         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1052         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1053         String prefix = null;
1054         boolean isNextHopRemoveReqd = false;
1055         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
1056         Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern();
1057         String vpnName = vpnInterface.getVpnInstanceName();
1058         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1059         String primaryRd = vpnUtil.getVpnRd(vpnName);
1060         if (adjList != null) {
1061             List<Adjacency> value = new ArrayList<>();
1062             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1063                     vpnInterface.getName(), vpnInterface.getDpnId(),
1064                     vpnInterface.getVpnInstanceName(), adjList);
1065             for (Adjacency adj : adjList) {
1066                 List<String> nhList = new ArrayList<>();
1067                 String rd = adj.getVrfId();
1068                 rd = rd != null ? rd : vpnName;
1069                 prefix = adj.getIpAddress();
1070                 List<String> nextHopList = adj.getNextHopIpList();
1071                 Uint32 label = adj.getLabel();
1072                 if (nextHopList != null && !nextHopList.isEmpty()) {
1073                     isNextHopRemoveReqd = true;
1074                 }
1075                 // If TEP is deleted , remove the nexthop from primary adjacency.
1076                 // Secondary adj nexthop will continue to point to primary adj IP address.
1077                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1078                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1079                 } else {
1080                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1081                     if (!vrfEntryOptional.isPresent()) {
1082                         continue;
1083                     }
1084                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1085                     if (nhList.contains(srcTepIp)) {
1086                         nhList.remove(srcTepIp);
1087                         isNextHopRemoveReqd = true;
1088                     }
1089                     value.add(adj);
1090                 }
1091
1092                 if (isNextHopRemoveReqd) {
1093                     updateLabelMapper(label, nhList);
1094                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1095                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1096                             prefix, nhList, vpnName,
1097                             vpnId, rd, vpnInterface.getName());
1098                     // Update the VRF entry with removed nextHop
1099                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1100                             label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1101
1102                     //Get the list of VPN's importing this route(prefix) .
1103                     // Then update the VRF entry with nhList
1104                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1105                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1106                         String vpnRd = vpn.getVrfId();
1107                         if (vpnRd != null) {
1108                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1109                                 srcTepIp, label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1110                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1111                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1112                                     nhList, label, vpnInterface.getName(), srcDpnId,
1113                                     vpnName,
1114                                     vpn.getVpnInstanceName(), vpnRd);
1115                         }
1116                     }
1117
1118                     // Withdraw prefix from BGP only for external vpn.
1119                     try {
1120                         if (!rd.equalsIgnoreCase(vpnName)) {
1121                             bgpManager.withdrawPrefix(rd, prefix);
1122                         }
1123                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1124                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1125                                 vpnInterface.getName(), srcDpnId,
1126                                 vpnName);
1127                     } catch (Exception ex) {
1128                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1129                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1130                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1131                     }
1132                 }
1133             }
1134             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1135             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1136                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1137                     .addAugmentation(AdjacenciesOp.class, aug).build();
1138             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1139                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1140             writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1141             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1142                          vpnInterface.getName(), srcDpnId, vpnName);
1143         }
1144     }
1145
1146     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1147         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1148
1149         String vpnRd = vpnUtil.getVpnRd(vpnName);
1150         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1151         if (vpnInstanceOpDataEntry == null) {
1152             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1153                     + " to check for vpns exporting the routes", vpnName);
1154             return vpnsToExportRoute;
1155         }
1156
1157         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1158             if (input.getVpnInstanceName() == null) {
1159                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1160                         input.getVrfId());
1161                 return false;
1162             }
1163             return !input.getVpnInstanceName().equals(vpnName);
1164         };
1165
1166         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1167             Iterable<String> commonRTs =
1168                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1169                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1170             return Iterators.size(commonRTs.iterator()) > 0;
1171         };
1172
1173         vpnsToExportRoute =
1174                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1175                         Collectors.toList());
1176         return vpnsToExportRoute;
1177     }
1178
1179     // TODO Clean up the exception handling
1180     @SuppressWarnings("checkstyle:IllegalCatch")
1181     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1182         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1183         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1184             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1185             if (vrfEntries != null) {
1186                 ListenableFutures.addErrorLogging(
1187                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1188                         for (VrfEntry vrfEntry : vrfEntries) {
1189                             try {
1190                                 if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1191                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1192                                     LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1193                                             + " is not a controller managed non intervpn link route. Ignoring.",
1194                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1195                                     continue;
1196                                 }
1197                                 String prefix = vrfEntry.getDestPrefix();
1198                                 String gwMac = vrfEntry.getGatewayMacAddress();
1199                                 vrfEntry.nonnullRoutePaths().forEach(routePath -> {
1200                                     String nh = routePath.getNexthopAddress();
1201                                     Uint32 label = routePath.getLabel();
1202                                     if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1203                                         vrfEntry.getOrigin()))) {
1204                                         LOG.info(
1205                                             "handleVpnsExportingRoutesImporting: Importing fib entry rd {}"
1206                                                 + " prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1207                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1208                                         fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1209                                             Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1210                                                 Uint32.ZERO /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1211                                             confTx);
1212                                     } else {
1213                                         LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry"
1214                                                 + " rd {} prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1215                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1216                                         SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1217                                         importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1218                                             confTx);
1219                                     }
1220                                 });
1221                             } catch (RuntimeException e) {
1222                                 LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1223                                         + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1224                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1225                             }
1226                         }
1227                     }), LOG, "Error handing VPN exporting routes");
1228             } else {
1229                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1230                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1231             }
1232         }
1233     }
1234
1235     @Override
1236     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1237         LOG.trace("Received VpnInterface remove event: vpnInterface={}", vpnInterface);
1238         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1239         final String interfaceName = key.getName();
1240         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
1241             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1242             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1243         }
1244     }
1245
1246     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1247                                 final VpnInterface vpnInterface, final String vpnName,
1248                                 final String interfaceName) {
1249         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1250             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1251                 ListenableFuture<Void> future =
1252                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1253                         deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1254                         LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1255                     });
1256                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1257                         vpnInterface.getName(), vpnName);
1258                 return Collections.singletonList(future);
1259             }, DJC_MAX_RETRIES);
1260         } else {
1261             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1262             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1263         }
1264     }
1265
1266     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1267     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1268                                            final VpnInterface vpnInterface, final String vpnName,
1269                                            final String interfaceName, final Interface interfaceState) {
1270         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1271                 vpnName, vpnInterface.getDpnId());
1272         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1273         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1274             () -> {
1275                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1276                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1277                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1278                         writeOperTxn -> futures.add(
1279                             txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeInvTxn -> {
1280                                 LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1281                                         interfaceName, vpnName);
1282                                 Uint64 dpId;
1283                                 int ifIndex;
1284                                 String gwMacAddress;
1285                                 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1286                                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1287                                 Optional<VpnInterfaceOpDataEntry> optVpnInterface;
1288                                 try {
1289                                     optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1290                                             LogicalDatastoreType.OPERATIONAL, interfaceId);
1291                                 } catch (ReadFailedException e) {
1292                                     LOG.error("remove: Failed to read data store for interface {} vpn {}",
1293                                             interfaceName, vpnName);
1294                                     return;
1295                                 }
1296                                 if (interfaceState != null) {
1297                                     try {
1298                                         dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1299                                     } catch (NumberFormatException | IllegalStateException e) {
1300                                         LOG.error("remove: Unable to retrieve dpnId from interface operational"
1301                                                         + " data store for interface {} on dpn {} for vpn {} Fetching"
1302                                                         + " from vpn interface op data store. ", interfaceName,
1303                                                 vpnInterface.getDpnId(), vpnName, e);
1304                                         dpId = Uint64.ZERO;
1305                                     }
1306                                     ifIndex = interfaceState.getIfIndex();
1307                                     gwMacAddress = interfaceState.getPhysAddress().getValue();
1308                                 } else {
1309                                     LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1310                                             + " from vpn interface op.", interfaceName);
1311                                     if (optVpnInterface.isPresent()) {
1312                                         VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1313                                         dpId = vpnOpInterface.getDpnId();
1314                                         ifIndex = vpnOpInterface.getLportTag().intValue();
1315                                         gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1316                                     } else {
1317                                         LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1318                                                 + " as interfaceState and vpn interface op is not"
1319                                                 + " available", interfaceName, vpnName);
1320                                         return;
1321                                     }
1322                                 }
1323                                 processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1324                                         optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1325                                         writeConfigTxn, writeOperTxn, writeInvTxn);
1326                                 LOG.info(
1327                                         "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1328                                                 + "successfully",
1329                                         interfaceName, vpnInterface.getDpnId(), vpnName);
1330                             })))));
1331                 futures.add(configFuture);
1332                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(
1333                         interfaceName, false, "Config"), MoreExecutors.directExecutor());
1334                 return futures;
1335             }, DJC_MAX_RETRIES);
1336     }
1337
1338     protected void processVpnInterfaceDown(Uint64 dpId,
1339                                            String interfaceName,
1340                                            int lportTag,
1341                                            String gwMac,
1342                                            VpnInterfaceOpDataEntry vpnOpInterface,
1343                                            boolean isInterfaceStateDown,
1344                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1345                                            TypedWriteTransaction<Operational> writeOperTxn,
1346                                            TypedReadWriteTransaction<Configuration> writeInvTxn)
1347             throws ExecutionException, InterruptedException {
1348         if (vpnOpInterface == null) {
1349             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1350                     + " as it is not available in operational data store", interfaceName, dpId);
1351             return;
1352         }
1353         final String vpnName = vpnOpInterface.getVpnInstanceName();
1354         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1355                                                     interfaceName, vpnName);
1356         if (!isInterfaceStateDown) {
1357             final Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1358             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName,  null);
1359             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1360             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1361                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1362             if (interfaceManager.isExternalInterface(interfaceName)) {
1363                 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1364                     NwConstants.DEL_FLOW);
1365             }
1366             if (!isBgpVpnInternetVpn) {
1367                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1368             }
1369             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1370                     + " successful", interfaceName, dpId, vpnName);
1371         } else {
1372             // Interface is retained in the DPN, but its Link Down.
1373             // Only withdraw the prefixes for this interface from BGP
1374             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1375         }
1376     }
1377
1378     private void removeAdjacenciesFromVpn(final Uint64 dpnId, final int lportTag, final String interfaceName,
1379                                           final String vpnName, final Uint32 vpnId, String gwMac,
1380                                           TypedWriteTransaction<Configuration> writeConfigTxn,
1381                                           TypedWriteTransaction<Operational> writeOperTxn,
1382                                           TypedReadWriteTransaction<Configuration> writeInvTxn)
1383             throws ExecutionException, InterruptedException {
1384         //Read NextHops
1385         try {
1386             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1387                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1388             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1389             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1390                     LogicalDatastoreType.OPERATIONAL, path);
1391             boolean isLearntIP = false;
1392             String primaryRd = vpnUtil.getVpnRd(vpnName);
1393             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1394                     interfaceName, dpnId, vpnName, primaryRd);
1395             if (adjacencies.isPresent() && adjacencies.get().getAdjacency() != null
1396                     && !adjacencies.get().getAdjacency().isEmpty()) {
1397                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1398                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1399                         interfaceName, dpnId, vpnName, nextHops);
1400                 for (Adjacency nextHop : nextHops) {
1401                     if (nextHop.isPhysNetworkFunc()) {
1402                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1403                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1404                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1405                                 null/*writeCfgTxn*/);
1406                     } else {
1407                         String rd = nextHop.getVrfId();
1408                         List<String> nhList;
1409                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1410                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1411                             isLearntIP = nextHop.getAdjacencyType() == AdjacencyType.LearntIp ? true : false;
1412                         } else {
1413                             // This is a primary adjacency
1414                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1415                                     : emptyList();
1416                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1417                                     interfaceName, writeInvTxn);
1418                         }
1419                         if (!nhList.isEmpty()) {
1420                             if (Objects.equals(primaryRd, vpnName)) {
1421                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1422                                 //remove from FIB directly
1423                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1424                                         interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1425                             } else {
1426                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1427                                         interfaceName, writeConfigTxn, writeOperTxn);
1428                             }
1429                         } else {
1430                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1431                                             + " interface {}", nextHop.getIpAddress(), rd,
1432                                     nextHop.getAdjacencyType().toString(), interfaceName);
1433                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1434                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1435                         }
1436                     }
1437                     String ip = nextHop.getIpAddress().split("/")[0];
1438                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1439                     if (vpnVipToPort != null && vpnVipToPort.getPortName().equals(interfaceName)) {
1440                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1441                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1442                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1443                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1444                     }
1445                     // Remove the MIP-IP from VpnPortIpToPort.
1446                     if (isLearntIP) {
1447                         VpnPortipToPort persistedIp = vpnUtil.getVpnPortipToPort(vpnName, ip);
1448                         if (persistedIp != null && persistedIp.isLearntIp()
1449                                 && persistedIp.getPortName().equals(interfaceName)) {
1450                             VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1451                             LOG.info(
1452                                     "removeAdjacenciesFromVpn: Learnt-IP: {} interface {} of vpn {} removed "
1453                                             + "from VpnPortipToPort",
1454                                     persistedIp.getPortFixedip(), persistedIp.getPortName(), vpnName);
1455                         }
1456                     }
1457                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1458                     if (vpnPortipToPort != null) {
1459                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1460                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1461                                  + "Interface {} ip {} on dpn {} for vpn {}",
1462                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1463                     }
1464                 }
1465             } else {
1466                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1467                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1468                         + " Removing it.", interfaceName, vpnName, dpnId);
1469                 writeOperTxn.delete(identifier);
1470             }
1471         } catch (ReadFailedException e) {
1472             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1473                     interfaceName, dpnId, vpnName);
1474         }
1475     }
1476
1477     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1478                                                             String interfaceName, Uint64 dpnId,
1479                                                             TypedWriteTransaction<Configuration> writeConfigTxn,
1480                                                             TypedWriteTransaction<Operational> writeOperTx) {
1481         return (nh) -> {
1482             String primaryRd = vpnUtil.getVpnRd(vpnName);
1483             String prefix = nextHop.getIpAddress();
1484             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1485             LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1486                     nextHop, vpnName, interfaceName, dpnId);
1487             // FIXME: separate this out somehow?
1488             final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnNamePrefixKey);
1489             lock.lock();
1490             try {
1491                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1492                         prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1493                     //If extra-route is present behind at least one VM, then do not remove or update
1494                     //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1495                     //prefixtointerface DS
1496                     return;
1497                 }
1498                 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1499                         writeConfigTxn);
1500             } finally {
1501                 lock.unlock();
1502             }
1503             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1504                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1505                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1506         };
1507     }
1508
1509     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1510                                            Uint64 dpnId, String rd, String interfaceName,
1511                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1512                                            TypedWriteTransaction<Operational> writeOperTx) {
1513         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1514                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1515         nhList.forEach((nh) -> {
1516             //IRT: remove routes from other vpns importing it
1517             vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1518                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1519             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1520                 String vpnRd = vpn.getVrfId();
1521                 if (vpnRd != null) {
1522                     fibManager.removeOrUpdateFibEntry(vpnRd,
1523                             nextHop.getIpAddress(), nh, writeConfigTxn);
1524                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1525                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1526                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1527                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1528                 }
1529             }
1530         });
1531     }
1532
1533     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, Uint32 vpnId, Uint64 dpnId,
1534                                                  int lportTag, String gwMac, String interfaceName,
1535                                                  TypedReadWriteTransaction<Configuration> writeInvTxn)
1536             throws ExecutionException, InterruptedException {
1537         final Uuid subnetId = nextHop.getSubnetId();
1538         if (nextHop.getSubnetGatewayMacAddress() == null) {
1539             // A valid mac-address was not available for this subnet-gateway-ip
1540             // So a connected-mac-address was used for this subnet and we need
1541             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1542             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1543         }
1544         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1545                 subnetId);
1546     }
1547
1548     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, Uint64 dpnId,
1549                                                   String interfaceName) {
1550         // This is either an extra-route (or) a learned IP via subnet-route
1551         List<String> nhList = null;
1552         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1553         if (nextHopIp == null || nextHopIp.isEmpty()) {
1554             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1555                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1556                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1557                     vpnName);
1558             nhList = emptyList();
1559         } else {
1560             nhList = Collections.singletonList(nextHopIp);
1561         }
1562         return nhList;
1563     }
1564
1565     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1566         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1567         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1568         // else use connected interface
1569         if (gwPort != null && gwPort.isSubnetIp()) {
1570             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1571                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1572             return Optional.of(gwPort.getMacAddress());
1573         }
1574         return Optional.absent();
1575     }
1576
1577     @Override
1578     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1579         final VpnInterface update) {
1580         LOG.trace("Received VpnInterface update event: original={}, update={}", original, update);
1581         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}", update.getName(),
1582                 update.getDpnId(), original.getVpnInstanceNames(), update.getVpnInstanceNames());
1583         if (original.equals(update)) {
1584             LOG.info("update: original {} update {} are same. No update required.", original, update);
1585             return;
1586         }
1587         final String vpnInterfaceName = update.getName();
1588         final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1589         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1590         //handles switching between <internal VPN - external VPN>
1591         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1592             List<ListenableFuture<Void>> futures = new ArrayList<>();
1593             if (handleVpnInstanceUpdateForVpnInterface(identifier, original, update, futures)) {
1594                 LOG.info("update: handled Instance update for VPNInterface {} on dpn {} from oldVpn(s) {} "
1595                                 + "to newVpn(s) {}",
1596                         original.getName(), dpnId,
1597                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1598                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1599                 return emptyList();
1600             }
1601             updateVpnInstanceAdjChange(original, update, vpnInterfaceName, futures);
1602             return futures;
1603         });
1604     }
1605
1606     private boolean handleVpnInstanceUpdateForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1607                                                            VpnInterface original, VpnInterface update,
1608                                                            List<ListenableFuture<Void>> futures) {
1609         boolean isVpnInstanceUpdate = false;
1610         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1611         final String interfaceName = key.getName();
1612         List<String> oldVpnList = VpnUtil.getVpnListForVpnInterface(original);
1613         List<String> oldVpnListCopy = new ArrayList<>();
1614         oldVpnListCopy.addAll(oldVpnList);
1615         List<String> newVpnList = VpnUtil.getVpnListForVpnInterface(update);
1616         List<String> newVpnListCopy = new ArrayList<>();
1617         newVpnListCopy.addAll(newVpnList);
1618
1619         oldVpnList.removeAll(newVpnList);
1620         newVpnList.removeAll(oldVpnListCopy);
1621         //This block will execute only on if there is a change in the VPN Instance.
1622         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1623             /*
1624              * Internet BGP-VPN Instance update with single router:
1625              * ====================================================
1626              * In this case single VPN Interface will be part of maximum 2 VPN Instance only.
1627              *     1st VPN Instance : router VPN or external BGP-VPN.
1628              *     2nd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1629              *
1630              * VPN Instance UPDATE:
1631              * oldVpnList = 0 and newVpnList = 1 (Internet BGP-VPN)
1632              * oldVpnList = 1 and newVpnList = 0 (Internet BGP-VPN)
1633              *
1634              * External BGP-VPN Instance update with single router:
1635              * ====================================================
1636              * In this case single VPN interface will be part of maximum 1 VPN Instance only.
1637              *
1638              * Updated VPN Instance will be always either internal router VPN to
1639              * external BGP-VPN or external BGP-VPN to internal router VPN swap.
1640              *
1641              * VPN Instance UPDATE:
1642              * oldVpnList = 1 and newVpnList = 1 (router VPN to Ext-BGPVPN)
1643              * oldVpnList = 1 and newVpnList = 1 (Ext-BGPVPN to router VPN)
1644              *
1645              * Dual Router VPN Instance Update:
1646              * ================================
1647              * In this case single VPN interface will be part of maximum 3 VPN Instance only.
1648              *
1649              * 1st VPN Instance : router VPN or external BGP-VPN-1.
1650              * 2nd VPN Instance : router VPN or external BGP-VPN-2.
1651              * 3rd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1652              *
1653              * Dual Router --> Associated with common external BGP-VPN Instance.
1654              * 1st router and 2nd router are getting associated with single External BGP-VPN
1655              * 1) add 1st router to external bgpvpn --> oldVpnList=1, newVpnList=1;
1656              * 2) add 2nd router to the same external bgpvpn --> oldVpnList=1, newVpnList=0
1657              * In this case, we need to call removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1658              *
1659              *
1660              */
1661             isVpnInstanceUpdate = true;
1662             if (VpnUtil.isDualRouterVpnUpdate(oldVpnListCopy, newVpnListCopy)) {
1663                 if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1664                         && oldVpnList.size() == 1 && newVpnList.isEmpty()) {
1665                     //Identify the external BGP-VPN Instance and pass that value as newVpnList
1666                     List<String> externalBgpVpnList = new ArrayList<>();
1667                     for (String newVpnName : newVpnListCopy) {
1668                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1669                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1670                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1671                                 .BgpvpnType.BGPVPNExternal) {
1672                             externalBgpVpnList.add(newVpnName);
1673                             break;
1674                         }
1675                     }
1676                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1677                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList,
1678                             externalBgpVpnList, oldVpnListCopy, futures);
1679
1680                 } else if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1681                         && oldVpnList.isEmpty() && newVpnList.size() == 1) {
1682                     //Identify the router VPN Instance and pass that value as oldVpnList
1683                     List<String> routerVpnList = new ArrayList<>();
1684                     for (String newVpnName : newVpnListCopy) {
1685                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1686                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1687                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1688                                 .BgpvpnType.VPN) {
1689                             routerVpnList.add(newVpnName);
1690                             break;
1691                         }
1692                     }
1693                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1694                     updateVpnInstanceChange(identifier, interfaceName, original, update, routerVpnList,
1695                             newVpnList, oldVpnListCopy, futures);
1696
1697                 } else {
1698                     //Handle remaining use cases.
1699                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1700                             oldVpnListCopy, futures);
1701                 }
1702             } else {
1703                 updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1704                         oldVpnListCopy, futures);
1705             }
1706         }
1707         return isVpnInstanceUpdate;
1708     }
1709
1710     private void updateVpnInstanceChange(InstanceIdentifier<VpnInterface> identifier, String interfaceName,
1711                                          VpnInterface original, VpnInterface update, List<String> oldVpnList,
1712                                          List<String> newVpnList, List<String> oldVpnListCopy,
1713                                          List<ListenableFuture<Void>> futures) {
1714         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1715         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency() != null
1716                 ? origAdjs.getAdjacency() : new ArrayList<>();
1717         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1718         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency() != null
1719                 ? updateAdjs.getAdjacency() : new ArrayList<>();
1720
1721         boolean isOldVpnRemoveCallExecuted = false;
1722         for (String oldVpnName : oldVpnList) {
1723             LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1724                     + "remove from vpnName {} ", interfaceName, oldVpnName);
1725             removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1726             LOG.info("updateVpnInstanceChange: Processed Remove for update on VPNInterface"
1727                             + " {} upon VPN update from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName,
1728                     newVpnList);
1729             isOldVpnRemoveCallExecuted = true;
1730         }
1731         //Wait for previous interface bindings to be removed
1732         if (isOldVpnRemoveCallExecuted && !newVpnList.isEmpty()) {
1733             try {
1734                 Thread.sleep(2000);
1735             } catch (InterruptedException e) {
1736                 //Ignore
1737             }
1738         }
1739         for (String newVpnName : newVpnList) {
1740             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1741             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1742                 LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1743                         + "onto vpnName {} ", interfaceName, newVpnName);
1744                 addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1745                 LOG.info("updateVpnInstanceChange: Processed Add for update on VPNInterface {}"
1746                                 + "from oldVpn(s) {} to newVpn {} ",
1747                         interfaceName, oldVpnListCopy, newVpnName);
1748                 /* This block will execute only if V6 subnet is associated with internet BGP-VPN.
1749                  * Use Case:
1750                  *     In Dual stack network, first V4 subnet only attached to router and router is associated
1751                  *     with internet BGP-VPN(router-gw). At this point VPN interface is having only router vpn instance.
1752                  *     Later V6 subnet is added to router, at this point existing VPN interface will get updated
1753                  *     with Internet BGP-VPN instance(Note: Internet BGP-VPN Instance update in vpn interface
1754                  *     is applicable for only on V6 subnet is added to router). newVpnList = Contains only Internet
1755                  *     BGP-VPN Instance. So we required V6 primary adjacency info needs to be populated onto
1756                  *     router VPN as well as Internet BGP-VPN.
1757                  *
1758                  *     addVpnInterfaceCall() --> It will create V6 Adj onto Internet BGP-VPN only.
1759                  *     updateVpnInstanceAdjChange() --> This method call is needed for second primary V6 Adj
1760                  *                                       update in existing router VPN instance.
1761                  */
1762                 if (vpnUtil.isBgpVpnInternet(newVpnName)) {
1763                     LOG.info("updateVpnInstanceChange: VPN Interface {} with new Adjacency {} in existing "
1764                             + "VPN instance {}", interfaceName, newAdjs, original.getVpnInstanceNames());
1765                     updateVpnInstanceAdjChange(original, update, interfaceName, futures);
1766                 }
1767             }
1768         }
1769     }
1770
1771     private List<ListenableFuture<Void>> updateVpnInstanceAdjChange(VpnInterface original, VpnInterface update,
1772                                                                     String vpnInterfaceName,
1773                                                                     List<ListenableFuture<Void>> futures) {
1774         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1775         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1776                 != null ? origAdjs.getAdjacency() : new ArrayList<>();
1777         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1778         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1779                 != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1780
1781         final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1782         for (VpnInstanceNames vpnInterfaceVpnInstance : update.nonnullVpnInstanceNames()) {
1783             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1784             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1785             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1786             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1787             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1788                 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1789                 //set of prefix used as entry in prefix-to-interface datastore
1790                 // is prerequisite for refresh Fib to avoid race condition leading to missing remote next hop
1791                 // in bucket actions on bgp-vpn delete
1792                 Set<String> prefixListForRefreshFib = new HashSet<>();
1793                 ListenableFuture<Void> configTxFuture = txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
1794                     confTx -> futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL,
1795                         operTx -> {
1796                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1797                                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1798                             LOG.info("VPN Interface update event-intfName {} onto vpnName {} running config-driven",
1799                                     update.getName(), newVpnName);
1800                             //handle both addition and removal of adjacencies
1801                             // currently, new adjacency may be an extra route
1802                             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1803                             if (!oldAdjs.equals(newAdjs)) {
1804                                 for (Adjacency adj : copyNewAdjs) {
1805                                     if (copyOldAdjs.contains(adj)) {
1806                                         copyOldAdjs.remove(adj);
1807                                     } else {
1808                                         // add new adjacency
1809                                         if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1810                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1811                                                     dpnId, operTx, confTx, confTx, prefixListForRefreshFib);
1812                                         }
1813                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} "
1814                                             + " added to vpn interface {} on vpn {} dpnId {}",
1815                                             adj.getIpAddress(), adj.getNextHopIpList(), adj.getLabel(),
1816                                             adj.getSubnetId(), update.getName(), newVpnName, dpnId);
1817                                     }
1818                                 }
1819                                 for (Adjacency adj : copyOldAdjs) {
1820                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1821                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1822                                             && !adj.isPhysNetworkFunc()) {
1823                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, operTx,
1824                                                 confTx);
1825                                             //remove FIB entry
1826                                             String vpnRd = vpnUtil.getVpnRd(newVpnName);
1827                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1828                                                 + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1829                                             //remove BGP entry
1830                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1831                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1832                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1833                                             }
1834                                         } else {
1835                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1836                                                 operTx, confTx);
1837                                         }
1838                                     }
1839                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1840                                         + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1841                                         adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1842                                 }
1843                             }
1844                         })));
1845                 Futures.addCallback(configTxFuture, new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
1846                     MoreExecutors.directExecutor());
1847                 futures.add(configTxFuture);
1848                 for (ListenableFuture<Void> future : futures) {
1849                     ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1850                             update.getName(), update.getVpnInstanceNames());
1851                 }
1852             } else {
1853                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1854                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1855             }
1856         }
1857         return futures;
1858     }
1859
1860     private void updateLabelMapper(Uint32 label, List<String> nextHopIpList) {
1861         final String labelStr = Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!")
1862                 .toString();
1863         // FIXME: separate this out somehow?
1864         final ReentrantLock lock = JvmGlobalLocks.getLockForString(labelStr);
1865         lock.lock();
1866         try {
1867             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1868                     .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1869             Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1870                 LogicalDatastoreType.OPERATIONAL, lriIid);
1871             if (opResult.isPresent()) {
1872                 LabelRouteInfo labelRouteInfo =
1873                         new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1874                 SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1875                     labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1876             }
1877             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1878                     nextHopIpList);
1879         } catch (ReadFailedException e) {
1880             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1881                     nextHopIpList);
1882         } catch (TransactionCommitFailedException e) {
1883             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1884                     nextHopIpList);
1885         } finally {
1886             lock.unlock();
1887         }
1888     }
1889
1890     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, Uint32 label,
1891         SubnetRoute route, String parentVpnRd, TypedWriteTransaction<Configuration> writeConfigTxn) {
1892
1893         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1894         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1895                 .addAugmentation(SubnetRoute.class, route).build();
1896         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1897         InstanceIdentifierBuilder<VrfTables> idBuilder =
1898             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1899         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1900         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1901         if (writeConfigTxn != null) {
1902             writeConfigTxn.merge(vrfTableId, vrfTableNew, CREATE_MISSING_PARENTS);
1903         } else {
1904             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1905         }
1906         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1907                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1908     }
1909
1910     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1911                                            Adjacency adj, Uint64 dpnId,
1912                                            TypedWriteTransaction<Operational> writeOperTxn,
1913                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1914                                            TypedReadWriteTransaction<Configuration> writeInvTxn,
1915                                            Set<String> prefixListForRefreshFib)
1916             throws ExecutionException, InterruptedException {
1917         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1918         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1919         try {
1920             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1921                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1922             if (optVpnInterface.isPresent()) {
1923                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1924                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1925                 String vpnName = currVpnIntf.getVpnInstanceName();
1926                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1927                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1928                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1929                         LogicalDatastoreType.OPERATIONAL, adjPath);
1930                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1931                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1932                 Uint32 l3vni = vpnInstanceOpData.getL3vni() == null ? Uint32.ZERO : vpnInstanceOpData.getL3vni();
1933                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1934                 List<Adjacency> adjacencies = new ArrayList<>();
1935                 if (optAdjacencies.isPresent() && optAdjacencies.get().getAdjacency() != null) {
1936                     adjacencies.addAll(optAdjacencies.get().getAdjacency());
1937                 }
1938                 Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1939                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1940                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1941                 Adjacency operationalAdjacency = null;
1942                 //Handling dual stack neutron port primary adjacency
1943                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1944                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1945                             currVpnIntf.getName(), vpnName);
1946                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1947                             currVpnIntf.getName());
1948                     if (interfaceState != null) {
1949                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1950                             currVpnIntf.getName(), vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState,
1951                             prefixListForRefreshFib);
1952                     }
1953                 }
1954                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1955                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1956                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.LearntIp ? RouteOrigin.DYNAMIC
1957                             : RouteOrigin.STATIC;
1958                     String nh = adj.getNextHopIpList().get(0);
1959                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1960                     // FIXME: separate out to somehow?
1961                     final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
1962                     lock.lock();
1963                     try {
1964                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1965                                 vpnId, null, prefix, vpnName, nh, dpnId);
1966                         if (rdToAllocate.isPresent()) {
1967                             input.setRd(rdToAllocate.get());
1968                             operationalAdjacency = populator.createOperationalAdjacency(input);
1969                             int label = operationalAdjacency.getLabel().intValue();
1970                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1971                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
1972                                     currVpnIntf.getName(), operationalAdjacency, encapType,
1973                                     prefixListForRefreshFib, writeConfigTxn);
1974                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1975                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1976                                     vpnName, label, currVpnIntf.getName(), dpnId);
1977                         } else {
1978                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
1979                                     vpnName, prefix);
1980                             return;
1981                         }
1982                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1983                         // Keeping the MPLS check for now.
1984                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1985                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1986                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1987                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
1988                             vpnsToImportRoute.forEach(vpn -> {
1989                                 if (vpn.getVrfId() != null) {
1990                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(),
1991                                             vpnId, prefix,
1992                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
1993                                             .ifPresent(
1994                                                 rds -> vpnManager.addExtraRoute(
1995                                                         vpnUtil.getVpnName(vpn.getVpnId()), adj.getIpAddress(),
1996                                                         nh, rds, currVpnIntf.getVpnInstanceName(), l3vni,
1997                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(), opAdjacency,
1998                                                         encapType, prefixListForRefreshFib, writeConfigTxn));
1999                                 }
2000                             });
2001                         }
2002                     } finally {
2003                         lock.unlock();
2004                     }
2005                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
2006                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
2007                             currVpnIntf.getName(), vpnName);
2008
2009                     InstanceIdentifier<VpnInterface> vpnIfaceConfigidentifier = VpnUtil
2010                             .getVpnInterfaceIdentifier(currVpnIntf.getName());
2011                     Optional<VpnInterface> vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2012                             LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier);
2013                     Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix,
2014                             Prefixes.PrefixCue.PhysNetFunc);
2015                     if (vpnIntefaceConfig.isPresent()) {
2016                         pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix,
2017                                 vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(),
2018                                 vpnIntefaceConfig.get().getSegmentationId().toJava(), Prefixes.PrefixCue.PhysNetFunc);
2019                     }
2020
2021                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
2022
2023                     writeOperTxn.merge(
2024                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
2025                                     prefix), pnfPrefix, true);
2026
2027                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
2028                             adj.getIpAddress(), emptyList(), null /* EncapType */, Uint32.ZERO /* label */,
2029                             Uint32.ZERO /*l3vni*/, null /* gw-mac */, parentVpnRd,
2030                             RouteOrigin.LOCAL, writeConfigTxn);
2031
2032                     input.setRd(adj.getVrfId());
2033                 }
2034                 if (operationalAdjacency == null) {
2035                     operationalAdjacency = populator.createOperationalAdjacency(input);
2036                 }
2037                 adjacencies.add(operationalAdjacency);
2038                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
2039                 VpnInterfaceOpDataEntry newVpnIntf =
2040                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
2041                                 aug, dpnId, currVpnIntf.getLportTag().toJava(),
2042                                 currVpnIntf.getGatewayMacAddress());
2043
2044                 writeOperTxn.merge(identifier, newVpnIntf, CREATE_MISSING_PARENTS);
2045             }
2046         } catch (ReadFailedException e) {
2047             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
2048                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
2049         }
2050     }
2051
2052     @Nullable
2053     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
2054         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
2055         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
2056     }
2057
2058     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
2059                                             Uint64 dpnId, TypedWriteTransaction<Operational> writeOperTxn,
2060                                             TypedWriteTransaction<Configuration> writeConfigTxn) {
2061         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
2062         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
2063         try {
2064             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
2065                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
2066             if (optVpnInterface.isPresent()) {
2067                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
2068                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
2069                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2070                         LogicalDatastoreType.OPERATIONAL, path);
2071                 if (optAdjacencies.isPresent()) {
2072                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
2073
2074                     if (adjacencies != null && !adjacencies.isEmpty()) {
2075                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
2076                         for (Adjacency adjacency : adjacencies) {
2077                             if (Objects.equals(adjacency.getIpAddress(), adj.getIpAddress())) {
2078                                 String rd = adjacency.getVrfId();
2079                                 if (adj.getNextHopIpList() != null) {
2080                                     for (String nh : adj.getNextHopIpList()) {
2081                                         deleteExtraRouteFromCurrentAndImportingVpns(
2082                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
2083                                                 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
2084                                     }
2085                                 } else if (adj.isPhysNetworkFunc()) {
2086                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
2087                                             adj.getIpAddress(), adj.getSubnetId());
2088                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
2089                                             writeConfigTxn);
2090                                 }
2091                                 break;
2092                             }
2093
2094                         }
2095                     }
2096                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
2097                             dpnId, adj.getVrfId());
2098                 } else {
2099                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
2100                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
2101                 }
2102             }
2103         } catch (ReadFailedException e) {
2104             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
2105                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
2106         }
2107     }
2108
2109     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
2110                                     String rd, String intfName, TypedWriteTransaction<Configuration> writeConfigTxn,
2111                                     TypedWriteTransaction<Operational> writeOperTx) {
2112         LOG.info("removing extra-route {} for nexthop {} in VPN {} intfName {} rd {}",
2113                 destination, nextHop, vpnName, intfName, rd);
2114         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
2115         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
2116         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
2117             String vpnRd = vpn.getVrfId();
2118             if (vpnRd != null) {
2119                 LOG.info("deleting extra-route {} for nexthop {} in VPN {} intfName {} vpnRd {}",
2120                         destination, nextHop, vpnName, intfName, vpnRd);
2121                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
2122                         writeOperTx);
2123             }
2124         }
2125     }
2126
2127     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, Uint64 dpnId) {
2128         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2129             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
2130             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
2131     }
2132
2133     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
2134         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2135             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
2136     }
2137
2138     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2139                                                 TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2140         if (vpnInterface == null) {
2141             return;
2142         }
2143         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
2144         if (adjs == null) {
2145             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2146                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2147             return;
2148         }
2149         for (Adjacency adj : adjs) {
2150             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2151                 String primaryInterfaceIp = adj.getIpAddress();
2152                 String macAddress = adj.getMacAddress();
2153                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2154
2155                 Uint32 label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
2156                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2157
2158                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2159                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2160                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
2161                         routerInt, label, writeConfigTxn);
2162                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2163                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2164                         macAddress);
2165             } else {
2166                 LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2167                                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}",
2168                         interfaceName, primaryRd, vpnName);
2169             }
2170         }
2171     }
2172
2173     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
2174             TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2175         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
2176         String rd = vpnUtil.getVpnRd(vpnName);
2177         if (adjs != null) {
2178             List<Adjacency> adjsList = adjs.nonnullAdjacency();
2179             for (Adjacency adj : adjsList) {
2180                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2181                     String primaryInterfaceIp = adj.getIpAddress();
2182                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2183                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
2184                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2185                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2186                 }
2187             }
2188         } else {
2189             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2190                     vpnInterface.getName(), rd);
2191         }
2192     }
2193
2194     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2195         final VpnInterfaceKey key = intefaceData.identifier.firstKeyOf(VpnInterface.class);
2196         final String interfaceName = key.getName();
2197         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2198                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2199         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2200                   intefaceData.identifier, vpnName);
2201     }
2202
2203     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2204                                               VpnInterface vpnInterface, String vpnName) {
2205         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2206                .get(vpnName);
2207         if (vpnInterfaces == null) {
2208             vpnInterfaces = new ConcurrentLinkedQueue<>();
2209         }
2210         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2211         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2212         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2213                 vpnInterface.getName(), vpnName);
2214     }
2215
2216     public boolean isVpnInstanceReady(String vpnInstanceName) {
2217         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2218         if (vpnRd == null) {
2219             return false;
2220         }
2221         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2222
2223         return vpnInstanceOpDataEntry != null;
2224     }
2225
2226     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2227         // FIXME: separate out to somehow?
2228         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnInstanceName);
2229         lock.lock();
2230         try {
2231             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2232                     unprocessedVpnInterfaces.get(vpnInstanceName);
2233             if (vpnInterfaces != null) {
2234                 while (!vpnInterfaces.isEmpty()) {
2235                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2236                     if (hasVpnInstanceCreatedSuccessfully) {
2237                         processSavedInterface(savedInterface, vpnInstanceName);
2238                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2239                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2240                     } else {
2241                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2242                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2243                     }
2244                 }
2245             } else {
2246                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2247             }
2248         } finally {
2249             lock.unlock();
2250         }
2251     }
2252
2253     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2254             VpnInterface vpnInterface) {
2255         // FIXME: use VpnInstanceNamesKey perhaps? What about nulls?
2256         final String firstVpnName = VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface);
2257         final ReentrantLock lock = JvmGlobalLocks.getLockForString(firstVpnName);
2258         lock.lock();
2259         try {
2260             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2261                     unprocessedVpnInterfaces.get(firstVpnName);
2262             if (vpnInterfaces != null) {
2263                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2264                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2265                             + "unprocessed list", vpnInterface.getName(), firstVpnName);
2266                 }
2267             } else {
2268                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}", firstVpnName);
2269             }
2270         } finally {
2271             lock.unlock();
2272         }
2273     }
2274
2275     public void vpnInstanceIsReady(String vpnInstanceName) {
2276         processSavedInterfaces(vpnInstanceName, true);
2277     }
2278
2279     public void vpnInstanceFailed(String vpnInstanceName) {
2280         processSavedInterfaces(vpnInstanceName, false);
2281     }
2282
2283     private static class UnprocessedVpnInterfaceData {
2284         InstanceIdentifier<VpnInterface> identifier;
2285         VpnInterface vpnInterface;
2286
2287         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2288             this.identifier = identifier;
2289             this.vpnInterface = vpnInterface;
2290         }
2291
2292         @Override
2293         public int hashCode() {
2294             final int prime = 31;
2295             int result = 1;
2296             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2297             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2298             return result;
2299         }
2300
2301         @Override
2302         public boolean equals(Object obj) {
2303             if (this == obj) {
2304                 return true;
2305             }
2306             if (obj == null) {
2307                 return false;
2308             }
2309             if (getClass() != obj.getClass()) {
2310                 return false;
2311             }
2312             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2313             if (identifier == null) {
2314                 if (other.identifier != null) {
2315                     return false;
2316                 }
2317             } else if (!identifier.equals(other.identifier)) {
2318                 return false;
2319             }
2320             if (vpnInterface == null) {
2321                 if (other.vpnInterface != null) {
2322                     return false;
2323                 }
2324             } else if (!vpnInterface.equals(other.vpnInterface)) {
2325                 return false;
2326             }
2327             return true;
2328         }
2329     }
2330
2331     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2332         String primaryRd = vpnUtil.getVpnRd(vpnName);
2333         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2334         if (vpnInstanceOpData == null) {
2335             return;
2336         }
2337         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2338         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2339             return;
2340         }
2341         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2342         vpnToDpnLists.forEach(vpnToDpnList -> {
2343             if (vpnToDpnList.getVpnInterfaces() == null) {
2344                 return;
2345             }
2346             vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2347                 try {
2348                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2349                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2350                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2351                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2352                     if (!vpnInterfaceOptional.isPresent()) {
2353                         return;
2354                     }
2355                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2356                             vpnInterface.getInterfaceName());
2357                     if (configVpnAdjacencies == null) {
2358                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2359                         return;
2360                     }
2361                     List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2362                             .augmentation(AdjacenciesOp.class).nonnullAdjacency();
2363                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2364                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2365                     // in operational DS. These unprocessed adjacencies will be handled below.
2366                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2367                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2368                     configVpnAdjacencies.stream()
2369                         .filter(adjacency -> operationVpnAdjacencies.stream()
2370                                 .noneMatch(operationalAdjacency ->
2371                                     Objects.equals(operationalAdjacency.getIpAddress(), adjacency.getIpAddress())))
2372                         .forEach(adjacency -> {
2373                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2374                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2375                                 () -> {
2376                                     // TODO Deal with sequencing — the config tx must only submitted
2377                                     // if the oper tx goes in
2378                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2379                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2380                                         futures.add(
2381                                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
2382                                                 //set of prefix used, as entry in prefix-to-interface datastore
2383                                                 // is prerequisite for refresh Fib to avoid race condition leading
2384                                                 // to missing remote next hop in bucket actions on bgp-vpn delete
2385                                                 Set<String> prefixListForRefreshFib = new HashSet<>();
2386                                                 ListenableFuture<Void> configTxFuture =
2387                                                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
2388                                                         confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2389                                                             primaryRd, adjacency,
2390                                                             vpnInterfaceOptional.get().getDpnId(),
2391                                                                 operTx, confTx, confTx, prefixListForRefreshFib));
2392                                                 Futures.addCallback(configTxFuture,
2393                                                     new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
2394                                                     MoreExecutors.directExecutor());
2395                                                 futures.add(configTxFuture);
2396                                             }));
2397                                         return futures;
2398                                     } else {
2399                                         return emptyList();
2400                                     }
2401                                 });
2402                         });
2403                 } catch (ReadFailedException e) {
2404                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2405                             vpnName, primaryRd);
2406                 }
2407             });
2408         });
2409     }
2410
2411     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2412         private final String interfaceName;
2413         private final boolean add;
2414         private final String txnDestination;
2415
2416         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2417             this.interfaceName = interfaceName;
2418             this.add = add;
2419             this.txnDestination = transactionDest;
2420         }
2421
2422         @Override
2423         public void onSuccess(Void voidObj) {
2424             if (add) {
2425                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2426                         interfaceName, txnDestination);
2427             } else {
2428                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2429             }
2430         }
2431
2432         @Override
2433         public void onFailure(Throwable throwable) {
2434             if (add) {
2435                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2436                         interfaceName, txnDestination, throwable);
2437             } else {
2438                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2439                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2440             }
2441         }
2442     }
2443
2444     private class VpnInterfaceCallBackHandler implements FutureCallback<Void> {
2445         private final String primaryRd;
2446         private final Set<String> prefixListForRefreshFib;
2447
2448         VpnInterfaceCallBackHandler(String primaryRd, Set<String> prefixListForRefreshFib) {
2449             this.primaryRd = primaryRd;
2450             this.prefixListForRefreshFib = prefixListForRefreshFib;
2451         }
2452
2453         @Override
2454         public void onSuccess(Void voidObj) {
2455             prefixListForRefreshFib.forEach(prefix -> {
2456                 fibManager.refreshVrfEntry(primaryRd, prefix);
2457             });
2458         }
2459
2460         @Override
2461         public void onFailure(Throwable throwable) {
2462             LOG.debug("write Tx config operation failed", throwable);
2463         }
2464     }
2465 }