vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14
  15 import com.google.common.base.Optional;
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.collect.Iterators;
  18 import com.google.common.util.concurrent.FutureCallback;
  19 import com.google.common.util.concurrent.Futures;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import com.google.common.util.concurrent.MoreExecutors;
  22 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  23 import java.util.ArrayList;
  24 import java.util.Collections;
  25 import java.util.HashSet;
  26 import java.util.List;
  27 import java.util.Map;
  28 import java.util.Objects;
  29 import java.util.Set;
  30 import java.util.concurrent.ConcurrentHashMap;
  31 import java.util.concurrent.ConcurrentLinkedQueue;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.locks.ReentrantLock;
  34 import java.util.function.Consumer;
  35 import java.util.function.Predicate;
  36 import java.util.stream.Collectors;
  37 import javax.annotation.PostConstruct;
  38 import javax.annotation.PreDestroy;
  39 import javax.inject.Inject;
  40 import javax.inject.Singleton;
  41 import org.eclipse.jdt.annotation.Nullable;
  42 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  43 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  44 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  45 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  46 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  47 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  48 import org.opendaylight.genius.infra.Datastore.Configuration;
  49 import org.opendaylight.genius.infra.Datastore.Operational;
  50 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  51 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  52 import org.opendaylight.genius.infra.TransactionAdapter;
  53 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  54 import org.opendaylight.genius.infra.TypedWriteTransaction;
  55 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  56 import org.opendaylight.genius.mdsalutil.NWUtil;
  57 import org.opendaylight.genius.mdsalutil.NwConstants;
  58 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  59 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  60 import org.opendaylight.genius.utils.JvmGlobalLocks;
  61 import org.opendaylight.infrautils.caches.CacheProvider;
  62 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  63 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  64 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  65 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  66 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  67 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  68 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  69 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  70 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  71 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  72 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  73 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  74 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  75 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  76 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  77 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  78 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  79 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  80 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 120 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 121 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 122 import org.opendaylight.yangtools.yang.common.Uint32;
 123 import org.opendaylight.yangtools.yang.common.Uint64;
 124 import org.slf4j.Logger;
 125 import org.slf4j.LoggerFactory;
 126
 127 @Singleton
 128 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 129
 130     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 131     private static final short DJC_MAX_RETRIES = 3;
 132
 133     private final DataBroker dataBroker;
 134     private final ManagedNewTransactionRunner txRunner;
 135     private final IBgpManager bgpManager;
 136     private final IFibManager fibManager;
 137     private final IMdsalApiManager mdsalManager;
 138     private final IdManagerService idManager;
 139     private final OdlInterfaceRpcService ifaceMgrRpcService;
 140     private final VpnFootprintService vpnFootprintService;
 141     private final IInterfaceManager interfaceManager;
 142     private final IVpnManager vpnManager;
 143     private final ArpResponderHandler arpResponderHandler;
 144     private final JobCoordinator jobCoordinator;
 145     private final VpnUtil vpnUtil;
 146
 147     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 148
 149     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 150             new ConcurrentHashMap<>();
 151
 152     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 153
 154     @Inject
 155     public VpnInterfaceManager(final DataBroker dataBroker,
 156                                final IBgpManager bgpManager,
 157                                final IdManagerService idManager,
 158                                final IMdsalApiManager mdsalManager,
 159                                final IFibManager fibManager,
 160                                final OdlInterfaceRpcService ifaceMgrRpcService,
 161                                final VpnFootprintService vpnFootprintService,
 162                                final IInterfaceManager interfaceManager,
 163                                final IVpnManager vpnManager,
 164                                final ArpResponderHandler arpResponderHandler,
 165                                final JobCoordinator jobCoordinator,
 166                                final CacheProvider cacheProvider,
 167                                final VpnUtil vpnUtil) {
 168         super(VpnInterface.class, VpnInterfaceManager.class);
 169
 170         this.dataBroker = dataBroker;
 171         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 172         this.bgpManager = bgpManager;
 173         this.idManager = idManager;
 174         this.mdsalManager = mdsalManager;
 175         this.fibManager = fibManager;
 176         this.ifaceMgrRpcService = ifaceMgrRpcService;
 177         this.vpnFootprintService = vpnFootprintService;
 178         this.interfaceManager = interfaceManager;
 179         this.vpnManager = vpnManager;
 180         this.arpResponderHandler = arpResponderHandler;
 181         this.jobCoordinator = jobCoordinator;
 182         this.vpnUtil = vpnUtil;
 183
 184         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 185                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 186                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 187     }
 188
 189     public Runnable isNotifyTaskQueued(String intfName) {
 190         return vpnIntfMap.remove(intfName);
 191     }
 192
 193     @PostConstruct
 194     public void start() {
 195         LOG.info("{} start", getClass().getSimpleName());
 196         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 197     }
 198
 199     @Override
 200     @PreDestroy
 201     public void close() {
 202         super.close();
 203         vpnInstanceOpDataEntryCache.close();
 204     }
 205
 206     @Override
 207     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 208         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 209     }
 210
 211     @Override
 212     protected VpnInterfaceManager getDataTreeChangeListener() {
 213         return VpnInterfaceManager.this;
 214     }
 215
 216     @Override
 217     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 218         LOG.trace("Received VpnInterface add event: vpnInterface={}", vpnInterface);
 219         LOG.info("add: intfName {} onto vpnName {}", vpnInterface.getName(),
 220                 VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 221         addVpnInterface(identifier, vpnInterface, null, null);
 222     }
 223
 224     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 225                           final VpnInterface vpnInterface, String vpnName) {
 226         // FIXME: separate this out somehow?
 227         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnName);
 228         lock.lock();
 229         try {
 230             if (isVpnInstanceReady(vpnName)) {
 231                 return true;
 232             }
 233             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 234             return false;
 235         } finally {
 236             lock.unlock();
 237         }
 238     }
 239
 240     // TODO Clean up the exception handling
 241     @SuppressWarnings("checkstyle:IllegalCatch")
 242     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 243                              final @Nullable List<Adjacency> oldAdjs, final @Nullable List<Adjacency> newAdjs) {
 244         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
 245             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 246             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 247         }
 248     }
 249
 250     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 251                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 252         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 253         final String interfaceName = key.getName();
 254
 255         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 256             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 257                   vpnName, vpnInterface.getName());
 258             return;
 259         }
 260         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 261              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 262         List<Adjacency> copyOldAdjs = null;
 263         if (oldAdjs != null) {
 264             copyOldAdjs = new ArrayList<>();
 265             copyOldAdjs.addAll(oldAdjs);
 266         }
 267         List<Adjacency> copyNewAdjs = null;
 268         if (newAdjs != null) {
 269             copyNewAdjs = new ArrayList<>();
 270             copyNewAdjs.addAll(newAdjs);
 271         }
 272         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 273     }
 274
 275     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 276                     final VpnInterface vpnInterface, final @Nullable List<Adjacency> oldAdjs,
 277                     final @Nullable List<Adjacency> newAdjs,
 278                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 279         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 280         final String interfaceName = key.getName();
 281         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 282         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 283             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 284             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 285             if (interfaceState != null) {
 286                 try {
 287                     final Uint64 dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 288                     final int ifIndex = interfaceState.getIfIndex();
 289                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 290                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 291                         // (the inventory tx goes in last)
 292                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 293                         //set of prefix used, as entry in prefix-to-interface datastore
 294                         // is prerequisite for refresh Fib to avoid race condition leading to
 295                         // missing remote next hop in bucket actions on bgp-vpn delete
 296                         Set<String> prefixListForRefreshFib = new HashSet<>();
 297                         ListenableFuture<Void> confFuture =
 298                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 299                                 confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
 300                                     operTx -> futures.add(
 301                                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, invTx -> {
 302                                             LOG.info(
 303                                                 "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 304                                                     + " on dpn {}",
 305                                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 306                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 307                                                 confTx, operTx, invTx, interfaceState, vpnName,
 308                                                 prefixListForRefreshFib);
 309                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 310                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 311                                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 312                                                 if (newAdjs != null) {
 313                                                     for (Adjacency adj : newAdjs) {
 314                                                         if (oldAdjs.contains(adj)) {
 315                                                             oldAdjs.remove(adj);
 316                                                         } else {
 317                                                             if (!isBgpVpnInternetVpn
 318                                                                 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 319                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 320                                                                     primaryRd, adj, dpnId, operTx, confTx, invTx,
 321                                                                     prefixListForRefreshFib);
 322                                                             }
 323                                                         }
 324                                                     }
 325                                                 }
 326                                                 for (Adjacency adj : oldAdjs) {
 327                                                     if (!isBgpVpnInternetVpn
 328                                                         || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 329                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 330                                                             operTx, confTx);
 331                                                     }
 332                                                 }
 333                                             }
 334                                         })))));
 335                         Futures.addCallback(confFuture,
 336                             new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
 337                             MoreExecutors.directExecutor());
 338                         futures.add(confFuture);
 339                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 340                             MoreExecutors.directExecutor());
 341                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 342                             + " processed successfully", interfaceName, vpnName, dpnId);
 343                         return futures;
 344                     });
 345                 } catch (NumberFormatException | IllegalStateException e) {
 346                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 347                             + "interface {}. Interface addition on vpn {} failed", interfaceName,
 348                         vpnName, e);
 349                     return;
 350                 }
 351             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 352                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 353                     () -> {
 354                         ListenableFuture<Void> future =
 355                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
 356                                 createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 357                                     confTx, vpnName);
 358                                 LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 359                                     vpnName, vpnInterface.getDpnId());
 360                             });
 361                         ListenableFutures.addErrorLogging(future, LOG,
 362                             "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 363                         return Collections.singletonList(future);
 364                     });
 365             } else {
 366                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 367                     + " is not available", interfaceName, vpnName);
 368             }
 369         } else {
 370             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 371                     + " as vpn is pending delete", interfaceName, vpnName,
 372                 vpnInterface.getDpnId());
 373         }
 374     }
 375
 376     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 377     // see comments below.
 378     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 379     protected void processVpnInterfaceUp(final Uint64 dpId, VpnInterface vpnInterface, final String primaryRd,
 380             final int lportTag, boolean isInterfaceUp,
 381             TypedWriteTransaction<Configuration> writeConfigTxn,
 382             TypedWriteTransaction<Operational> writeOperTxn,
 383             TypedReadWriteTransaction<Configuration> writeInvTxn,
 384             Interface interfaceState, final String vpnName,
 385             Set<String> prefixListForRefreshFib) throws ExecutionException, InterruptedException {
 386         final String interfaceName = vpnInterface.getName();
 387         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 388                 vpnName);
 389         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 390         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 391         if (!isInterfaceUp) {
 392             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 393                      interfaceName, dpId, vpnName);
 394             Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 395             if (VpnConstants.INVALID_ID.equals(vpnId)) {
 396                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 397                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 398                         dpId);
 399                 return;
 400             }
 401
 402             boolean waitForVpnInterfaceOpRemoval = false;
 403             if (opVpnInterface != null) {
 404                 String opVpnName = opVpnInterface.getVpnInstanceName();
 405                 String primaryInterfaceIp = null;
 406                 if (Objects.equals(opVpnName, vpnName)) {
 407                     // Please check if the primary VRF Entry does not exist for VPNInterface
 408                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 409                     // back to back
 410                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 411                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 412                     if (adjs == null) {
 413                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 414                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 415                                 vpnName);
 416                         return;
 417                     }
 418                     for (Adjacency adj : adjs) {
 419                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 420                             primaryInterfaceIp = adj.getIpAddress();
 421                             break;
 422                         }
 423                     }
 424                     if (primaryInterfaceIp == null) {
 425                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 426                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 427                                 dpId, vpnName);
 428                         return;
 429                     }
 430                     // Get the rd of the vpn instance
 431                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 432                     if (vrf != null) {
 433                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 434                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 435                         return;
 436                     }
 437                     waitForVpnInterfaceOpRemoval = true;
 438                 } else {
 439                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 440                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 441                 }
 442             }
 443             if (!waitForVpnInterfaceOpRemoval) {
 444                 // Add the VPNInterface and quit
 445                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 446                         null/*ipAddressSourceValuePair*/,
 447                         true /* add */);
 448                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 449                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 450                 if (!isBgpVpnInternetVpn) {
 451                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 452                 }
 453                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 454                         dpId, vpnName);
 455                 if (interfaceManager.isExternalInterface(interfaceName)) {
 456                     processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
 457                         NwConstants.ADD_FLOW);
 458                 }
 459                 return;
 460             }
 461
 462             // FIB didn't get a chance yet to clean up this VPNInterface
 463             // Let us give it a chance here !
 464             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 465                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 466             try {
 467                 Runnable notifyTask = new VpnNotifyTask();
 468                 synchronized (notifyTask) {
 469                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 470                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 471                     // published here while holding the lock on it so this path will hit the wait before notify can be
 472                     // invoked.
 473                     vpnIntfMap.put(interfaceName, notifyTask);
 474                     try {
 475                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 476                     } catch (InterruptedException e) {
 477                         // Ignored
 478                     }
 479                 }
 480             } finally {
 481                 vpnIntfMap.remove(interfaceName);
 482             }
 483
 484             if (opVpnInterface != null) {
 485                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 486                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 487                         dpId, vpnName);
 488                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 489                 return;
 490             }
 491             // VPNInterface got removed, proceed with Add
 492             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 493                     interfaceName, dpId, vpnName);
 494             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 495                     null/*ipAddressSourceValuePair*/,
 496                     true /* add */);
 497             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 498                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 499             if (!isBgpVpnInternetVpn) {
 500                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 501             }
 502             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 503                     + " FIB to clean up", interfaceName, dpId, vpnName);
 504             if (interfaceManager.isExternalInterface(interfaceName)) {
 505                 processExternalVpnInterface(interfaceName, vpnName, dpId,
 506                                lportTag, NwConstants.ADD_FLOW);
 507             }
 508         } else {
 509             try {
 510                 // Interface is retained in the DPN, but its Link Up.
 511                 // Advertise prefixes again for this interface to BGP
 512                 InstanceIdentifier<VpnInterface> identifier =
 513                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 514                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 515                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 516                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 517                 // Perform similar operation as interface add event for extraroutes.
 518                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 519                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 520                         LogicalDatastoreType.CONFIGURATION, path);
 521                 if (!optAdjacencies.isPresent()) {
 522                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 523                     return;
 524                 }
 525                 List<Adjacency> adjacencies = optAdjacencies.get().nonnullAdjacency();
 526                 for (Adjacency adjacency : adjacencies) {
 527                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 528                         continue;
 529                     }
 530                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 531                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 532                         continue;
 533                     }
 534                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 535                             dpId, writeOperTxn, writeConfigTxn, writeInvTxn, prefixListForRefreshFib);
 536                 }
 537             } catch (ReadFailedException e) {
 538                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 539                         interfaceName, vpnName, primaryRd, dpId);
 540             }
 541         }
 542     }
 543
 544     private void processExternalVpnInterface(String interfaceName, String vpnName, Uint64 dpId,
 545         int lportTag, int addOrRemove) {
 546         Uuid extNetworkId;
 547         try {
 548             // vpn instance of ext-net interface is the network-id
 549             extNetworkId = new Uuid(vpnName);
 550         } catch (IllegalArgumentException e) {
 551             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 552                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 553             return;
 554         }
 555
 556         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 557         if (routerIds == null || routerIds.isEmpty()) {
 558             LOG.info("processExternalVpnInterface: No router is associated with {}."
 559                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 560                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 561             return;
 562         }
 563
 564         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 565                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 566         for (Uuid routerId : routerIds) {
 567             String routerName = routerId.getValue();
 568             Uint64 primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 569             if (Objects.equals(primarySwitch, dpId)) {
 570                 Routers router = vpnUtil.getExternalRouter(routerName);
 571                 if (router != null) {
 572                     if (addOrRemove == NwConstants.ADD_FLOW) {
 573                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 574                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 575                                 dpId, interfaceName, lportTag);
 576                     } else {
 577                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 578                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 579                                 dpId, interfaceName, lportTag);
 580                     }
 581                 } else {
 582                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 583                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 584                             interfaceName, dpId, vpnName);
 585                 }
 586             }
 587         }
 588     }
 589
 590     // TODO Clean up the exception handling
 591     @SuppressWarnings("checkstyle:IllegalCatch")
 592     private void advertiseAdjacenciesForVpnToBgp(final String rd, Uint64 dpnId,
 593                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 594                                                   String vpnName, String interfaceName) {
 595         if (rd == null) {
 596             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 597                   interfaceName, dpnId, vpnName);
 598             return;
 599         } else {
 600             if (rd.equals(vpnName)) {
 601                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 602                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 603                 return;
 604             }
 605         }
 606         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 607                 interfaceName, dpnId, vpnName, rd);
 608
 609         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 610         if (nextHopIp == null) {
 611             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 612                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 613                     rd, vpnName);
 614             return;
 615         }
 616
 617         try {
 618             //Read NextHops
 619             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 620             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 621                     LogicalDatastoreType.OPERATIONAL, path);
 622             if (adjacencies.isPresent()) {
 623                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 624                 if (nextHops != null && !nextHops.isEmpty()) {
 625                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 626                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 627                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 628                     Uint32 l3vni = vpnInstanceOpData.getL3vni();
 629                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 630                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 631                     for (Adjacency nextHop : nextHops) {
 632                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 633                             continue;
 634                         }
 635                         String gatewayMac = null;
 636                         Uint32 label = Uint32.ZERO;
 637                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 638                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 639                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 640                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 641                                     .get();
 642                         } else {
 643                             label = nextHop.getLabel();
 644                         }
 645                         try {
 646                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 647                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 648                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 649                                     encapType, label, l3vni, Uint32.ZERO /*l2vni*/,
 650                                     gatewayMac);
 651                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 652                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 653                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 654                         } catch (Exception e) {
 655                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 656                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 657                                     interfaceName, dpnId, e);
 658                         }
 659                     }
 660                 }
 661             }
 662         } catch (ReadFailedException e) {
 663             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 664                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 665         }
 666     }
 667
 668     // TODO Clean up the exception handling
 669     @SuppressWarnings("checkstyle:IllegalCatch")
 670     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 671                         String vpnName, String interfaceName, TypedWriteTransaction<Configuration> writeConfigTxn,
 672                         TypedWriteTransaction<Operational> writeOperTx) {
 673         //Read NextHops
 674         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 675         String rd = vpnUtil.getVpnRd(interfaceName);
 676         if (rd == null) {
 677             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 678                 interfaceName, vpnName);
 679             return;
 680         } else {
 681             if (rd.equals(vpnName)) {
 682                 LOG.info(
 683                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 684                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 685                 return;
 686             }
 687         }
 688         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 689                vpnName, rd);
 690         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 691         try {
 692             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 693                     path);
 694         } catch (ReadFailedException e) {
 695             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 696                     interfaceName, vpnName);
 697         }
 698         if (adjacencies.isPresent()) {
 699             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 700
 701             if (nextHops != null && !nextHops.isEmpty()) {
 702                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 703                         nextHops, interfaceName, vpnName, rd);
 704                 for (Adjacency nextHop : nextHops) {
 705                     try {
 706                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 707                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 708                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 709                                     interfaceName, vpnName);
 710                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 711                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 712                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 713                                     interfaceName, vpnName);
 714                         } else if (nextHop.getNextHopIpList() != null) {
 715                             // Perform similar operation as interface delete event for extraroutes.
 716                             String allocatedRd = nextHop.getVrfId();
 717                             for (String nh : nextHop.getNextHopIpList()) {
 718                                 deleteExtraRouteFromCurrentAndImportingVpns(
 719                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
 720                                     writeOperTx);
 721                             }
 722                         }
 723                     } catch (Exception e) {
 724                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 725                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 726                     }
 727                 }
 728             }
 729         }
 730     }
 731
 732     @SuppressWarnings("checkstyle:IllegalCatch")
 733     protected void processVpnInterfaceAdjacencies(Uint64 dpnId, final int lportTag, String vpnName,
 734                                                   String primaryRd, String interfaceName, final Uint32 vpnId,
 735                                                   TypedWriteTransaction<Configuration> writeConfigTxn,
 736                                                   TypedWriteTransaction<Operational> writeOperTxn,
 737                                                   TypedReadWriteTransaction<Configuration> writeInvTxn,
 738                                                   Interface interfaceState, Set<String> prefixListForRefreshFib)
 739             throws ExecutionException, InterruptedException {
 740         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 741         // Read NextHops
 742         Optional<VpnInterface> vpnInteface = Optional.absent();
 743         try {
 744             vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 745             LogicalDatastoreType.CONFIGURATION, identifier);
 746         } catch (ReadFailedException e) {
 747             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 748                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 749         }
 750         Uuid intfnetworkUuid = null;
 751         NetworkType networkType = null;
 752         Long segmentationId = Long.valueOf(-1);
 753         Adjacencies adjacencies = null;
 754         if (vpnInteface.isPresent()) {
 755             intfnetworkUuid = vpnInteface.get().getNetworkId();
 756             networkType = vpnInteface.get().getNetworkType();
 757             segmentationId = vpnInteface.get().getSegmentationId().toJava();
 758             adjacencies = vpnInteface.get().augmentation(Adjacencies.class);
 759             if (adjacencies == null) {
 760                 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 761                         null/*gwMac*/, writeOperTxn);
 762                 return;
 763             }
 764         }
 765         // Get the rd of the vpn instance
 766         String nextHopIp = null;
 767         try {
 768             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 769         } catch (Exception e) {
 770             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 771                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 772         }
 773         List<String> nhList = new ArrayList<>();
 774         if (nextHopIp != null) {
 775             nhList.add(nextHopIp);
 776             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 777                     interfaceName, dpnId, vpnName, nhList);
 778         }
 779         Optional<String> gwMac = Optional.absent();
 780         String vpnInterfaceSubnetGwMacAddress = null;
 781         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 782         Uint32 l3vni = vpnInstanceOpData.getL3vni() != null ? vpnInstanceOpData.getL3vni() : Uint32.ZERO;
 783         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 784         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 785         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 786         List<Adjacency> nextHops = adjacencies != null ? adjacencies.getAdjacency() : emptyList();
 787         List<Adjacency> value = new ArrayList<>();
 788         for (Adjacency nextHop : nextHops) {
 789             String rd = primaryRd;
 790             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 791             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 792                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 793                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 794                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 795                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 796                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 797                 continue;
 798             }
 799             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 800                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 801                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 802                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 803                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 804                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 805
 806                 Prefixes prefixes = intfnetworkUuid != null
 807                     ? VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, intfnetworkUuid ,networkType,
 808                             segmentationId, prefixCue) :
 809                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, prefixCue);
 810                 writeOperTxn.merge(VpnUtil.getPrefixToInterfaceIdentifier(
 811                         vpnUtil.getVpnId(vpnName), prefix), prefixes, true);
 812                 final Uuid subnetId = nextHop.getSubnetId();
 813
 814                 String gatewayIp = nextHop.getSubnetGatewayIp();
 815                 if (gatewayIp == null) {
 816                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 817                     if (gatewayIpOptional.isPresent()) {
 818                         gatewayIp = gatewayIpOptional.get();
 819                     }
 820                 }
 821
 822                 if (gatewayIp != null) {
 823                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 824                     if (gwMac.isPresent()) {
 825                         // A valid mac-address is available for this subnet-gateway-ip
 826                         // Use this for programming ARP_RESPONDER table here.  And save this
 827                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 828                         // to populate L3_GW_MAC_TABLE there.
 829                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 830                                 gatewayIp, gwMac.get());
 831                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 832                     } else {
 833                         // A valid mac-address is not available for this subnet-gateway-ip
 834                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 835                         // Save this connected-mac-address as gateway-mac-address for the
 836                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 837                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 838                         if (gwMac.isPresent()) {
 839                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 840                                     NwConstants.ADD_FLOW, gwMac.get());
 841                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 842                                     gatewayIp, gwMac.get());
 843                         } else {
 844                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 845                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 846                                 + "gwIp {}",
 847                                 subnetId, interfaceName, vpnName, gatewayIp);
 848                         }
 849                     }
 850                 } else {
 851                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 852                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 853                         subnetId, interfaceName, vpnName);
 854                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 855                 }
 856                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 857                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 858             } else {
 859                 //Extra route adjacency
 860                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 861                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 862                 // FIXME: separate this out somehow?
 863                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
 864                 lock.lock();
 865                 try {
 866                     java.util.Optional<String> rdToAllocate = vpnUtil
 867                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 868                                     nextHop.getNextHopIpList().get(0), dpnId);
 869                     if (rdToAllocate.isPresent()) {
 870                         rd = rdToAllocate.get();
 871                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 872                             rd, prefix);
 873                     } else {
 874                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 875                         continue;
 876                     }
 877                 } finally {
 878                     lock.unlock();
 879                 }
 880                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 881                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 882                         interfaceName, dpnId);
 883             }
 884             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 885             // can be different from the gateway-mac-address within the VRFEntry as the
 886             // gateway-mac-address is a superset.
 887             RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 888             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 889                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 890                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 891             Adjacency operationalAdjacency = null;
 892             try {
 893                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 894             } catch (NullPointerException e) {
 895                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 896                     input, e.getMessage());
 897                 return;
 898             }
 899             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 900                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 901                     vpnName, l3vni, origin, interfaceName, operationalAdjacency, encapType, prefixListForRefreshFib,
 902                     writeConfigTxn);
 903             }
 904             value.add(operationalAdjacency);
 905         }
 906
 907         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 908         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 909                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 910
 911         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni.longValue()).setPrimaryRd(primaryRd)
 912                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 913                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 914
 915         for (Adjacency nextHop : aug.getAdjacency()) {
 916             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 917             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 918                 RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 919                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 920                 registeredPopulator.populateFib(input, writeConfigTxn);
 921             }
 922         }
 923     }
 924
 925     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, Uint64 dpnId, AdjacenciesOp aug,
 926                                               long lportTag, String gwMac,
 927                                               TypedWriteTransaction<Operational> writeOperTxn) {
 928         VpnInterfaceOpDataEntry opInterface =
 929               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac);
 930         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 931             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 932         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
 933         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 934                 interfaceName, dpnId, vpnName);
 935     }
 936
 937     // TODO Clean up the exception handling
 938     @SuppressWarnings("checkstyle:IllegalCatch")
 939     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 940                                            StateTunnelList stateTunnelList,
 941                                            TypedWriteTransaction<Configuration> writeConfigTxn,
 942                                            TypedWriteTransaction<Operational> writeOperTxn) {
 943
 944         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
 945         Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern();
 946         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 947         List<Adjacency> adjList =
 948             adjacencies != null && adjacencies.getAdjacency() != null ? adjacencies.getAdjacency() : emptyList();
 949         if (adjList.isEmpty()) {
 950             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 951                     vpnInterface, srcDpnId);
 952             return;
 953         }
 954         String prefix = null;
 955         List<Adjacency> value = new ArrayList<>();
 956         boolean isNextHopAddReqd = false;
 957         String vpnName = vpnInterface.getVpnInstanceName();
 958         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 959         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 960         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 961                 vpnInterface.getName(), vpnInterface.getDpnId(),
 962                 vpnInterface.getVpnInstanceName(), adjList);
 963         for (Adjacency adj : adjList) {
 964             String rd = adj.getVrfId();
 965             rd = rd != null ? rd : vpnName;
 966             prefix = adj.getIpAddress();
 967             Uint32 label = adj.getLabel();
 968             List<String> nhList = Collections.singletonList(srcTepIp);
 969             List<String> nextHopList = adj.getNextHopIpList();
 970             // If TEP is added , update the nexthop of primary adjacency.
 971             // Secondary adj nexthop is already pointing to primary adj IP address.
 972             if (nextHopList == null || nextHopList.isEmpty()) {
 973                 isNextHopAddReqd = true;
 974             }
 975
 976             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 977                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 978             } else {
 979                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 980                 if (!vrfEntryOptional.isPresent()) {
 981                     continue;
 982                 }
 983                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 984                 if (!nhList.contains(srcTepIp)) {
 985                     nhList.add(srcTepIp);
 986                     isNextHopAddReqd = true;
 987                 }
 988                 value.add(adj);
 989             }
 990
 991             if (isNextHopAddReqd) {
 992                 updateLabelMapper(label, nhList);
 993                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 994                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 995                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 996                 // Update the VRF entry with nextHop
 997                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 998                         label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
 999
1000                 //Get the list of VPN's importing this route(prefix) .
1001                 // Then update the VRF entry with nhList
1002                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1003                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1004                     String vpnRd = vpn.getVrfId();
1005                     if (vpnRd != null) {
1006                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1007                             srcTepIp, label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1008                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
1009                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
1010                             vpnInterface.getName(), srcDpnId, vpnName,
1011                             vpn.getVpnInstanceName(), vpnRd);
1012                     }
1013                 }
1014                 // Advertise the prefix to BGP only for external vpn
1015                 // since there is a nexthop change.
1016                 try {
1017                     if (!rd.equalsIgnoreCase(vpnName)) {
1018                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
1019                                 VrfEntry.EncapType.Mplsgre, label, Uint32.ZERO /*evi*/, Uint32.ZERO /*l2vni*/,
1020                                 null /*gatewayMacAddress*/);
1021                     }
1022                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
1023                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
1024                             srcDpnId, vpnName);
1025                 } catch (Exception ex) {
1026                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
1027                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1028                             vpnInterface.getName(), srcDpnId, vpnName, ex);
1029                 }
1030             }
1031         }
1032         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1033         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1034                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1035                 .addAugmentation(AdjacenciesOp.class, aug).build();
1036         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1037                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1038         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1039         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
1040                 vpnInterface.getName(), srcDpnId, vpnName);
1041
1042     }
1043
1044     // TODO Clean up the exception handling
1045     @SuppressWarnings("checkstyle:IllegalCatch")
1046     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1047                                               StateTunnelList stateTunnelList,
1048                                               TypedWriteTransaction<Configuration> writeConfigTxn,
1049                                               TypedWriteTransaction<Operational> writeOperTxn) {
1050
1051         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1052         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1053         String prefix = null;
1054         boolean isNextHopRemoveReqd = false;
1055         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
1056         Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern();
1057         String vpnName = vpnInterface.getVpnInstanceName();
1058         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1059         String primaryRd = vpnUtil.getVpnRd(vpnName);
1060         if (adjList != null) {
1061             List<Adjacency> value = new ArrayList<>();
1062             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1063                     vpnInterface.getName(), vpnInterface.getDpnId(),
1064                     vpnInterface.getVpnInstanceName(), adjList);
1065             for (Adjacency adj : adjList) {
1066                 List<String> nhList = new ArrayList<>();
1067                 String rd = adj.getVrfId();
1068                 rd = rd != null ? rd : vpnName;
1069                 prefix = adj.getIpAddress();
1070                 List<String> nextHopList = adj.getNextHopIpList();
1071                 Uint32 label = adj.getLabel();
1072                 if (nextHopList != null && !nextHopList.isEmpty()) {
1073                     isNextHopRemoveReqd = true;
1074                 }
1075                 // If TEP is deleted , remove the nexthop from primary adjacency.
1076                 // Secondary adj nexthop will continue to point to primary adj IP address.
1077                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1078                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1079                 } else {
1080                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1081                     if (!vrfEntryOptional.isPresent()) {
1082                         continue;
1083                     }
1084                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1085                     if (nhList.contains(srcTepIp)) {
1086                         nhList.remove(srcTepIp);
1087                         isNextHopRemoveReqd = true;
1088                     }
1089                     value.add(adj);
1090                 }
1091
1092                 if (isNextHopRemoveReqd) {
1093                     updateLabelMapper(label, nhList);
1094                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1095                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1096                             prefix, nhList, vpnName,
1097                             vpnId, rd, vpnInterface.getName());
1098                     // Update the VRF entry with removed nextHop
1099                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1100                             label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1101
1102                     //Get the list of VPN's importing this route(prefix) .
1103                     // Then update the VRF entry with nhList
1104                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1105                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1106                         String vpnRd = vpn.getVrfId();
1107                         if (vpnRd != null) {
1108                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1109                                 srcTepIp, label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1110                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1111                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1112                                     nhList, label, vpnInterface.getName(), srcDpnId,
1113                                     vpnName,
1114                                     vpn.getVpnInstanceName(), vpnRd);
1115                         }
1116                     }
1117
1118                     // Withdraw prefix from BGP only for external vpn.
1119                     try {
1120                         if (!rd.equalsIgnoreCase(vpnName)) {
1121                             bgpManager.withdrawPrefix(rd, prefix);
1122                         }
1123                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1124                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1125                                 vpnInterface.getName(), srcDpnId,
1126                                 vpnName);
1127                     } catch (Exception ex) {
1128                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1129                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1130                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1131                     }
1132                 }
1133             }
1134             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1135             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1136                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1137                     .addAugmentation(AdjacenciesOp.class, aug).build();
1138             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1139                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1140             writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1141             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1142                          vpnInterface.getName(), srcDpnId, vpnName);
1143         }
1144     }
1145
1146     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1147         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1148
1149         String vpnRd = vpnUtil.getVpnRd(vpnName);
1150         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1151         if (vpnInstanceOpDataEntry == null) {
1152             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1153                     + " to check for vpns exporting the routes", vpnName);
1154             return vpnsToExportRoute;
1155         }
1156
1157         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1158             if (input.getVpnInstanceName() == null) {
1159                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1160                         input.getVrfId());
1161                 return false;
1162             }
1163             return !input.getVpnInstanceName().equals(vpnName);
1164         };
1165
1166         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1167             Iterable<String> commonRTs =
1168                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1169                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1170             return Iterators.size(commonRTs.iterator()) > 0;
1171         };
1172
1173         vpnsToExportRoute =
1174                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1175                         Collectors.toList());
1176         return vpnsToExportRoute;
1177     }
1178
1179     // TODO Clean up the exception handling
1180     @SuppressWarnings("checkstyle:IllegalCatch")
1181     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1182         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1183         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1184             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1185             if (vrfEntries != null) {
1186                 ListenableFutures.addErrorLogging(
1187                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1188                         for (VrfEntry vrfEntry : vrfEntries) {
1189                             try {
1190                                 if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1191                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1192                                     LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1193                                             + " is not a controller managed non intervpn link route. Ignoring.",
1194                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1195                                     continue;
1196                                 }
1197                                 String prefix = vrfEntry.getDestPrefix();
1198                                 String gwMac = vrfEntry.getGatewayMacAddress();
1199                                 vrfEntry.nonnullRoutePaths().forEach(routePath -> {
1200                                     String nh = routePath.getNexthopAddress();
1201                                     Uint32 label = routePath.getLabel();
1202                                     if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1203                                         vrfEntry.getOrigin()))) {
1204                                         LOG.info(
1205                                             "handleVpnsExportingRoutesImporting: Importing fib entry rd {}"
1206                                                 + " prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1207                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1208                                         fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1209                                             Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1210                                                 Uint32.ZERO /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1211                                             confTx);
1212                                     } else {
1213                                         LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry"
1214                                                 + " rd {} prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1215                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1216                                         SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1217                                         importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1218                                             confTx);
1219                                     }
1220                                 });
1221                             } catch (RuntimeException e) {
1222                                 LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1223                                         + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1224                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1225                             }
1226                         }
1227                     }), LOG, "Error handing VPN exporting routes");
1228             } else {
1229                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1230                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1231             }
1232         }
1233     }
1234
1235     @Override
1236     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1237         LOG.trace("Received VpnInterface remove event: vpnInterface={}", vpnInterface);
1238         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1239         final String interfaceName = key.getName();
1240         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
1241             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1242             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1243         }
1244     }
1245
1246     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1247                                 final VpnInterface vpnInterface, final String vpnName,
1248                                 final String interfaceName) {
1249         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1250             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1251                 ListenableFuture<Void> future =
1252                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1253                         deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1254                         LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1255                     });
1256                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1257                         vpnInterface.getName(), vpnName);
1258                 return Collections.singletonList(future);
1259             }, DJC_MAX_RETRIES);
1260         } else {
1261             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1262             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1263         }
1264     }
1265
1266     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1267     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1268                                            final VpnInterface vpnInterface, final String vpnName,
1269                                            final String interfaceName, final Interface interfaceState) {
1270         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1271                 vpnName, vpnInterface.getDpnId());
1272         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1273         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1274             () -> {
1275                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1276                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1277                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1278                         writeOperTxn -> futures.add(
1279                             txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeInvTxn -> {
1280                                 LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1281                                         interfaceName, vpnName);
1282                                 Uint64 dpId;
1283                                 int ifIndex;
1284                                 String gwMacAddress;
1285                                 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1286                                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1287                                 Optional<VpnInterfaceOpDataEntry> optVpnInterface;
1288                                 try {
1289                                     optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1290                                             LogicalDatastoreType.OPERATIONAL, interfaceId);
1291                                 } catch (ReadFailedException e) {
1292                                     LOG.error("remove: Failed to read data store for interface {} vpn {}",
1293                                             interfaceName, vpnName);
1294                                     return;
1295                                 }
1296                                 if (interfaceState != null) {
1297                                     try {
1298                                         dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1299                                     } catch (NumberFormatException | IllegalStateException e) {
1300                                         LOG.error("remove: Unable to retrieve dpnId from interface operational"
1301                                                         + " data store for interface {} on dpn {} for vpn {} Fetching"
1302                                                         + " from vpn interface op data store. ", interfaceName,
1303                                                 vpnInterface.getDpnId(), vpnName, e);
1304                                         dpId = Uint64.ZERO;
1305                                     }
1306                                     ifIndex = interfaceState.getIfIndex();
1307                                     gwMacAddress = interfaceState.getPhysAddress().getValue();
1308                                 } else {
1309                                     LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1310                                             + " from vpn interface op.", interfaceName);
1311                                     if (optVpnInterface.isPresent()) {
1312                                         VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1313                                         dpId = vpnOpInterface.getDpnId();
1314                                         ifIndex = vpnOpInterface.getLportTag().intValue();
1315                                         gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1316                                     } else {
1317                                         LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1318                                                 + " as interfaceState and vpn interface op is not"
1319                                                 + " available", interfaceName, vpnName);
1320                                         return;
1321                                     }
1322                                 }
1323                                 processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1324                                         optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1325                                         writeConfigTxn, writeOperTxn, writeInvTxn);
1326                                 LOG.info(
1327                                         "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1328                                                 + "successfully",
1329                                         interfaceName, vpnInterface.getDpnId(), vpnName);
1330                             })))));
1331                 futures.add(configFuture);
1332                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(
1333                         interfaceName, false, "Config"), MoreExecutors.directExecutor());
1334                 return futures;
1335             }, DJC_MAX_RETRIES);
1336     }
1337
1338     protected void processVpnInterfaceDown(Uint64 dpId,
1339                                            String interfaceName,
1340                                            int lportTag,
1341                                            String gwMac,
1342                                            VpnInterfaceOpDataEntry vpnOpInterface,
1343                                            boolean isInterfaceStateDown,
1344                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1345                                            TypedWriteTransaction<Operational> writeOperTxn,
1346                                            TypedReadWriteTransaction<Configuration> writeInvTxn)
1347             throws ExecutionException, InterruptedException {
1348         if (vpnOpInterface == null) {
1349             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1350                     + " as it is not available in operational data store", interfaceName, dpId);
1351             return;
1352         }
1353         final String vpnName = vpnOpInterface.getVpnInstanceName();
1354         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1355                                                     interfaceName, vpnName);
1356         if (!isInterfaceStateDown) {
1357             final Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1358             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName,  null);
1359             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1360             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1361                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1362             if (interfaceManager.isExternalInterface(interfaceName)) {
1363                 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1364                     NwConstants.DEL_FLOW);
1365             }
1366             if (!isBgpVpnInternetVpn) {
1367                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1368             }
1369             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1370                     + " successful", interfaceName, dpId, vpnName);
1371         } else {
1372             // Interface is retained in the DPN, but its Link Down.
1373             // Only withdraw the prefixes for this interface from BGP
1374             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1375         }
1376     }
1377
1378     private void removeAdjacenciesFromVpn(final Uint64 dpnId, final int lportTag, final String interfaceName,
1379                                           final String vpnName, final Uint32 vpnId, String gwMac,
1380                                           TypedWriteTransaction<Configuration> writeConfigTxn,
1381                                           TypedWriteTransaction<Operational> writeOperTxn,
1382                                           TypedReadWriteTransaction<Configuration> writeInvTxn)
1383             throws ExecutionException, InterruptedException {
1384         //Read NextHops
1385         try {
1386             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1387                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1388             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1389             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1390                     LogicalDatastoreType.OPERATIONAL, path);
1391             boolean isLearntIP = false;
1392             String primaryRd = vpnUtil.getVpnRd(vpnName);
1393             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1394                     interfaceName, dpnId, vpnName, primaryRd);
1395             if (adjacencies.isPresent() && adjacencies.get().getAdjacency() != null
1396                     && !adjacencies.get().getAdjacency().isEmpty()) {
1397                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1398                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1399                         interfaceName, dpnId, vpnName, nextHops);
1400                 for (Adjacency nextHop : nextHops) {
1401                     if (nextHop.isPhysNetworkFunc()) {
1402                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1403                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1404                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1405                                 null/*writeCfgTxn*/);
1406                     } else {
1407                         String rd = nextHop.getVrfId();
1408                         List<String> nhList;
1409                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1410                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1411                             isLearntIP = nextHop.getAdjacencyType() == AdjacencyType.LearntIp ? true : false;
1412                         } else {
1413                             // This is a primary adjacency
1414                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1415                                     : emptyList();
1416                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1417                                     interfaceName, writeInvTxn);
1418                         }
1419                         if (!nhList.isEmpty()) {
1420                             if (Objects.equals(primaryRd, vpnName)) {
1421                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1422                                 //remove from FIB directly
1423                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1424                                         interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1425                             } else {
1426                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1427                                         interfaceName, writeConfigTxn, writeOperTxn);
1428                             }
1429                         } else {
1430                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1431                                             + " interface {}", nextHop.getIpAddress(), rd,
1432                                     nextHop.getAdjacencyType().toString(), interfaceName);
1433                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1434                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1435                         }
1436                     }
1437                     String ip = nextHop.getIpAddress().split("/")[0];
1438                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1439                     if (vpnVipToPort != null && vpnVipToPort.getPortName().equals(interfaceName)) {
1440                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1441                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1442                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1443                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1444                     }
1445                     // Remove the MIP-IP from VpnPortIpToPort.
1446                     if (isLearntIP) {
1447                         VpnPortipToPort persistedIp = vpnUtil.getVpnPortipToPort(vpnName, ip);
1448                         if (persistedIp != null && persistedIp.isLearntIp()
1449                                 && persistedIp.getPortName().equals(interfaceName)) {
1450                             VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1451                             LOG.info(
1452                                     "removeAdjacenciesFromVpn: Learnt-IP: {} interface {} of vpn {} removed "
1453                                             + "from VpnPortipToPort",
1454                                     persistedIp.getPortFixedip(), persistedIp.getPortName(), vpnName);
1455                         }
1456                     }
1457                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1458                     if (vpnPortipToPort != null) {
1459                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1460                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1461                                  + "Interface {} ip {} on dpn {} for vpn {}",
1462                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1463                     }
1464                 }
1465             } else {
1466                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1467                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1468                         + " Removing it.", interfaceName, vpnName, dpnId);
1469                 writeOperTxn.delete(identifier);
1470             }
1471         } catch (ReadFailedException e) {
1472             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1473                     interfaceName, dpnId, vpnName);
1474         }
1475     }
1476
1477     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1478                                                             String interfaceName, Uint64 dpnId,
1479                                                             TypedWriteTransaction<Configuration> writeConfigTxn,
1480                                                             TypedWriteTransaction<Operational> writeOperTx) {
1481         return (nh) -> {
1482             String primaryRd = vpnUtil.getVpnRd(vpnName);
1483             String prefix = nextHop.getIpAddress();
1484             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1485             LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1486                     nextHop, vpnName, interfaceName, dpnId);
1487             // FIXME: separate this out somehow?
1488             final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnNamePrefixKey);
1489             lock.lock();
1490             try {
1491                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1492                         prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1493                     //If extra-route is present behind at least one VM, then do not remove or update
1494                     //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1495                     //prefixtointerface DS
1496                     return;
1497                 }
1498                 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1499                         writeConfigTxn);
1500             } finally {
1501                 lock.unlock();
1502             }
1503             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1504                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1505                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1506         };
1507     }
1508
1509     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1510                                            Uint64 dpnId, String rd, String interfaceName,
1511                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1512                                            TypedWriteTransaction<Operational> writeOperTx) {
1513         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1514                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1515         nhList.forEach((nh) -> {
1516             //IRT: remove routes from other vpns importing it
1517             vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1518                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1519             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1520                 String vpnRd = vpn.getVrfId();
1521                 if (vpnRd != null) {
1522                     fibManager.removeOrUpdateFibEntry(vpnRd,
1523                             nextHop.getIpAddress(), nh, writeConfigTxn);
1524                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1525                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1526                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1527                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1528                 }
1529             }
1530         });
1531     }
1532
1533     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, Uint32 vpnId, Uint64 dpnId,
1534                                                  int lportTag, String gwMac, String interfaceName,
1535                                                  TypedReadWriteTransaction<Configuration> writeInvTxn)
1536             throws ExecutionException, InterruptedException {
1537         final Uuid subnetId = nextHop.getSubnetId();
1538         if (nextHop.getSubnetGatewayMacAddress() == null) {
1539             // A valid mac-address was not available for this subnet-gateway-ip
1540             // So a connected-mac-address was used for this subnet and we need
1541             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1542             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1543         }
1544         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1545                 subnetId);
1546     }
1547
1548     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, Uint64 dpnId,
1549                                                   String interfaceName) {
1550         // This is either an extra-route (or) a learned IP via subnet-route
1551         List<String> nhList = null;
1552         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1553         if (nextHopIp == null || nextHopIp.isEmpty()) {
1554             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1555                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1556                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1557                     vpnName);
1558             nhList = emptyList();
1559         } else {
1560             nhList = Collections.singletonList(nextHopIp);
1561         }
1562         return nhList;
1563     }
1564
1565     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1566         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1567         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1568         // else use connected interface
1569         if (gwPort != null && gwPort.isSubnetIp()) {
1570             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1571                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1572             return Optional.of(gwPort.getMacAddress());
1573         }
1574         return Optional.absent();
1575     }
1576
1577     @Override
1578     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1579         final VpnInterface update) {
1580         LOG.trace("Received VpnInterface update event: original={}, update={}", original, update);
1581         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}", update.getName(),
1582                 update.getDpnId(), original.getVpnInstanceNames(), update.getVpnInstanceNames());
1583         if (original.equals(update)) {
1584             LOG.info("update: original {} update {} are same. No update required.", original, update);
1585             return;
1586         }
1587         final String vpnInterfaceName = update.getName();
1588         final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1589         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1590         //handles switching between <internal VPN - external VPN>
1591         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1592             List<ListenableFuture<Void>> futures = new ArrayList<>();
1593             if (handleVpnInstanceUpdateForVpnInterface(identifier, original, update, futures)) {
1594                 LOG.info("update: handled Instance update for VPNInterface {} on dpn {} from oldVpn(s) {} "
1595                                 + "to newVpn(s) {}",
1596                         original.getName(), dpnId,
1597                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1598                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1599                 return emptyList();
1600             }
1601             updateVpnInstanceAdjChange(original, update, vpnInterfaceName, futures);
1602             return futures;
1603         });
1604     }
1605
1606     private boolean handleVpnInstanceUpdateForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1607                                                            VpnInterface original, VpnInterface update,
1608                                                            List<ListenableFuture<Void>> futures) {
1609         boolean isVpnInstanceUpdate = false;
1610         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1611         final String interfaceName = key.getName();
1612         List<String> oldVpnList = VpnUtil.getVpnListForVpnInterface(original);
1613         List<String> oldVpnListCopy = new ArrayList<>();
1614         oldVpnListCopy.addAll(oldVpnList);
1615         List<String> newVpnList = VpnUtil.getVpnListForVpnInterface(update);
1616         List<String> newVpnListCopy = new ArrayList<>();
1617         newVpnListCopy.addAll(newVpnList);
1618
1619         oldVpnList.removeAll(newVpnList);
1620         newVpnList.removeAll(oldVpnListCopy);
1621         //This block will execute only on if there is a change in the VPN Instance.
1622         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1623             /*
1624              * Internet BGP-VPN Instance update with single router:
1625              * ====================================================
1626              * In this case single VPN Interface will be part of maximum 2 VPN Instance only.
1627              *     1st VPN Instance : router VPN or external BGP-VPN.
1628              *     2nd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1629              *
1630              * VPN Instance UPDATE:
1631              * oldVpnList = 0 and newVpnList = 1 (Internet BGP-VPN)
1632              * oldVpnList = 1 and newVpnList = 0 (Internet BGP-VPN)
1633              *
1634              * External BGP-VPN Instance update with single router:
1635              * ====================================================
1636              * In this case single VPN interface will be part of maximum 1 VPN Instance only.
1637              *
1638              * Updated VPN Instance will be always either internal router VPN to
1639              * external BGP-VPN or external BGP-VPN to internal router VPN swap.
1640              *
1641              * VPN Instance UPDATE:
1642              * oldVpnList = 1 and newVpnList = 1 (router VPN to Ext-BGPVPN)
1643              * oldVpnList = 1 and newVpnList = 1 (Ext-BGPVPN to router VPN)
1644              *
1645              * Dual Router VPN Instance Update:
1646              * ================================
1647              * In this case single VPN interface will be part of maximum 3 VPN Instance only.
1648              *
1649              * 1st VPN Instance : router VPN or external BGP-VPN-1.
1650              * 2nd VPN Instance : router VPN or external BGP-VPN-2.
1651              * 3rd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1652              *
1653              * Dual Router --> Associated with common external BGP-VPN Instance.
1654              * 1st router and 2nd router are getting associated with single External BGP-VPN
1655              * 1) add 1st router to external bgpvpn --> oldVpnList=1, newVpnList=1;
1656              * 2) add 2nd router to the same external bgpvpn --> oldVpnList=1, newVpnList=0
1657              * In this case, we need to call removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1658              *
1659              *
1660              */
1661             isVpnInstanceUpdate = true;
1662             if (VpnUtil.isDualRouterVpnUpdate(oldVpnListCopy, newVpnListCopy)) {
1663                 if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1664                         && oldVpnList.size() == 1 && newVpnList.isEmpty()) {
1665                     //Identify the external BGP-VPN Instance and pass that value as newVpnList
1666                     List<String> externalBgpVpnList = new ArrayList<>();
1667                     for (String newVpnName : newVpnListCopy) {
1668                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1669                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1670                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1671                                 .BgpvpnType.BGPVPNExternal) {
1672                             externalBgpVpnList.add(newVpnName);
1673                             break;
1674                         }
1675                     }
1676                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1677                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList,
1678                             externalBgpVpnList, oldVpnListCopy, futures);
1679
1680                 } else if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1681                         && oldVpnList.isEmpty() && newVpnList.size() == 1) {
1682                     //Identify the router VPN Instance and pass that value as oldVpnList
1683                     List<String> routerVpnList = new ArrayList<>();
1684                     for (String newVpnName : newVpnListCopy) {
1685                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1686                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1687                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1688                                 .BgpvpnType.VPN) {
1689                             routerVpnList.add(newVpnName);
1690                             break;
1691                         }
1692                     }
1693                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1694                     updateVpnInstanceChange(identifier, interfaceName, original, update, routerVpnList,
1695                             newVpnList, oldVpnListCopy, futures);
1696
1697                 } else {
1698                     //Handle remaining use cases.
1699                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1700                             oldVpnListCopy, futures);
1701                 }
1702             } else {
1703                 updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1704                         oldVpnListCopy, futures);
1705             }
1706         }
1707         return isVpnInstanceUpdate;
1708     }
1709
1710     private void updateVpnInstanceChange(InstanceIdentifier<VpnInterface> identifier, String interfaceName,
1711                                          VpnInterface original, VpnInterface update, List<String> oldVpnList,
1712                                          List<String> newVpnList, List<String> oldVpnListCopy,
1713                                          List<ListenableFuture<Void>> futures) {
1714         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1715         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency() != null
1716                 ? origAdjs.getAdjacency() : new ArrayList<>();
1717         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1718         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency() != null
1719                 ? updateAdjs.getAdjacency() : new ArrayList<>();
1720
1721         boolean isOldVpnRemoveCallExecuted = false;
1722         for (String oldVpnName : oldVpnList) {
1723             LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1724                     + "remove from vpnName {} ", interfaceName, oldVpnName);
1725             removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1726             LOG.info("updateVpnInstanceChange: Processed Remove for update on VPNInterface"
1727                             + " {} upon VPN update from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName,
1728                     newVpnList);
1729             isOldVpnRemoveCallExecuted = true;
1730         }
1731         //Wait for previous interface bindings to be removed
1732         if (isOldVpnRemoveCallExecuted && !newVpnList.isEmpty()) {
1733             try {
1734                 Thread.sleep(2000);
1735             } catch (InterruptedException e) {
1736                 LOG.error("updateVpnInstanceChange: InterruptedException caught for interface {}", interfaceName, e);
1737             }
1738         }
1739         for (String newVpnName : newVpnList) {
1740             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1741             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1742                 LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1743                         + "onto vpnName {} ", interfaceName, newVpnName);
1744                 addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1745                 LOG.info("updateVpnInstanceChange: Processed Add for update on VPNInterface {}"
1746                                 + "from oldVpn(s) {} to newVpn {} ",
1747                         interfaceName, oldVpnListCopy, newVpnName);
1748                 /* This block will execute only if V6 subnet is associated with internet BGP-VPN.
1749                  * Use Case:
1750                  *     In Dual stack network, first V4 subnet only attached to router and router is associated
1751                  *     with internet BGP-VPN(router-gw). At this point VPN interface is having only router vpn instance.
1752                  *     Later V6 subnet is added to router, at this point existing VPN interface will get updated
1753                  *     with Internet BGP-VPN instance(Note: Internet BGP-VPN Instance update in vpn interface
1754                  *     is applicable for only on V6 subnet is added to router). newVpnList = Contains only Internet
1755                  *     BGP-VPN Instance. So we required V6 primary adjacency info needs to be populated onto
1756                  *     router VPN as well as Internet BGP-VPN.
1757                  *
1758                  *     addVpnInterfaceCall() --> It will create V6 Adj onto Internet BGP-VPN only.
1759                  *     updateVpnInstanceAdjChange() --> This method call is needed for second primary V6 Adj
1760                  *                                       update in existing router VPN instance.
1761                  */
1762                 if (vpnUtil.isBgpVpnInternet(newVpnName)) {
1763                     LOG.info("updateVpnInstanceChange: VPN Interface {} with new Adjacency {} in existing "
1764                             + "VPN instance {}", interfaceName, newAdjs, original.getVpnInstanceNames());
1765                     updateVpnInstanceAdjChange(original, update, interfaceName, futures);
1766                 }
1767             } else {
1768                 LOG.info("updateVpnInstanceChange: failed to Add for update on VPNInterface {} from oldVpn(s) {} to "
1769                                 + "newVpn {} as the new vpn does not exist in oper DS or it is in PENDING_DELETE state",
1770                         interfaceName, oldVpnListCopy, newVpnName);
1771             }
1772         }
1773     }
1774
1775     // TODO Clean up the exception handling
1776     @SuppressWarnings("checkstyle:IllegalCatch")
1777     private List<ListenableFuture<Void>> updateVpnInstanceAdjChange(VpnInterface original, VpnInterface update,
1778                                                                     String vpnInterfaceName,
1779                                                                     List<ListenableFuture<Void>> futures) {
1780         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1781         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1782                 != null ? origAdjs.getAdjacency() : new ArrayList<>();
1783         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1784         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1785                 != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1786
1787         final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1788         for (VpnInstanceNames vpnInterfaceVpnInstance : update.nonnullVpnInstanceNames()) {
1789             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1790             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1791             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1792             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1793             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1794                 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1795                 //set of prefix used as entry in prefix-to-interface datastore
1796                 // is prerequisite for refresh Fib to avoid race condition leading to missing remote next hop
1797                 // in bucket actions on bgp-vpn delete
1798                 Set<String> prefixListForRefreshFib = new HashSet<>();
1799                 ListenableFuture<Void> configTxFuture = txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
1800                     confTx -> futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL,
1801                         operTx -> {
1802                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1803                                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1804                             LOG.info("VPN Interface update event-intfName {} onto vpnName {} running config-driven",
1805                                     update.getName(), newVpnName);
1806                             //handle both addition and removal of adjacencies
1807                             // currently, new adjacency may be an extra route
1808                             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1809                             if (!oldAdjs.equals(newAdjs)) {
1810                                 for (Adjacency adj : copyNewAdjs) {
1811                                     if (copyOldAdjs.contains(adj)) {
1812                                         copyOldAdjs.remove(adj);
1813                                     } else {
1814                                         // add new adjacency
1815                                         if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1816                                             try {
1817                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1818                                                     dpnId, operTx, confTx, confTx, prefixListForRefreshFib);
1819                                             } catch (RuntimeException e) {
1820                                                 LOG.error("Failed to add adjacency {} to vpn interface {} with"
1821                                                         + " dpnId {}", adj, vpnInterfaceName, dpnId, e);
1822                                             }
1823                                         }
1824                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} "
1825                                             + " added to vpn interface {} on vpn {} dpnId {}",
1826                                             adj.getIpAddress(), adj.getNextHopIpList(), adj.getLabel(),
1827                                             adj.getSubnetId(), update.getName(), newVpnName, dpnId);
1828                                     }
1829                                 }
1830                                 for (Adjacency adj : copyOldAdjs) {
1831                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1832                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1833                                             && !adj.isPhysNetworkFunc()) {
1834                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, operTx,
1835                                                 confTx);
1836                                             //remove FIB entry
1837                                             String vpnRd = vpnUtil.getVpnRd(newVpnName);
1838                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1839                                                 + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1840                                             //remove BGP entry
1841                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1842                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1843                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1844                                             }
1845                                         } else {
1846                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1847                                                 operTx, confTx);
1848                                         }
1849                                     }
1850                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1851                                         + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1852                                         adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1853                                 }
1854                             }
1855                         })));
1856                 Futures.addCallback(configTxFuture, new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
1857                     MoreExecutors.directExecutor());
1858                 futures.add(configTxFuture);
1859                 for (ListenableFuture<Void> future : futures) {
1860                     ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1861                             update.getName(), update.getVpnInstanceNames());
1862                 }
1863             } else {
1864                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1865                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1866             }
1867         }
1868         return futures;
1869     }
1870
1871     private void updateLabelMapper(Uint32 label, List<String> nextHopIpList) {
1872         final String labelStr = Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!")
1873                 .toString();
1874         // FIXME: separate this out somehow?
1875         final ReentrantLock lock = JvmGlobalLocks.getLockForString(labelStr);
1876         lock.lock();
1877         try {
1878             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1879                     .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1880             Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1881                 LogicalDatastoreType.OPERATIONAL, lriIid);
1882             if (opResult.isPresent()) {
1883                 LabelRouteInfo labelRouteInfo =
1884                         new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1885                 SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1886                     labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1887             }
1888             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1889                     nextHopIpList);
1890         } catch (ReadFailedException e) {
1891             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1892                     nextHopIpList);
1893         } catch (TransactionCommitFailedException e) {
1894             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1895                     nextHopIpList);
1896         } finally {
1897             lock.unlock();
1898         }
1899     }
1900
1901     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, Uint32 label,
1902         SubnetRoute route, String parentVpnRd, TypedWriteTransaction<Configuration> writeConfigTxn) {
1903
1904         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1905         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1906                 .addAugmentation(SubnetRoute.class, route).build();
1907         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1908         InstanceIdentifierBuilder<VrfTables> idBuilder =
1909             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1910         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1911         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1912         if (writeConfigTxn != null) {
1913             writeConfigTxn.merge(vrfTableId, vrfTableNew, CREATE_MISSING_PARENTS);
1914         } else {
1915             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1916         }
1917         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1918                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1919     }
1920
1921     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1922                                            Adjacency adj, Uint64 dpnId,
1923                                            TypedWriteTransaction<Operational> writeOperTxn,
1924                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1925                                            TypedReadWriteTransaction<Configuration> writeInvTxn,
1926                                            Set<String> prefixListForRefreshFib)
1927             throws ExecutionException, InterruptedException {
1928         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1929         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1930         try {
1931             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1932                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1933             if (optVpnInterface.isPresent()) {
1934                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1935                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1936                 String vpnName = currVpnIntf.getVpnInstanceName();
1937                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1938                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1939                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1940                         LogicalDatastoreType.OPERATIONAL, adjPath);
1941                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1942                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1943                 Uint32 l3vni = vpnInstanceOpData.getL3vni() == null ? Uint32.ZERO : vpnInstanceOpData.getL3vni();
1944                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1945                 List<Adjacency> adjacencies = new ArrayList<>();
1946                 if (optAdjacencies.isPresent() && optAdjacencies.get().getAdjacency() != null) {
1947                     adjacencies.addAll(optAdjacencies.get().getAdjacency());
1948                 }
1949                 Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1950                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1951                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1952                 Adjacency operationalAdjacency = null;
1953                 //Handling dual stack neutron port primary adjacency
1954                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1955                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1956                             currVpnIntf.getName(), vpnName);
1957                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1958                             currVpnIntf.getName());
1959                     if (interfaceState != null) {
1960                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1961                             currVpnIntf.getName(), vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState,
1962                             prefixListForRefreshFib);
1963                     }
1964                 }
1965                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1966                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1967                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.LearntIp ? RouteOrigin.DYNAMIC
1968                             : RouteOrigin.STATIC;
1969                     String nh = adj.getNextHopIpList().get(0);
1970                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1971                     // FIXME: separate out to somehow?
1972                     final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
1973                     lock.lock();
1974                     try {
1975                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1976                                 vpnId, null, prefix, vpnName, nh, dpnId);
1977                         if (rdToAllocate.isPresent()) {
1978                             input.setRd(rdToAllocate.get());
1979                             operationalAdjacency = populator.createOperationalAdjacency(input);
1980                             int label = operationalAdjacency.getLabel().intValue();
1981                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1982                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
1983                                     currVpnIntf.getName(), operationalAdjacency, encapType,
1984                                     prefixListForRefreshFib, writeConfigTxn);
1985                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1986                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1987                                     vpnName, label, currVpnIntf.getName(), dpnId);
1988                         } else {
1989                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
1990                                     vpnName, prefix);
1991                             return;
1992                         }
1993                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1994                         // Keeping the MPLS check for now.
1995                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1996                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1997                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1998                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
1999                             vpnsToImportRoute.forEach(vpn -> {
2000                                 if (vpn.getVrfId() != null) {
2001                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(),
2002                                             vpnId, prefix,
2003                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
2004                                             .ifPresent(
2005                                                 rds -> vpnManager.addExtraRoute(
2006                                                         vpnUtil.getVpnName(vpn.getVpnId()), adj.getIpAddress(),
2007                                                         nh, rds, currVpnIntf.getVpnInstanceName(), l3vni,
2008                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(), opAdjacency,
2009                                                         encapType, prefixListForRefreshFib, writeConfigTxn));
2010                                 }
2011                             });
2012                         }
2013                     } finally {
2014                         lock.unlock();
2015                     }
2016                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
2017                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
2018                             currVpnIntf.getName(), vpnName);
2019
2020                     InstanceIdentifier<VpnInterface> vpnIfaceConfigidentifier = VpnUtil
2021                             .getVpnInterfaceIdentifier(currVpnIntf.getName());
2022                     Optional<VpnInterface> vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2023                             LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier);
2024                     Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix,
2025                             Prefixes.PrefixCue.PhysNetFunc);
2026                     if (vpnIntefaceConfig.isPresent()) {
2027                         pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix,
2028                                 vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(),
2029                                 vpnIntefaceConfig.get().getSegmentationId().toJava(), Prefixes.PrefixCue.PhysNetFunc);
2030                     }
2031
2032                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
2033
2034                     writeOperTxn.merge(
2035                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
2036                                     prefix), pnfPrefix, true);
2037
2038                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
2039                             adj.getIpAddress(), emptyList(), null /* EncapType */, Uint32.ZERO /* label */,
2040                             Uint32.ZERO /*l3vni*/, null /* gw-mac */, parentVpnRd,
2041                             RouteOrigin.LOCAL, writeConfigTxn);
2042
2043                     input.setRd(adj.getVrfId());
2044                 }
2045                 if (operationalAdjacency == null) {
2046                     operationalAdjacency = populator.createOperationalAdjacency(input);
2047                 }
2048                 adjacencies.add(operationalAdjacency);
2049                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
2050                 VpnInterfaceOpDataEntry newVpnIntf =
2051                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
2052                                 aug, dpnId, currVpnIntf.getLportTag().toJava(),
2053                                 currVpnIntf.getGatewayMacAddress());
2054
2055                 writeOperTxn.merge(identifier, newVpnIntf, CREATE_MISSING_PARENTS);
2056             }
2057         } catch (ReadFailedException e) {
2058             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
2059                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
2060         }
2061     }
2062
2063     @Nullable
2064     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
2065         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
2066         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
2067     }
2068
2069     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
2070                                             Uint64 dpnId, TypedWriteTransaction<Operational> writeOperTxn,
2071                                             TypedWriteTransaction<Configuration> writeConfigTxn) {
2072         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
2073         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
2074         try {
2075             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
2076                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
2077             if (optVpnInterface.isPresent()) {
2078                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
2079                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
2080                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2081                         LogicalDatastoreType.OPERATIONAL, path);
2082                 if (optAdjacencies.isPresent()) {
2083                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
2084
2085                     if (adjacencies != null && !adjacencies.isEmpty()) {
2086                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
2087                         for (Adjacency adjacency : adjacencies) {
2088                             if (Objects.equals(adjacency.getIpAddress(), adj.getIpAddress())) {
2089                                 String rd = adjacency.getVrfId();
2090                                 if (adj.getNextHopIpList() != null) {
2091                                     for (String nh : adj.getNextHopIpList()) {
2092                                         deleteExtraRouteFromCurrentAndImportingVpns(
2093                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
2094                                                 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
2095                                     }
2096                                 } else if (adj.isPhysNetworkFunc()) {
2097                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
2098                                             adj.getIpAddress(), adj.getSubnetId());
2099                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
2100                                             writeConfigTxn);
2101                                 }
2102                                 break;
2103                             }
2104
2105                         }
2106                     }
2107                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
2108                             dpnId, adj.getVrfId());
2109                 } else {
2110                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
2111                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
2112                 }
2113             }
2114         } catch (ReadFailedException e) {
2115             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
2116                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
2117         }
2118     }
2119
2120     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
2121                                     String rd, String intfName, TypedWriteTransaction<Configuration> writeConfigTxn,
2122                                     TypedWriteTransaction<Operational> writeOperTx) {
2123         LOG.info("removing extra-route {} for nexthop {} in VPN {} intfName {} rd {}",
2124                 destination, nextHop, vpnName, intfName, rd);
2125         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
2126         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
2127         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
2128             String vpnRd = vpn.getVrfId();
2129             if (vpnRd != null) {
2130                 LOG.info("deleting extra-route {} for nexthop {} in VPN {} intfName {} vpnRd {}",
2131                         destination, nextHop, vpnName, intfName, vpnRd);
2132                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
2133                         writeOperTx);
2134             }
2135         }
2136     }
2137
2138     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, Uint64 dpnId) {
2139         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2140             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
2141             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
2142     }
2143
2144     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
2145         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2146             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
2147     }
2148
2149     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2150                                                 TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2151         if (vpnInterface == null) {
2152             return;
2153         }
2154         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
2155         if (adjs == null) {
2156             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2157                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2158             return;
2159         }
2160         for (Adjacency adj : adjs) {
2161             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2162                 String primaryInterfaceIp = adj.getIpAddress();
2163                 String macAddress = adj.getMacAddress();
2164                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2165
2166                 Uint32 label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
2167                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2168
2169                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2170                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2171                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
2172                         routerInt, label, writeConfigTxn);
2173                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2174                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2175                         macAddress);
2176             } else {
2177                 LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2178                                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}",
2179                         interfaceName, primaryRd, vpnName);
2180             }
2181         }
2182     }
2183
2184     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
2185             TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2186         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
2187         String rd = vpnUtil.getVpnRd(vpnName);
2188         if (adjs != null) {
2189             List<Adjacency> adjsList = adjs.nonnullAdjacency();
2190             for (Adjacency adj : adjsList) {
2191                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2192                     String primaryInterfaceIp = adj.getIpAddress();
2193                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2194                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
2195                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2196                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2197                 }
2198             }
2199         } else {
2200             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2201                     vpnInterface.getName(), rd);
2202         }
2203     }
2204
2205     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2206         final VpnInterfaceKey key = intefaceData.identifier.firstKeyOf(VpnInterface.class);
2207         final String interfaceName = key.getName();
2208         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2209                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2210         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2211                   intefaceData.identifier, vpnName);
2212     }
2213
2214     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2215                                               VpnInterface vpnInterface, String vpnName) {
2216         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2217                .get(vpnName);
2218         if (vpnInterfaces == null) {
2219             vpnInterfaces = new ConcurrentLinkedQueue<>();
2220         }
2221         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2222         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2223         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2224                 vpnInterface.getName(), vpnName);
2225     }
2226
2227     public boolean isVpnInstanceReady(String vpnInstanceName) {
2228         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2229         if (vpnRd == null) {
2230             return false;
2231         }
2232         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2233
2234         return vpnInstanceOpDataEntry != null;
2235     }
2236
2237     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2238         // FIXME: separate out to somehow?
2239         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnInstanceName);
2240         lock.lock();
2241         try {
2242             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2243                     unprocessedVpnInterfaces.get(vpnInstanceName);
2244             if (vpnInterfaces != null) {
2245                 while (!vpnInterfaces.isEmpty()) {
2246                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2247                     if (hasVpnInstanceCreatedSuccessfully) {
2248                         processSavedInterface(savedInterface, vpnInstanceName);
2249                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2250                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2251                     } else {
2252                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2253                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2254                     }
2255                 }
2256             } else {
2257                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2258             }
2259         } finally {
2260             lock.unlock();
2261         }
2262     }
2263
2264     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2265             VpnInterface vpnInterface) {
2266         // FIXME: use VpnInstanceNamesKey perhaps? What about nulls?
2267         final String firstVpnName = VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface);
2268         final ReentrantLock lock = JvmGlobalLocks.getLockForString(firstVpnName);
2269         lock.lock();
2270         try {
2271             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2272                     unprocessedVpnInterfaces.get(firstVpnName);
2273             if (vpnInterfaces != null) {
2274                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2275                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2276                             + "unprocessed list", vpnInterface.getName(), firstVpnName);
2277                 }
2278             } else {
2279                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}", firstVpnName);
2280             }
2281         } finally {
2282             lock.unlock();
2283         }
2284     }
2285
2286     public void vpnInstanceIsReady(String vpnInstanceName) {
2287         processSavedInterfaces(vpnInstanceName, true);
2288     }
2289
2290     public void vpnInstanceFailed(String vpnInstanceName) {
2291         processSavedInterfaces(vpnInstanceName, false);
2292     }
2293
2294     private static class UnprocessedVpnInterfaceData {
2295         InstanceIdentifier<VpnInterface> identifier;
2296         VpnInterface vpnInterface;
2297
2298         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2299             this.identifier = identifier;
2300             this.vpnInterface = vpnInterface;
2301         }
2302
2303         @Override
2304         public int hashCode() {
2305             final int prime = 31;
2306             int result = 1;
2307             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2308             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2309             return result;
2310         }
2311
2312         @Override
2313         public boolean equals(Object obj) {
2314             if (this == obj) {
2315                 return true;
2316             }
2317             if (obj == null) {
2318                 return false;
2319             }
2320             if (getClass() != obj.getClass()) {
2321                 return false;
2322             }
2323             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2324             if (identifier == null) {
2325                 if (other.identifier != null) {
2326                     return false;
2327                 }
2328             } else if (!identifier.equals(other.identifier)) {
2329                 return false;
2330             }
2331             if (vpnInterface == null) {
2332                 if (other.vpnInterface != null) {
2333                     return false;
2334                 }
2335             } else if (!vpnInterface.equals(other.vpnInterface)) {
2336                 return false;
2337             }
2338             return true;
2339         }
2340     }
2341
2342     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2343         String primaryRd = vpnUtil.getVpnRd(vpnName);
2344         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2345         if (vpnInstanceOpData == null) {
2346             return;
2347         }
2348         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2349         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2350             return;
2351         }
2352         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2353         vpnToDpnLists.forEach(vpnToDpnList -> {
2354             if (vpnToDpnList.getVpnInterfaces() == null) {
2355                 return;
2356             }
2357             vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2358                 try {
2359                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2360                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2361                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2362                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2363                     if (!vpnInterfaceOptional.isPresent()) {
2364                         return;
2365                     }
2366                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2367                             vpnInterface.getInterfaceName());
2368                     if (configVpnAdjacencies == null) {
2369                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2370                         return;
2371                     }
2372                     List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2373                             .augmentation(AdjacenciesOp.class).nonnullAdjacency();
2374                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2375                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2376                     // in operational DS. These unprocessed adjacencies will be handled below.
2377                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2378                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2379                     configVpnAdjacencies.stream()
2380                         .filter(adjacency -> operationVpnAdjacencies.stream()
2381                                 .noneMatch(operationalAdjacency ->
2382                                     Objects.equals(operationalAdjacency.getIpAddress(), adjacency.getIpAddress())))
2383                         .forEach(adjacency -> {
2384                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2385                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2386                                 () -> {
2387                                     // TODO Deal with sequencing — the config tx must only submitted
2388                                     // if the oper tx goes in
2389                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2390                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2391                                         futures.add(
2392                                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
2393                                                 //set of prefix used, as entry in prefix-to-interface datastore
2394                                                 // is prerequisite for refresh Fib to avoid race condition leading
2395                                                 // to missing remote next hop in bucket actions on bgp-vpn delete
2396                                                 Set<String> prefixListForRefreshFib = new HashSet<>();
2397                                                 ListenableFuture<Void> configTxFuture =
2398                                                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
2399                                                         confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2400                                                             primaryRd, adjacency,
2401                                                             vpnInterfaceOptional.get().getDpnId(),
2402                                                                 operTx, confTx, confTx, prefixListForRefreshFib));
2403                                                 Futures.addCallback(configTxFuture,
2404                                                     new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
2405                                                     MoreExecutors.directExecutor());
2406                                                 futures.add(configTxFuture);
2407                                             }));
2408                                         return futures;
2409                                     } else {
2410                                         return emptyList();
2411                                     }
2412                                 });
2413                         });
2414                 } catch (ReadFailedException e) {
2415                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2416                             vpnName, primaryRd);
2417                 }
2418             });
2419         });
2420     }
2421
2422     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2423         private final String interfaceName;
2424         private final boolean add;
2425         private final String txnDestination;
2426
2427         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2428             this.interfaceName = interfaceName;
2429             this.add = add;
2430             this.txnDestination = transactionDest;
2431         }
2432
2433         @Override
2434         public void onSuccess(Void voidObj) {
2435             if (add) {
2436                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2437                         interfaceName, txnDestination);
2438             } else {
2439                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2440             }
2441         }
2442
2443         @Override
2444         public void onFailure(Throwable throwable) {
2445             if (add) {
2446                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2447                         interfaceName, txnDestination, throwable);
2448             } else {
2449                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2450                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2451             }
2452         }
2453     }
2454
2455     private class VpnInterfaceCallBackHandler implements FutureCallback<Void> {
2456         private final String primaryRd;
2457         private final Set<String> prefixListForRefreshFib;
2458
2459         VpnInterfaceCallBackHandler(String primaryRd, Set<String> prefixListForRefreshFib) {
2460             this.primaryRd = primaryRd;
2461             this.prefixListForRefreshFib = prefixListForRefreshFib;
2462         }
2463
2464         @Override
2465         public void onSuccess(Void voidObj) {
2466             prefixListForRefreshFib.forEach(prefix -> {
2467                 fibManager.refreshVrfEntry(primaryRd, prefix);
2468             });
2469         }
2470
2471         @Override
2472         public void onFailure(Throwable throwable) {
2473             LOG.debug("write Tx config operation failed", throwable);
2474         }
2475     }
2476 }