vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14
  15 import com.google.common.base.Optional;
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.collect.Iterators;
  18 import com.google.common.util.concurrent.FutureCallback;
  19 import com.google.common.util.concurrent.Futures;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import com.google.common.util.concurrent.MoreExecutors;
  22 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  23 import java.util.ArrayList;
  24 import java.util.Collections;
  25 import java.util.HashSet;
  26 import java.util.List;
  27 import java.util.Map;
  28 import java.util.Objects;
  29 import java.util.Set;
  30 import java.util.concurrent.ConcurrentHashMap;
  31 import java.util.concurrent.ConcurrentLinkedQueue;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.concurrent.locks.ReentrantLock;
  34 import java.util.function.Consumer;
  35 import java.util.function.Predicate;
  36 import java.util.stream.Collectors;
  37 import javax.annotation.PostConstruct;
  38 import javax.annotation.PreDestroy;
  39 import javax.inject.Inject;
  40 import javax.inject.Singleton;
  41 import org.eclipse.jdt.annotation.Nullable;
  42 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  43 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  44 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  45 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  46 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  47 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  48 import org.opendaylight.genius.infra.Datastore.Configuration;
  49 import org.opendaylight.genius.infra.Datastore.Operational;
  50 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  51 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  52 import org.opendaylight.genius.infra.TransactionAdapter;
  53 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  54 import org.opendaylight.genius.infra.TypedWriteTransaction;
  55 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  56 import org.opendaylight.genius.mdsalutil.NWUtil;
  57 import org.opendaylight.genius.mdsalutil.NwConstants;
  58 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  59 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  60 import org.opendaylight.genius.utils.JvmGlobalLocks;
  61 import org.opendaylight.infrautils.caches.CacheProvider;
  62 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  63 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  64 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  65 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  66 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  67 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  68 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  69 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  70 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  71 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  72 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  73 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  74 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  75 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  76 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  77 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  78 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  79 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  80 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 120 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 121 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 122 import org.opendaylight.yangtools.yang.common.Uint32;
 123 import org.opendaylight.yangtools.yang.common.Uint64;
 124 import org.slf4j.Logger;
 125 import org.slf4j.LoggerFactory;
 126
 127 @Singleton
 128 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 129
 130     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 131     private static final short DJC_MAX_RETRIES = 3;
 132
 133     private final DataBroker dataBroker;
 134     private final ManagedNewTransactionRunner txRunner;
 135     private final IBgpManager bgpManager;
 136     private final IFibManager fibManager;
 137     private final IMdsalApiManager mdsalManager;
 138     private final IdManagerService idManager;
 139     private final OdlInterfaceRpcService ifaceMgrRpcService;
 140     private final VpnFootprintService vpnFootprintService;
 141     private final IInterfaceManager interfaceManager;
 142     private final IVpnManager vpnManager;
 143     private final ArpResponderHandler arpResponderHandler;
 144     private final JobCoordinator jobCoordinator;
 145     private final VpnUtil vpnUtil;
 146
 147     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 148
 149     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 150             new ConcurrentHashMap<>();
 151
 152     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 153
 154     @Inject
 155     public VpnInterfaceManager(final DataBroker dataBroker,
 156                                final IBgpManager bgpManager,
 157                                final IdManagerService idManager,
 158                                final IMdsalApiManager mdsalManager,
 159                                final IFibManager fibManager,
 160                                final OdlInterfaceRpcService ifaceMgrRpcService,
 161                                final VpnFootprintService vpnFootprintService,
 162                                final IInterfaceManager interfaceManager,
 163                                final IVpnManager vpnManager,
 164                                final ArpResponderHandler arpResponderHandler,
 165                                final JobCoordinator jobCoordinator,
 166                                final CacheProvider cacheProvider,
 167                                final VpnUtil vpnUtil) {
 168         super(VpnInterface.class, VpnInterfaceManager.class);
 169
 170         this.dataBroker = dataBroker;
 171         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 172         this.bgpManager = bgpManager;
 173         this.idManager = idManager;
 174         this.mdsalManager = mdsalManager;
 175         this.fibManager = fibManager;
 176         this.ifaceMgrRpcService = ifaceMgrRpcService;
 177         this.vpnFootprintService = vpnFootprintService;
 178         this.interfaceManager = interfaceManager;
 179         this.vpnManager = vpnManager;
 180         this.arpResponderHandler = arpResponderHandler;
 181         this.jobCoordinator = jobCoordinator;
 182         this.vpnUtil = vpnUtil;
 183
 184         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 185                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 186                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 187     }
 188
 189     public Runnable isNotifyTaskQueued(String intfName) {
 190         return vpnIntfMap.remove(intfName);
 191     }
 192
 193     @PostConstruct
 194     public void start() {
 195         LOG.info("{} start", getClass().getSimpleName());
 196         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 197     }
 198
 199     @Override
 200     @PreDestroy
 201     public void close() {
 202         super.close();
 203         vpnInstanceOpDataEntryCache.close();
 204     }
 205
 206     @Override
 207     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 208         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 209     }
 210
 211     @Override
 212     protected VpnInterfaceManager getDataTreeChangeListener() {
 213         return VpnInterfaceManager.this;
 214     }
 215
 216     @Override
 217     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 218         LOG.trace("Received VpnInterface add event: vpnInterface={}", vpnInterface);
 219         LOG.info("add: intfName {} onto vpnName {}", vpnInterface.getName(),
 220                 VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 221         addVpnInterface(identifier, vpnInterface, null, null);
 222     }
 223
 224     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 225                           final VpnInterface vpnInterface, String vpnName) {
 226         // FIXME: separate this out somehow?
 227         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnName);
 228         lock.lock();
 229         try {
 230             if (isVpnInstanceReady(vpnName)) {
 231                 return true;
 232             }
 233             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 234             return false;
 235         } finally {
 236             lock.unlock();
 237         }
 238     }
 239
 240     // TODO Clean up the exception handling
 241     @SuppressWarnings("checkstyle:IllegalCatch")
 242     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 243                              final @Nullable List<Adjacency> oldAdjs, final @Nullable List<Adjacency> newAdjs) {
 244         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
 245             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 246             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 247         }
 248     }
 249
 250     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 251                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 252         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 253         final String interfaceName = key.getName();
 254
 255         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 256             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 257                   vpnName, vpnInterface.getName());
 258             return;
 259         }
 260         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 261              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 262         List<Adjacency> copyOldAdjs = null;
 263         if (oldAdjs != null) {
 264             copyOldAdjs = new ArrayList<>();
 265             copyOldAdjs.addAll(oldAdjs);
 266         }
 267         List<Adjacency> copyNewAdjs = null;
 268         if (newAdjs != null) {
 269             copyNewAdjs = new ArrayList<>();
 270             copyNewAdjs.addAll(newAdjs);
 271         }
 272         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 273     }
 274
 275     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 276                     final VpnInterface vpnInterface, final @Nullable List<Adjacency> oldAdjs,
 277                     final @Nullable List<Adjacency> newAdjs,
 278                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 279         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 280         final String interfaceName = key.getName();
 281         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 282         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 283             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 284             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 285             if (interfaceState != null) {
 286                 try {
 287                     final Uint64 dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 288                     final int ifIndex = interfaceState.getIfIndex();
 289                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 290                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 291                         // (the inventory tx goes in last)
 292                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 293                         //set of prefix used, as entry in prefix-to-interface datastore
 294                         // is prerequisite for refresh Fib to avoid race condition leading to
 295                         // missing remote next hop in bucket actions on bgp-vpn delete
 296                         Set<String> prefixListForRefreshFib = new HashSet<>();
 297                         ListenableFuture<Void> confFuture =
 298                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 299                                 confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
 300                                     operTx -> futures.add(
 301                                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, invTx -> {
 302                                             LOG.info(
 303                                                 "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 304                                                     + " on dpn {}",
 305                                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 306                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 307                                                 confTx, operTx, invTx, interfaceState, vpnName,
 308                                                 prefixListForRefreshFib);
 309                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 310                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 311                                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 312                                                 if (newAdjs != null) {
 313                                                     for (Adjacency adj : newAdjs) {
 314                                                         if (oldAdjs.contains(adj)) {
 315                                                             oldAdjs.remove(adj);
 316                                                         } else {
 317                                                             if (!isBgpVpnInternetVpn
 318                                                                 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 319                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 320                                                                     primaryRd, adj, dpnId, operTx, confTx, invTx,
 321                                                                     prefixListForRefreshFib);
 322                                                             }
 323                                                         }
 324                                                     }
 325                                                 }
 326                                                 for (Adjacency adj : oldAdjs) {
 327                                                     if (!isBgpVpnInternetVpn
 328                                                         || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 329                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 330                                                             operTx, confTx);
 331                                                     }
 332                                                 }
 333                                             }
 334                                         })))));
 335                         Futures.addCallback(confFuture,
 336                             new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
 337                             MoreExecutors.directExecutor());
 338                         futures.add(confFuture);
 339                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 340                             MoreExecutors.directExecutor());
 341                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 342                             + " processed successfully", interfaceName, vpnName, dpnId);
 343                         return futures;
 344                     });
 345                 } catch (NumberFormatException | IllegalStateException e) {
 346                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 347                             + "interface {}. Interface addition on vpn {} failed", interfaceName,
 348                         vpnName, e);
 349                     return;
 350                 }
 351             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 352                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 353                     () -> {
 354                         ListenableFuture<Void> future =
 355                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
 356                                 createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 357                                     confTx, vpnName);
 358                                 LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 359                                     vpnName, vpnInterface.getDpnId());
 360                             });
 361                         ListenableFutures.addErrorLogging(future, LOG,
 362                             "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 363                         return Collections.singletonList(future);
 364                     });
 365             } else {
 366                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 367                     + " is not available", interfaceName, vpnName);
 368             }
 369         } else {
 370             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 371                     + " as vpn is pending delete", interfaceName, vpnName,
 372                 vpnInterface.getDpnId());
 373         }
 374     }
 375
 376     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 377     // see comments below.
 378     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 379     protected void processVpnInterfaceUp(final Uint64 dpId, VpnInterface vpnInterface, final String primaryRd,
 380             final int lportTag, boolean isInterfaceUp,
 381             TypedWriteTransaction<Configuration> writeConfigTxn,
 382             TypedWriteTransaction<Operational> writeOperTxn,
 383             TypedReadWriteTransaction<Configuration> writeInvTxn,
 384             Interface interfaceState, final String vpnName,
 385             Set<String> prefixListForRefreshFib) throws ExecutionException, InterruptedException {
 386         final String interfaceName = vpnInterface.getName();
 387         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 388                 vpnName);
 389         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 390         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 391         if (!isInterfaceUp) {
 392             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 393                      interfaceName, dpId, vpnName);
 394             Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 395             if (VpnConstants.INVALID_ID.equals(vpnId)) {
 396                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 397                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 398                         dpId);
 399                 return;
 400             }
 401
 402             boolean waitForVpnInterfaceOpRemoval = false;
 403             if (opVpnInterface != null) {
 404                 String opVpnName = opVpnInterface.getVpnInstanceName();
 405                 String primaryInterfaceIp = null;
 406                 if (Objects.equals(opVpnName, vpnName)) {
 407                     // Please check if the primary VRF Entry does not exist for VPNInterface
 408                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 409                     // back to back
 410                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 411                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 412                     if (adjs == null) {
 413                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 414                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 415                                 vpnName);
 416                         return;
 417                     }
 418                     for (Adjacency adj : adjs) {
 419                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 420                             primaryInterfaceIp = adj.getIpAddress();
 421                             break;
 422                         }
 423                     }
 424                     if (primaryInterfaceIp == null) {
 425                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 426                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 427                                 dpId, vpnName);
 428                         return;
 429                     }
 430                     // Get the rd of the vpn instance
 431                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 432                     if (vrf != null) {
 433                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 434                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 435                         return;
 436                     }
 437                     waitForVpnInterfaceOpRemoval = true;
 438                 } else {
 439                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 440                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 441                 }
 442             }
 443             if (!waitForVpnInterfaceOpRemoval) {
 444                 // Add the VPNInterface and quit
 445                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 446                         null/*ipAddressSourceValuePair*/,
 447                         true /* add */);
 448                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 449                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 450                 if (!isBgpVpnInternetVpn) {
 451                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 452                 }
 453                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 454                         dpId, vpnName);
 455                 if (interfaceManager.isExternalInterface(interfaceName)) {
 456                     processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
 457                         NwConstants.ADD_FLOW);
 458                 }
 459                 return;
 460             }
 461
 462             // FIB didn't get a chance yet to clean up this VPNInterface
 463             // Let us give it a chance here !
 464             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 465                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 466             try {
 467                 Runnable notifyTask = new VpnNotifyTask();
 468                 synchronized (notifyTask) {
 469                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 470                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 471                     // published here while holding the lock on it so this path will hit the wait before notify can be
 472                     // invoked.
 473                     vpnIntfMap.put(interfaceName, notifyTask);
 474                     try {
 475                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 476                     } catch (InterruptedException e) {
 477                         // Ignored
 478                     }
 479                 }
 480             } finally {
 481                 vpnIntfMap.remove(interfaceName);
 482             }
 483
 484             if (opVpnInterface != null) {
 485                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 486                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 487                         dpId, vpnName);
 488                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 489                 return;
 490             }
 491             // VPNInterface got removed, proceed with Add
 492             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 493                     interfaceName, dpId, vpnName);
 494             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 495                     null/*ipAddressSourceValuePair*/,
 496                     true /* add */);
 497             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 498                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 499             if (!isBgpVpnInternetVpn) {
 500                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 501             }
 502             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 503                     + " FIB to clean up", interfaceName, dpId, vpnName);
 504             if (interfaceManager.isExternalInterface(interfaceName)) {
 505                 processExternalVpnInterface(interfaceName, vpnName, dpId,
 506                                lportTag, NwConstants.ADD_FLOW);
 507             }
 508         } else {
 509             try {
 510                 // Interface is retained in the DPN, but its Link Up.
 511                 // Advertise prefixes again for this interface to BGP
 512                 InstanceIdentifier<VpnInterface> identifier =
 513                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 514                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 515                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 516                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 517                 // Perform similar operation as interface add event for extraroutes.
 518                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 519                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 520                         LogicalDatastoreType.CONFIGURATION, path);
 521                 if (!optAdjacencies.isPresent()) {
 522                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 523                     return;
 524                 }
 525                 List<Adjacency> adjacencies = optAdjacencies.get().nonnullAdjacency();
 526                 for (Adjacency adjacency : adjacencies) {
 527                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 528                         continue;
 529                     }
 530                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 531                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 532                         continue;
 533                     }
 534                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 535                             dpId, writeOperTxn, writeConfigTxn, writeInvTxn, prefixListForRefreshFib);
 536                 }
 537             } catch (ReadFailedException e) {
 538                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 539                         interfaceName, vpnName, primaryRd, dpId);
 540             }
 541         }
 542     }
 543
 544     private void processExternalVpnInterface(String interfaceName, String vpnName, Uint64 dpId,
 545         int lportTag, int addOrRemove) {
 546         Uuid extNetworkId;
 547         try {
 548             // vpn instance of ext-net interface is the network-id
 549             extNetworkId = new Uuid(vpnName);
 550         } catch (IllegalArgumentException e) {
 551             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 552                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 553             return;
 554         }
 555
 556         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 557         if (routerIds == null || routerIds.isEmpty()) {
 558             LOG.info("processExternalVpnInterface: No router is associated with {}."
 559                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 560                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 561             return;
 562         }
 563
 564         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 565                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 566         for (Uuid routerId : routerIds) {
 567             String routerName = routerId.getValue();
 568             Uint64 primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 569             if (Objects.equals(primarySwitch, dpId)) {
 570                 Routers router = vpnUtil.getExternalRouter(routerName);
 571                 if (router != null) {
 572                     if (addOrRemove == NwConstants.ADD_FLOW) {
 573                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 574                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 575                                 dpId, interfaceName, lportTag);
 576                     } else {
 577                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 578                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 579                                 dpId, interfaceName, lportTag);
 580                     }
 581                 } else {
 582                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 583                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 584                             interfaceName, dpId, vpnName);
 585                 }
 586             }
 587         }
 588     }
 589
 590     // TODO Clean up the exception handling
 591     @SuppressWarnings("checkstyle:IllegalCatch")
 592     private void advertiseAdjacenciesForVpnToBgp(final String rd, Uint64 dpnId,
 593                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 594                                                   String vpnName, String interfaceName) {
 595         if (rd == null) {
 596             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 597                   interfaceName, dpnId, vpnName);
 598             return;
 599         } else {
 600             if (rd.equals(vpnName)) {
 601                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 602                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 603                 return;
 604             }
 605         }
 606         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 607                 interfaceName, dpnId, vpnName, rd);
 608
 609         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 610         if (nextHopIp == null) {
 611             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 612                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 613                     rd, vpnName);
 614             return;
 615         }
 616
 617         try {
 618             //Read NextHops
 619             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 620             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 621                     LogicalDatastoreType.OPERATIONAL, path);
 622             if (adjacencies.isPresent()) {
 623                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 624                 if (nextHops != null && !nextHops.isEmpty()) {
 625                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 626                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 627                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 628                     Uint32 l3vni = vpnInstanceOpData.getL3vni();
 629                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 630                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 631                     for (Adjacency nextHop : nextHops) {
 632                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 633                             continue;
 634                         }
 635                         String gatewayMac = null;
 636                         Uint32 label = Uint32.ZERO;
 637                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 638                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 639                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 640                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 641                                     .get();
 642                         } else {
 643                             label = nextHop.getLabel();
 644                         }
 645                         try {
 646                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 647                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 648                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 649                                     encapType, label, l3vni, Uint32.ZERO /*l2vni*/,
 650                                     gatewayMac);
 651                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 652                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 653                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 654                         } catch (Exception e) {
 655                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 656                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 657                                     interfaceName, dpnId, e);
 658                         }
 659                     }
 660                 }
 661             }
 662         } catch (ReadFailedException e) {
 663             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 664                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 665         }
 666     }
 667
 668     // TODO Clean up the exception handling
 669     @SuppressWarnings("checkstyle:IllegalCatch")
 670     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 671                         String vpnName, String interfaceName, TypedWriteTransaction<Configuration> writeConfigTxn,
 672                         TypedWriteTransaction<Operational> writeOperTx) {
 673         //Read NextHops
 674         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 675         String rd = vpnUtil.getVpnRd(interfaceName);
 676         if (rd == null) {
 677             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 678                 interfaceName, vpnName);
 679             return;
 680         } else {
 681             if (rd.equals(vpnName)) {
 682                 LOG.info(
 683                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 684                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 685                 return;
 686             }
 687         }
 688         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 689                vpnName, rd);
 690         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 691         try {
 692             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 693                     path);
 694         } catch (ReadFailedException e) {
 695             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 696                     interfaceName, vpnName);
 697         }
 698         if (adjacencies.isPresent()) {
 699             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 700
 701             if (nextHops != null && !nextHops.isEmpty()) {
 702                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 703                         nextHops, interfaceName, vpnName, rd);
 704                 for (Adjacency nextHop : nextHops) {
 705                     try {
 706                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 707                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 708                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 709                                     interfaceName, vpnName);
 710                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 711                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 712                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 713                                     interfaceName, vpnName);
 714                         } else if (nextHop.getNextHopIpList() != null) {
 715                             // Perform similar operation as interface delete event for extraroutes.
 716                             String allocatedRd = nextHop.getVrfId();
 717                             for (String nh : nextHop.getNextHopIpList()) {
 718                                 deleteExtraRouteFromCurrentAndImportingVpns(
 719                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
 720                                     writeOperTx);
 721                             }
 722                         }
 723                     } catch (Exception e) {
 724                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 725                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 726                     }
 727                 }
 728             }
 729         }
 730     }
 731
 732     @SuppressWarnings("checkstyle:IllegalCatch")
 733     protected void processVpnInterfaceAdjacencies(Uint64 dpnId, final int lportTag, String vpnName,
 734                                                   String primaryRd, String interfaceName, final Uint32 vpnId,
 735                                                   TypedWriteTransaction<Configuration> writeConfigTxn,
 736                                                   TypedWriteTransaction<Operational> writeOperTxn,
 737                                                   TypedReadWriteTransaction<Configuration> writeInvTxn,
 738                                                   Interface interfaceState, Set<String> prefixListForRefreshFib)
 739             throws ExecutionException, InterruptedException {
 740         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 741         // Read NextHops
 742         Optional<VpnInterface> vpnInteface = Optional.absent();
 743         try {
 744             vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 745             LogicalDatastoreType.CONFIGURATION, identifier);
 746         } catch (ReadFailedException e) {
 747             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 748                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 749         }
 750         Uuid intfnetworkUuid = null;
 751         NetworkType networkType = null;
 752         Long segmentationId = Long.valueOf(-1);
 753         Adjacencies adjacencies = null;
 754         if (vpnInteface.isPresent()) {
 755             intfnetworkUuid = vpnInteface.get().getNetworkId();
 756             networkType = vpnInteface.get().getNetworkType();
 757             segmentationId = vpnInteface.get().getSegmentationId().toJava();
 758             adjacencies = vpnInteface.get().augmentation(Adjacencies.class);
 759             if (adjacencies == null) {
 760                 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 761                         null/*gwMac*/, writeOperTxn);
 762                 return;
 763             }
 764         }
 765         // Get the rd of the vpn instance
 766         String nextHopIp = null;
 767         try {
 768             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 769         } catch (Exception e) {
 770             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 771                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 772         }
 773         List<String> nhList = new ArrayList<>();
 774         if (nextHopIp != null) {
 775             nhList.add(nextHopIp);
 776             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 777                     interfaceName, dpnId, vpnName, nhList);
 778         }
 779         Optional<String> gwMac = Optional.absent();
 780         String vpnInterfaceSubnetGwMacAddress = null;
 781         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 782         Uint32 l3vni = vpnInstanceOpData.getL3vni() != null ? vpnInstanceOpData.getL3vni() : Uint32.ZERO;
 783         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 784         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 785         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 786         List<Adjacency> nextHops = adjacencies != null ? adjacencies.getAdjacency() : emptyList();
 787         List<Adjacency> value = new ArrayList<>();
 788         for (Adjacency nextHop : nextHops) {
 789             String rd = primaryRd;
 790             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 791             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 792                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 793                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 794                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 795                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 796                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 797                 continue;
 798             }
 799             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 800                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 801                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 802                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 803                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 804                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 805
 806                 Prefixes prefixes = intfnetworkUuid != null
 807                     ? VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, intfnetworkUuid ,networkType,
 808                             segmentationId, prefixCue) :
 809                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, prefixCue);
 810                 writeOperTxn.merge(VpnUtil.getPrefixToInterfaceIdentifier(
 811                         vpnUtil.getVpnId(vpnName), prefix), prefixes, true);
 812                 final Uuid subnetId = nextHop.getSubnetId();
 813
 814                 String gatewayIp = nextHop.getSubnetGatewayIp();
 815                 if (gatewayIp == null) {
 816                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 817                     if (gatewayIpOptional.isPresent()) {
 818                         gatewayIp = gatewayIpOptional.get();
 819                     }
 820                 }
 821
 822                 if (gatewayIp != null) {
 823                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 824                     if (gwMac.isPresent()) {
 825                         // A valid mac-address is available for this subnet-gateway-ip
 826                         // Use this for programming ARP_RESPONDER table here.  And save this
 827                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 828                         // to populate L3_GW_MAC_TABLE there.
 829                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 830                                 gatewayIp, gwMac.get());
 831                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 832                     } else {
 833                         // A valid mac-address is not available for this subnet-gateway-ip
 834                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 835                         // Save this connected-mac-address as gateway-mac-address for the
 836                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 837                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 838                         if (gwMac.isPresent()) {
 839                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 840                                     NwConstants.ADD_FLOW, gwMac.get());
 841                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 842                                     gatewayIp, gwMac.get());
 843                         } else {
 844                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 845                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 846                                 + "gwIp {}",
 847                                 subnetId, interfaceName, vpnName, gatewayIp);
 848                         }
 849                     }
 850                 } else {
 851                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 852                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 853                         subnetId, interfaceName, vpnName);
 854                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 855                 }
 856                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 857                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 858             } else {
 859                 //Extra route adjacency
 860                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 861                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 862                 // FIXME: separate this out somehow?
 863                 final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
 864                 lock.lock();
 865                 try {
 866                     java.util.Optional<String> rdToAllocate = vpnUtil
 867                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 868                                     nextHop.getNextHopIpList().get(0), dpnId);
 869                     if (rdToAllocate.isPresent()) {
 870                         rd = rdToAllocate.get();
 871                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 872                             rd, prefix);
 873                     } else {
 874                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 875                         continue;
 876                     }
 877                 } finally {
 878                     lock.unlock();
 879                 }
 880                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 881                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 882                         interfaceName, dpnId);
 883             }
 884             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 885             // can be different from the gateway-mac-address within the VRFEntry as the
 886             // gateway-mac-address is a superset.
 887             RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 888             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 889                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 890                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 891             Adjacency operationalAdjacency = null;
 892             try {
 893                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 894             } catch (NullPointerException e) {
 895                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 896                     input, e.getMessage());
 897                 return;
 898             }
 899             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 900                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 901                     vpnName, l3vni, origin, interfaceName, operationalAdjacency, encapType, prefixListForRefreshFib,
 902                     writeConfigTxn);
 903             }
 904             value.add(operationalAdjacency);
 905         }
 906
 907         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 908         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 909                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 910
 911         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni.longValue()).setPrimaryRd(primaryRd)
 912                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 913                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 914
 915         for (Adjacency nextHop : aug.getAdjacency()) {
 916             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 917             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 918                 RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 919                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 920                 registeredPopulator.populateFib(input, writeConfigTxn);
 921             }
 922         }
 923     }
 924
 925     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, Uint64 dpnId, AdjacenciesOp aug,
 926                                               long lportTag, String gwMac,
 927                                               TypedWriteTransaction<Operational> writeOperTxn) {
 928         VpnInterfaceOpDataEntry opInterface =
 929               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac);
 930         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 931             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 932         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
 933         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 934                 interfaceName, dpnId, vpnName);
 935     }
 936
 937     // TODO Clean up the exception handling
 938     @SuppressWarnings("checkstyle:IllegalCatch")
 939     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 940                                            StateTunnelList stateTunnelList,
 941                                            TypedWriteTransaction<Configuration> writeConfigTxn,
 942                                            TypedWriteTransaction<Operational> writeOperTxn) {
 943
 944         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
 945         Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern();
 946         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 947         List<Adjacency> adjList =
 948             adjacencies != null && adjacencies.getAdjacency() != null ? adjacencies.getAdjacency() : emptyList();
 949         if (adjList.isEmpty()) {
 950             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 951                     vpnInterface, srcDpnId);
 952             return;
 953         }
 954         String prefix = null;
 955         List<Adjacency> value = new ArrayList<>();
 956         boolean isNextHopAddReqd = false;
 957         String vpnName = vpnInterface.getVpnInstanceName();
 958         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
 959         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 960         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 961                 vpnInterface.getName(), vpnInterface.getDpnId(),
 962                 vpnInterface.getVpnInstanceName(), adjList);
 963         for (Adjacency adj : adjList) {
 964             String rd = adj.getVrfId();
 965             rd = rd != null ? rd : vpnName;
 966             prefix = adj.getIpAddress();
 967             Uint32 label = adj.getLabel();
 968             List<String> nhList = Collections.singletonList(srcTepIp);
 969             List<String> nextHopList = adj.getNextHopIpList();
 970             // If TEP is added , update the nexthop of primary adjacency.
 971             // Secondary adj nexthop is already pointing to primary adj IP address.
 972             if (nextHopList == null || nextHopList.isEmpty()) {
 973                 isNextHopAddReqd = true;
 974             }
 975
 976             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 977                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 978             } else {
 979                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 980                 if (!vrfEntryOptional.isPresent()) {
 981                     continue;
 982                 }
 983                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 984                 if (!nhList.contains(srcTepIp)) {
 985                     nhList.add(srcTepIp);
 986                     isNextHopAddReqd = true;
 987                 }
 988                 value.add(adj);
 989             }
 990
 991             if (isNextHopAddReqd) {
 992                 updateLabelMapper(label, nhList);
 993                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 994                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 995                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 996                 // Update the VRF entry with nextHop
 997                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 998                         label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
 999
1000                 //Get the list of VPN's importing this route(prefix) .
1001                 // Then update the VRF entry with nhList
1002                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1003                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1004                     String vpnRd = vpn.getVrfId();
1005                     if (vpnRd != null) {
1006                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1007                             srcTepIp, label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1008                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
1009                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
1010                             vpnInterface.getName(), srcDpnId, vpnName,
1011                             vpn.getVpnInstanceName(), vpnRd);
1012                     }
1013                 }
1014                 // Advertise the prefix to BGP only for external vpn
1015                 // since there is a nexthop change.
1016                 try {
1017                     if (!rd.equalsIgnoreCase(vpnName)) {
1018                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
1019                                 VrfEntry.EncapType.Mplsgre, label, Uint32.ZERO /*evi*/, Uint32.ZERO /*l2vni*/,
1020                                 null /*gatewayMacAddress*/);
1021                     }
1022                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
1023                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
1024                             srcDpnId, vpnName);
1025                 } catch (Exception ex) {
1026                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
1027                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1028                             vpnInterface.getName(), srcDpnId, vpnName, ex);
1029                 }
1030             }
1031         }
1032         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1033         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1034                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1035                 .addAugmentation(AdjacenciesOp.class, aug).build();
1036         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1037                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1038         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1039         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
1040                 vpnInterface.getName(), srcDpnId, vpnName);
1041
1042     }
1043
1044     // TODO Clean up the exception handling
1045     @SuppressWarnings("checkstyle:IllegalCatch")
1046     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1047                                               StateTunnelList stateTunnelList,
1048                                               TypedWriteTransaction<Configuration> writeConfigTxn,
1049                                               TypedWriteTransaction<Operational> writeOperTxn) {
1050
1051         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1052         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1053         String prefix = null;
1054         boolean isNextHopRemoveReqd = false;
1055         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
1056         Uint64 srcDpnId = Uint64.valueOf(stateTunnelList.getSrcInfo().getTepDeviceId()).intern();
1057         String vpnName = vpnInterface.getVpnInstanceName();
1058         Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1059         String primaryRd = vpnUtil.getVpnRd(vpnName);
1060         if (adjList != null) {
1061             List<Adjacency> value = new ArrayList<>();
1062             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1063                     vpnInterface.getName(), vpnInterface.getDpnId(),
1064                     vpnInterface.getVpnInstanceName(), adjList);
1065             for (Adjacency adj : adjList) {
1066                 List<String> nhList = new ArrayList<>();
1067                 String rd = adj.getVrfId();
1068                 rd = rd != null ? rd : vpnName;
1069                 prefix = adj.getIpAddress();
1070                 List<String> nextHopList = adj.getNextHopIpList();
1071                 Uint32 label = adj.getLabel();
1072                 if (nextHopList != null && !nextHopList.isEmpty()) {
1073                     isNextHopRemoveReqd = true;
1074                 }
1075                 // If TEP is deleted , remove the nexthop from primary adjacency.
1076                 // Secondary adj nexthop will continue to point to primary adj IP address.
1077                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1078                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1079                 } else {
1080                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1081                     if (!vrfEntryOptional.isPresent()) {
1082                         continue;
1083                     }
1084                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1085                     if (nhList.contains(srcTepIp)) {
1086                         nhList.remove(srcTepIp);
1087                         isNextHopRemoveReqd = true;
1088                     }
1089                     value.add(adj);
1090                 }
1091
1092                 if (isNextHopRemoveReqd) {
1093                     updateLabelMapper(label, nhList);
1094                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1095                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1096                             prefix, nhList, vpnName,
1097                             vpnId, rd, vpnInterface.getName());
1098                     // Update the VRF entry with removed nextHop
1099                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1100                             label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1101
1102                     //Get the list of VPN's importing this route(prefix) .
1103                     // Then update the VRF entry with nhList
1104                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1105                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1106                         String vpnRd = vpn.getVrfId();
1107                         if (vpnRd != null) {
1108                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1109                                 srcTepIp, label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1110                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1111                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1112                                     nhList, label, vpnInterface.getName(), srcDpnId,
1113                                     vpnName,
1114                                     vpn.getVpnInstanceName(), vpnRd);
1115                         }
1116                     }
1117
1118                     // Withdraw prefix from BGP only for external vpn.
1119                     try {
1120                         if (!rd.equalsIgnoreCase(vpnName)) {
1121                             bgpManager.withdrawPrefix(rd, prefix);
1122                         }
1123                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1124                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1125                                 vpnInterface.getName(), srcDpnId,
1126                                 vpnName);
1127                     } catch (Exception ex) {
1128                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1129                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1130                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1131                     }
1132                 }
1133             }
1134             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1135             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1136                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1137                     .addAugmentation(AdjacenciesOp.class, aug).build();
1138             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1139                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1140             writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1141             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1142                          vpnInterface.getName(), srcDpnId, vpnName);
1143         }
1144     }
1145
1146     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1147         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1148
1149         String vpnRd = vpnUtil.getVpnRd(vpnName);
1150         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1151         if (vpnInstanceOpDataEntry == null) {
1152             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1153                     + " to check for vpns exporting the routes", vpnName);
1154             return vpnsToExportRoute;
1155         }
1156
1157         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1158             if (input.getVpnInstanceName() == null) {
1159                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1160                         input.getVrfId());
1161                 return false;
1162             }
1163             return !input.getVpnInstanceName().equals(vpnName);
1164         };
1165
1166         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1167             Iterable<String> commonRTs =
1168                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1169                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1170             return Iterators.size(commonRTs.iterator()) > 0;
1171         };
1172
1173         vpnsToExportRoute =
1174                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1175                         Collectors.toList());
1176         return vpnsToExportRoute;
1177     }
1178
1179     // TODO Clean up the exception handling
1180     @SuppressWarnings("checkstyle:IllegalCatch")
1181     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1182         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1183         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1184             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1185             if (vrfEntries != null) {
1186                 ListenableFutures.addErrorLogging(
1187                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1188                         for (VrfEntry vrfEntry : vrfEntries) {
1189                             try {
1190                                 if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1191                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1192                                     LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1193                                             + " is not a controller managed non intervpn link route. Ignoring.",
1194                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1195                                     continue;
1196                                 }
1197                                 String prefix = vrfEntry.getDestPrefix();
1198                                 String gwMac = vrfEntry.getGatewayMacAddress();
1199                                 vrfEntry.nonnullRoutePaths().forEach(routePath -> {
1200                                     String nh = routePath.getNexthopAddress();
1201                                     Uint32 label = routePath.getLabel();
1202                                     if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1203                                         vrfEntry.getOrigin()))) {
1204                                         LOG.info(
1205                                             "handleVpnsExportingRoutesImporting: Importing fib entry rd {}"
1206                                                 + " prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1207                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1208                                         fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1209                                             Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1210                                                 Uint32.ZERO /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1211                                             confTx);
1212                                     } else {
1213                                         LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry"
1214                                                 + " rd {} prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1215                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1216                                         SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1217                                         importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1218                                             confTx);
1219                                     }
1220                                 });
1221                             } catch (RuntimeException e) {
1222                                 LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1223                                         + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1224                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1225                             }
1226                         }
1227                     }), LOG, "Error handing VPN exporting routes");
1228             } else {
1229                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1230                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1231             }
1232         }
1233     }
1234
1235     @Override
1236     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1237         LOG.trace("Received VpnInterface remove event: vpnInterface={}", vpnInterface);
1238         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1239         final String interfaceName = key.getName();
1240         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
1241             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1242             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1243         }
1244     }
1245
1246     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1247                                 final VpnInterface vpnInterface, final String vpnName,
1248                                 final String interfaceName) {
1249         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1250             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1251                 ListenableFuture<Void> future =
1252                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1253                         deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1254                         LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1255                     });
1256                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1257                         vpnInterface.getName(), vpnName);
1258                 return Collections.singletonList(future);
1259             }, DJC_MAX_RETRIES);
1260         } else {
1261             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1262             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1263         }
1264     }
1265
1266     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1267     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1268                                            final VpnInterface vpnInterface, final String vpnName,
1269                                            final String interfaceName, final Interface interfaceState) {
1270         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1271                 vpnName, vpnInterface.getDpnId());
1272         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1273         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1274             () -> {
1275                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1276                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1277                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1278                         writeOperTxn -> futures.add(
1279                             txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeInvTxn -> {
1280                                 LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1281                                         interfaceName, vpnName);
1282                                 Uint64 dpId;
1283                                 int ifIndex;
1284                                 String gwMacAddress;
1285                                 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1286                                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1287                                 Optional<VpnInterfaceOpDataEntry> optVpnInterface;
1288                                 try {
1289                                     optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1290                                             LogicalDatastoreType.OPERATIONAL, interfaceId);
1291                                 } catch (ReadFailedException e) {
1292                                     LOG.error("remove: Failed to read data store for interface {} vpn {}",
1293                                             interfaceName, vpnName);
1294                                     return;
1295                                 }
1296                                 if (interfaceState != null) {
1297                                     try {
1298                                         dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1299                                     } catch (NumberFormatException | IllegalStateException e) {
1300                                         LOG.error("remove: Unable to retrieve dpnId from interface operational"
1301                                                         + " data store for interface {} on dpn {} for vpn {} Fetching"
1302                                                         + " from vpn interface op data store. ", interfaceName,
1303                                                 vpnInterface.getDpnId(), vpnName, e);
1304                                         dpId = Uint64.ZERO;
1305                                     }
1306                                     ifIndex = interfaceState.getIfIndex();
1307                                     gwMacAddress = interfaceState.getPhysAddress().getValue();
1308                                 } else {
1309                                     LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1310                                             + " from vpn interface op.", interfaceName);
1311                                     if (optVpnInterface.isPresent()) {
1312                                         VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1313                                         dpId = vpnOpInterface.getDpnId();
1314                                         ifIndex = vpnOpInterface.getLportTag().intValue();
1315                                         gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1316                                     } else {
1317                                         LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1318                                                 + " as interfaceState and vpn interface op is not"
1319                                                 + " available", interfaceName, vpnName);
1320                                         return;
1321                                     }
1322                                 }
1323                                 processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1324                                         optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1325                                         writeConfigTxn, writeOperTxn, writeInvTxn);
1326                                 LOG.info(
1327                                         "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1328                                                 + "successfully",
1329                                         interfaceName, vpnInterface.getDpnId(), vpnName);
1330                             })))));
1331                 futures.add(configFuture);
1332                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(
1333                         interfaceName, false, "Config"), MoreExecutors.directExecutor());
1334                 return futures;
1335             }, DJC_MAX_RETRIES);
1336     }
1337
1338     protected void processVpnInterfaceDown(Uint64 dpId,
1339                                            String interfaceName,
1340                                            int lportTag,
1341                                            String gwMac,
1342                                            VpnInterfaceOpDataEntry vpnOpInterface,
1343                                            boolean isInterfaceStateDown,
1344                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1345                                            TypedWriteTransaction<Operational> writeOperTxn,
1346                                            TypedReadWriteTransaction<Configuration> writeInvTxn)
1347             throws ExecutionException, InterruptedException {
1348         if (vpnOpInterface == null) {
1349             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1350                     + " as it is not available in operational data store", interfaceName, dpId);
1351             return;
1352         }
1353         final String vpnName = vpnOpInterface.getVpnInstanceName();
1354         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1355                                                     interfaceName, vpnName);
1356         if (!isInterfaceStateDown) {
1357             final Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1358             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName,  null);
1359             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1360             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1361                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1362             if (interfaceManager.isExternalInterface(interfaceName)) {
1363                 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1364                     NwConstants.DEL_FLOW);
1365             }
1366             if (!isBgpVpnInternetVpn) {
1367                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1368             }
1369             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1370                     + " successful", interfaceName, dpId, vpnName);
1371         } else {
1372             // Interface is retained in the DPN, but its Link Down.
1373             // Only withdraw the prefixes for this interface from BGP
1374             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1375         }
1376     }
1377
1378     private void removeAdjacenciesFromVpn(final Uint64 dpnId, final int lportTag, final String interfaceName,
1379                                           final String vpnName, final Uint32 vpnId, String gwMac,
1380                                           TypedWriteTransaction<Configuration> writeConfigTxn,
1381                                           TypedWriteTransaction<Operational> writeOperTxn,
1382                                           TypedReadWriteTransaction<Configuration> writeInvTxn)
1383             throws ExecutionException, InterruptedException {
1384         //Read NextHops
1385         try {
1386             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1387                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1388             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1389             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1390                     LogicalDatastoreType.OPERATIONAL, path);
1391             boolean isLearntIP = false;
1392             String primaryRd = vpnUtil.getVpnRd(vpnName);
1393             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1394                     interfaceName, dpnId, vpnName, primaryRd);
1395             if (adjacencies.isPresent() && adjacencies.get().getAdjacency() != null
1396                     && !adjacencies.get().getAdjacency().isEmpty()) {
1397                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1398                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1399                         interfaceName, dpnId, vpnName, nextHops);
1400                 for (Adjacency nextHop : nextHops) {
1401                     if (nextHop.isPhysNetworkFunc()) {
1402                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1403                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1404                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1405                                 null/*writeCfgTxn*/);
1406                     } else {
1407                         String rd = nextHop.getVrfId();
1408                         List<String> nhList;
1409                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1410                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1411                             isLearntIP = nextHop.getAdjacencyType() == AdjacencyType.LearntIp ? true : false;
1412                         } else {
1413                             // This is a primary adjacency
1414                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1415                                     : emptyList();
1416                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1417                                     interfaceName, writeInvTxn);
1418                         }
1419                         if (!nhList.isEmpty()) {
1420                             if (Objects.equals(primaryRd, vpnName)) {
1421                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1422                                 //remove from FIB directly
1423                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1424                                         interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1425                             } else {
1426                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1427                                         interfaceName, writeConfigTxn, writeOperTxn);
1428                             }
1429                         } else {
1430                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1431                                             + " interface {}", nextHop.getIpAddress(), rd,
1432                                     nextHop.getAdjacencyType().toString(), interfaceName);
1433                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1434                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1435                         }
1436                     }
1437                     String ip = nextHop.getIpAddress().split("/")[0];
1438                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1439                     if (vpnVipToPort != null && vpnVipToPort.getPortName().equals(interfaceName)) {
1440                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1441                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1442                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1443                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1444                     }
1445                     // Remove the MIP-IP from VpnPortIpToPort.
1446                     if (isLearntIP) {
1447                         VpnPortipToPort persistedIp = vpnUtil.getVpnPortipToPort(vpnName, ip);
1448                         if (persistedIp != null && persistedIp.isLearntIp()
1449                                 && persistedIp.getPortName().equals(interfaceName)) {
1450                             VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1451                             LOG.info(
1452                                     "removeAdjacenciesFromVpn: Learnt-IP: {} interface {} of vpn {} removed "
1453                                             + "from VpnPortipToPort",
1454                                     persistedIp.getPortFixedip(), persistedIp.getPortName(), vpnName);
1455                         }
1456                     }
1457                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1458                     if (vpnPortipToPort != null) {
1459                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1460                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1461                                  + "Interface {} ip {} on dpn {} for vpn {}",
1462                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1463                     }
1464                 }
1465             } else {
1466                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1467                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1468                         + " Removing it.", interfaceName, vpnName, dpnId);
1469                 writeOperTxn.delete(identifier);
1470             }
1471         } catch (ReadFailedException e) {
1472             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1473                     interfaceName, dpnId, vpnName);
1474         }
1475     }
1476
1477     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1478                                                             String interfaceName, Uint64 dpnId,
1479                                                             TypedWriteTransaction<Configuration> writeConfigTxn,
1480                                                             TypedWriteTransaction<Operational> writeOperTx) {
1481         return (nh) -> {
1482             String primaryRd = vpnUtil.getVpnRd(vpnName);
1483             String prefix = nextHop.getIpAddress();
1484             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1485             LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1486                     nextHop, vpnName, interfaceName, dpnId);
1487             // FIXME: separate this out somehow?
1488             final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnNamePrefixKey);
1489             lock.lock();
1490             try {
1491                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1492                         prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1493                     //If extra-route is present behind at least one VM, then do not remove or update
1494                     //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1495                     //prefixtointerface DS
1496                     return;
1497                 }
1498                 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1499                         writeConfigTxn);
1500             } finally {
1501                 lock.unlock();
1502             }
1503             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1504                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1505                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1506         };
1507     }
1508
1509     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1510                                            Uint64 dpnId, String rd, String interfaceName,
1511                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1512                                            TypedWriteTransaction<Operational> writeOperTx) {
1513         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1514                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1515         nhList.forEach((nh) -> {
1516             //IRT: remove routes from other vpns importing it
1517             vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1518                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1519             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1520                 String vpnRd = vpn.getVrfId();
1521                 if (vpnRd != null) {
1522                     fibManager.removeOrUpdateFibEntry(vpnRd,
1523                             nextHop.getIpAddress(), nh, writeConfigTxn);
1524                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1525                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1526                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1527                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1528                 }
1529             }
1530         });
1531     }
1532
1533     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, Uint32 vpnId, Uint64 dpnId,
1534                                                  int lportTag, String gwMac, String interfaceName,
1535                                                  TypedReadWriteTransaction<Configuration> writeInvTxn)
1536             throws ExecutionException, InterruptedException {
1537         final Uuid subnetId = nextHop.getSubnetId();
1538         if (nextHop.getSubnetGatewayMacAddress() == null) {
1539             // A valid mac-address was not available for this subnet-gateway-ip
1540             // So a connected-mac-address was used for this subnet and we need
1541             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1542             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1543         }
1544         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1545                 subnetId);
1546     }
1547
1548     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, Uint64 dpnId,
1549                                                   String interfaceName) {
1550         // This is either an extra-route (or) a learned IP via subnet-route
1551         List<String> nhList = null;
1552         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1553         if (nextHopIp == null || nextHopIp.isEmpty()) {
1554             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1555                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1556                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1557                     vpnName);
1558             nhList = emptyList();
1559         } else {
1560             nhList = Collections.singletonList(nextHopIp);
1561         }
1562         return nhList;
1563     }
1564
1565     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1566         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1567         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1568         // else use connected interface
1569         if (gwPort != null && gwPort.isSubnetIp()) {
1570             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1571                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1572             return Optional.of(gwPort.getMacAddress());
1573         }
1574         return Optional.absent();
1575     }
1576
1577     @Override
1578     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1579         final VpnInterface update) {
1580         LOG.trace("Received VpnInterface update event: original={}, update={}", original, update);
1581         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}", update.getName(),
1582                 update.getDpnId(), original.getVpnInstanceNames(), update.getVpnInstanceNames());
1583         if (original.equals(update)) {
1584             LOG.info("update: original {} update {} are same. No update required.", original, update);
1585             return;
1586         }
1587         final String vpnInterfaceName = update.getName();
1588         final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1589         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1590         //handles switching between <internal VPN - external VPN>
1591         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1592             List<ListenableFuture<Void>> futures = new ArrayList<>();
1593             if (handleVpnInstanceUpdateForVpnInterface(identifier, original, update, futures)) {
1594                 LOG.info("update: handled Instance update for VPNInterface {} on dpn {} from oldVpn(s) {} "
1595                                 + "to newVpn(s) {}",
1596                         original.getName(), dpnId,
1597                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1598                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1599                 return emptyList();
1600             }
1601             updateVpnInstanceAdjChange(original, update, vpnInterfaceName, futures);
1602             return futures;
1603         });
1604     }
1605
1606     private boolean handleVpnInstanceUpdateForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1607                                                            VpnInterface original, VpnInterface update,
1608                                                            List<ListenableFuture<Void>> futures) {
1609         boolean isVpnInstanceUpdate = false;
1610         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1611         final String interfaceName = key.getName();
1612         List<String> oldVpnList = VpnUtil.getVpnListForVpnInterface(original);
1613         List<String> oldVpnListCopy = new ArrayList<>();
1614         oldVpnListCopy.addAll(oldVpnList);
1615         List<String> newVpnList = VpnUtil.getVpnListForVpnInterface(update);
1616         List<String> newVpnListCopy = new ArrayList<>();
1617         newVpnListCopy.addAll(newVpnList);
1618
1619         oldVpnList.removeAll(newVpnList);
1620         newVpnList.removeAll(oldVpnListCopy);
1621         //This block will execute only on if there is a change in the VPN Instance.
1622         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1623             /*
1624              * Internet BGP-VPN Instance update with single router:
1625              * ====================================================
1626              * In this case single VPN Interface will be part of maximum 2 VPN Instance only.
1627              *     1st VPN Instance : router VPN or external BGP-VPN.
1628              *     2nd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1629              *
1630              * VPN Instance UPDATE:
1631              * oldVpnList = 0 and newVpnList = 1 (Internet BGP-VPN)
1632              * oldVpnList = 1 and newVpnList = 0 (Internet BGP-VPN)
1633              *
1634              * External BGP-VPN Instance update with single router:
1635              * ====================================================
1636              * In this case single VPN interface will be part of maximum 1 VPN Instance only.
1637              *
1638              * Updated VPN Instance will be always either internal router VPN to
1639              * external BGP-VPN or external BGP-VPN to internal router VPN swap.
1640              *
1641              * VPN Instance UPDATE:
1642              * oldVpnList = 1 and newVpnList = 1 (router VPN to Ext-BGPVPN)
1643              * oldVpnList = 1 and newVpnList = 1 (Ext-BGPVPN to router VPN)
1644              *
1645              * Dual Router VPN Instance Update:
1646              * ================================
1647              * In this case single VPN interface will be part of maximum 3 VPN Instance only.
1648              *
1649              * 1st VPN Instance : router VPN or external BGP-VPN-1.
1650              * 2nd VPN Instance : router VPN or external BGP-VPN-2.
1651              * 3rd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1652              *
1653              * Dual Router --> Associated with common external BGP-VPN Instance.
1654              * 1st router and 2nd router are getting associated with single External BGP-VPN
1655              * 1) add 1st router to external bgpvpn --> oldVpnList=1, newVpnList=1;
1656              * 2) add 2nd router to the same external bgpvpn --> oldVpnList=1, newVpnList=0
1657              * In this case, we need to call removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1658              *
1659              *
1660              */
1661             isVpnInstanceUpdate = true;
1662             if (VpnUtil.isDualRouterVpnUpdate(oldVpnListCopy, newVpnListCopy)) {
1663                 if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1664                         && oldVpnList.size() == 1 && newVpnList.isEmpty()) {
1665                     //Identify the external BGP-VPN Instance and pass that value as newVpnList
1666                     List<String> externalBgpVpnList = new ArrayList<>();
1667                     for (String newVpnName : newVpnListCopy) {
1668                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1669                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1670                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1671                                 .BgpvpnType.BGPVPNExternal) {
1672                             externalBgpVpnList.add(newVpnName);
1673                             break;
1674                         }
1675                     }
1676                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1677                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList,
1678                             externalBgpVpnList, oldVpnListCopy, futures);
1679
1680                 } else if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1681                         && oldVpnList.isEmpty() && newVpnList.size() == 1) {
1682                     //Identify the router VPN Instance and pass that value as oldVpnList
1683                     List<String> routerVpnList = new ArrayList<>();
1684                     for (String newVpnName : newVpnListCopy) {
1685                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1686                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1687                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1688                                 .BgpvpnType.VPN) {
1689                             routerVpnList.add(newVpnName);
1690                             break;
1691                         }
1692                     }
1693                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1694                     updateVpnInstanceChange(identifier, interfaceName, original, update, routerVpnList,
1695                             newVpnList, oldVpnListCopy, futures);
1696
1697                 } else {
1698                     //Handle remaining use cases.
1699                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1700                             oldVpnListCopy, futures);
1701                 }
1702             } else {
1703                 updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1704                         oldVpnListCopy, futures);
1705             }
1706         }
1707         return isVpnInstanceUpdate;
1708     }
1709
1710     private void updateVpnInstanceChange(InstanceIdentifier<VpnInterface> identifier, String interfaceName,
1711                                          VpnInterface original, VpnInterface update, List<String> oldVpnList,
1712                                          List<String> newVpnList, List<String> oldVpnListCopy,
1713                                          List<ListenableFuture<Void>> futures) {
1714         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1715         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency() != null
1716                 ? origAdjs.getAdjacency() : new ArrayList<>();
1717         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1718         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency() != null
1719                 ? updateAdjs.getAdjacency() : new ArrayList<>();
1720
1721         boolean isOldVpnRemoveCallExecuted = false;
1722         for (String oldVpnName : oldVpnList) {
1723             LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1724                     + "remove from vpnName {} ", interfaceName, oldVpnName);
1725             removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1726             LOG.info("updateVpnInstanceChange: Processed Remove for update on VPNInterface"
1727                             + " {} upon VPN update from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName,
1728                     newVpnList);
1729             isOldVpnRemoveCallExecuted = true;
1730         }
1731         //Wait for previous interface bindings to be removed
1732         if (isOldVpnRemoveCallExecuted && !newVpnList.isEmpty()) {
1733             try {
1734                 Thread.sleep(2000);
1735             } catch (InterruptedException e) {
1736                 //Ignore
1737             }
1738         }
1739         for (String newVpnName : newVpnList) {
1740             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1741             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1742                 LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1743                         + "onto vpnName {} ", interfaceName, newVpnName);
1744                 addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1745                 LOG.info("updateVpnInstanceChange: Processed Add for update on VPNInterface {}"
1746                                 + "from oldVpn(s) {} to newVpn {} ",
1747                         interfaceName, oldVpnListCopy, newVpnName);
1748                 /* This block will execute only if V6 subnet is associated with internet BGP-VPN.
1749                  * Use Case:
1750                  *     In Dual stack network, first V4 subnet only attached to router and router is associated
1751                  *     with internet BGP-VPN(router-gw). At this point VPN interface is having only router vpn instance.
1752                  *     Later V6 subnet is added to router, at this point existing VPN interface will get updated
1753                  *     with Internet BGP-VPN instance(Note: Internet BGP-VPN Instance update in vpn interface
1754                  *     is applicable for only on V6 subnet is added to router). newVpnList = Contains only Internet
1755                  *     BGP-VPN Instance. So we required V6 primary adjacency info needs to be populated onto
1756                  *     router VPN as well as Internet BGP-VPN.
1757                  *
1758                  *     addVpnInterfaceCall() --> It will create V6 Adj onto Internet BGP-VPN only.
1759                  *     updateVpnInstanceAdjChange() --> This method call is needed for second primary V6 Adj
1760                  *                                       update in existing router VPN instance.
1761                  */
1762                 if (vpnUtil.isBgpVpnInternet(newVpnName)) {
1763                     LOG.info("updateVpnInstanceChange: VPN Interface {} with new Adjacency {} in existing "
1764                             + "VPN instance {}", interfaceName, newAdjs, original.getVpnInstanceNames());
1765                     updateVpnInstanceAdjChange(original, update, interfaceName, futures);
1766                 }
1767             }
1768         }
1769     }
1770
1771     // TODO Clean up the exception handling
1772     @SuppressWarnings("checkstyle:IllegalCatch")
1773     private List<ListenableFuture<Void>> updateVpnInstanceAdjChange(VpnInterface original, VpnInterface update,
1774                                                                     String vpnInterfaceName,
1775                                                                     List<ListenableFuture<Void>> futures) {
1776         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1777         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1778                 != null ? origAdjs.getAdjacency() : new ArrayList<>();
1779         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1780         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1781                 != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1782
1783         final Uint64 dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1784         for (VpnInstanceNames vpnInterfaceVpnInstance : update.nonnullVpnInstanceNames()) {
1785             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1786             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1787             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1788             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1789             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1790                 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1791                 //set of prefix used as entry in prefix-to-interface datastore
1792                 // is prerequisite for refresh Fib to avoid race condition leading to missing remote next hop
1793                 // in bucket actions on bgp-vpn delete
1794                 Set<String> prefixListForRefreshFib = new HashSet<>();
1795                 ListenableFuture<Void> configTxFuture = txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
1796                     confTx -> futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL,
1797                         operTx -> {
1798                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1799                                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1800                             LOG.info("VPN Interface update event-intfName {} onto vpnName {} running config-driven",
1801                                     update.getName(), newVpnName);
1802                             //handle both addition and removal of adjacencies
1803                             // currently, new adjacency may be an extra route
1804                             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1805                             if (!oldAdjs.equals(newAdjs)) {
1806                                 for (Adjacency adj : copyNewAdjs) {
1807                                     if (copyOldAdjs.contains(adj)) {
1808                                         copyOldAdjs.remove(adj);
1809                                     } else {
1810                                         // add new adjacency
1811                                         if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1812                                             try {
1813                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1814                                                     dpnId, operTx, confTx, confTx, prefixListForRefreshFib);
1815                                             } catch (RuntimeException e) {
1816                                                 LOG.error("Failed to add adjacency {} to vpn interface {} with"
1817                                                         + " dpnId {}", adj, vpnInterfaceName, dpnId, e);
1818                                             }
1819                                         }
1820                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} "
1821                                             + " added to vpn interface {} on vpn {} dpnId {}",
1822                                             adj.getIpAddress(), adj.getNextHopIpList(), adj.getLabel(),
1823                                             adj.getSubnetId(), update.getName(), newVpnName, dpnId);
1824                                     }
1825                                 }
1826                                 for (Adjacency adj : copyOldAdjs) {
1827                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1828                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1829                                             && !adj.isPhysNetworkFunc()) {
1830                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, operTx,
1831                                                 confTx);
1832                                             //remove FIB entry
1833                                             String vpnRd = vpnUtil.getVpnRd(newVpnName);
1834                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1835                                                 + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1836                                             //remove BGP entry
1837                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1838                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1839                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1840                                             }
1841                                         } else {
1842                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1843                                                 operTx, confTx);
1844                                         }
1845                                     }
1846                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1847                                         + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1848                                         adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1849                                 }
1850                             }
1851                         })));
1852                 Futures.addCallback(configTxFuture, new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
1853                     MoreExecutors.directExecutor());
1854                 futures.add(configTxFuture);
1855                 for (ListenableFuture<Void> future : futures) {
1856                     ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1857                             update.getName(), update.getVpnInstanceNames());
1858                 }
1859             } else {
1860                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1861                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1862             }
1863         }
1864         return futures;
1865     }
1866
1867     private void updateLabelMapper(Uint32 label, List<String> nextHopIpList) {
1868         final String labelStr = Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!")
1869                 .toString();
1870         // FIXME: separate this out somehow?
1871         final ReentrantLock lock = JvmGlobalLocks.getLockForString(labelStr);
1872         lock.lock();
1873         try {
1874             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1875                     .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1876             Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1877                 LogicalDatastoreType.OPERATIONAL, lriIid);
1878             if (opResult.isPresent()) {
1879                 LabelRouteInfo labelRouteInfo =
1880                         new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1881                 SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1882                     labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1883             }
1884             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1885                     nextHopIpList);
1886         } catch (ReadFailedException e) {
1887             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1888                     nextHopIpList);
1889         } catch (TransactionCommitFailedException e) {
1890             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1891                     nextHopIpList);
1892         } finally {
1893             lock.unlock();
1894         }
1895     }
1896
1897     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, Uint32 label,
1898         SubnetRoute route, String parentVpnRd, TypedWriteTransaction<Configuration> writeConfigTxn) {
1899
1900         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1901         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1902                 .addAugmentation(SubnetRoute.class, route).build();
1903         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1904         InstanceIdentifierBuilder<VrfTables> idBuilder =
1905             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1906         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1907         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1908         if (writeConfigTxn != null) {
1909             writeConfigTxn.merge(vrfTableId, vrfTableNew, CREATE_MISSING_PARENTS);
1910         } else {
1911             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1912         }
1913         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1914                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1915     }
1916
1917     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1918                                            Adjacency adj, Uint64 dpnId,
1919                                            TypedWriteTransaction<Operational> writeOperTxn,
1920                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1921                                            TypedReadWriteTransaction<Configuration> writeInvTxn,
1922                                            Set<String> prefixListForRefreshFib)
1923             throws ExecutionException, InterruptedException {
1924         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1925         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1926         try {
1927             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1928                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1929             if (optVpnInterface.isPresent()) {
1930                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1931                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1932                 String vpnName = currVpnIntf.getVpnInstanceName();
1933                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1934                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1935                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1936                         LogicalDatastoreType.OPERATIONAL, adjPath);
1937                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1938                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1939                 Uint32 l3vni = vpnInstanceOpData.getL3vni() == null ? Uint32.ZERO : vpnInstanceOpData.getL3vni();
1940                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1941                 List<Adjacency> adjacencies = new ArrayList<>();
1942                 if (optAdjacencies.isPresent() && optAdjacencies.get().getAdjacency() != null) {
1943                     adjacencies.addAll(optAdjacencies.get().getAdjacency());
1944                 }
1945                 Uint32 vpnId = vpnUtil.getVpnId(vpnName);
1946                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1947                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1948                 Adjacency operationalAdjacency = null;
1949                 //Handling dual stack neutron port primary adjacency
1950                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1951                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1952                             currVpnIntf.getName(), vpnName);
1953                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1954                             currVpnIntf.getName());
1955                     if (interfaceState != null) {
1956                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1957                             currVpnIntf.getName(), vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState,
1958                             prefixListForRefreshFib);
1959                     }
1960                 }
1961                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1962                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1963                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.LearntIp ? RouteOrigin.DYNAMIC
1964                             : RouteOrigin.STATIC;
1965                     String nh = adj.getNextHopIpList().get(0);
1966                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1967                     // FIXME: separate out to somehow?
1968                     final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnPrefixKey);
1969                     lock.lock();
1970                     try {
1971                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1972                                 vpnId, null, prefix, vpnName, nh, dpnId);
1973                         if (rdToAllocate.isPresent()) {
1974                             input.setRd(rdToAllocate.get());
1975                             operationalAdjacency = populator.createOperationalAdjacency(input);
1976                             int label = operationalAdjacency.getLabel().intValue();
1977                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1978                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
1979                                     currVpnIntf.getName(), operationalAdjacency, encapType,
1980                                     prefixListForRefreshFib, writeConfigTxn);
1981                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1982                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1983                                     vpnName, label, currVpnIntf.getName(), dpnId);
1984                         } else {
1985                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
1986                                     vpnName, prefix);
1987                             return;
1988                         }
1989                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1990                         // Keeping the MPLS check for now.
1991                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1992                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1993                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1994                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
1995                             vpnsToImportRoute.forEach(vpn -> {
1996                                 if (vpn.getVrfId() != null) {
1997                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(),
1998                                             vpnId, prefix,
1999                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
2000                                             .ifPresent(
2001                                                 rds -> vpnManager.addExtraRoute(
2002                                                         vpnUtil.getVpnName(vpn.getVpnId()), adj.getIpAddress(),
2003                                                         nh, rds, currVpnIntf.getVpnInstanceName(), l3vni,
2004                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(), opAdjacency,
2005                                                         encapType, prefixListForRefreshFib, writeConfigTxn));
2006                                 }
2007                             });
2008                         }
2009                     } finally {
2010                         lock.unlock();
2011                     }
2012                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
2013                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
2014                             currVpnIntf.getName(), vpnName);
2015
2016                     InstanceIdentifier<VpnInterface> vpnIfaceConfigidentifier = VpnUtil
2017                             .getVpnInterfaceIdentifier(currVpnIntf.getName());
2018                     Optional<VpnInterface> vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2019                             LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier);
2020                     Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix,
2021                             Prefixes.PrefixCue.PhysNetFunc);
2022                     if (vpnIntefaceConfig.isPresent()) {
2023                         pnfPrefix = VpnUtil.getPrefixToInterface(Uint64.ZERO, currVpnIntf.getName(), prefix,
2024                                 vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(),
2025                                 vpnIntefaceConfig.get().getSegmentationId().toJava(), Prefixes.PrefixCue.PhysNetFunc);
2026                     }
2027
2028                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
2029
2030                     writeOperTxn.merge(
2031                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
2032                                     prefix), pnfPrefix, true);
2033
2034                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
2035                             adj.getIpAddress(), emptyList(), null /* EncapType */, Uint32.ZERO /* label */,
2036                             Uint32.ZERO /*l3vni*/, null /* gw-mac */, parentVpnRd,
2037                             RouteOrigin.LOCAL, writeConfigTxn);
2038
2039                     input.setRd(adj.getVrfId());
2040                 }
2041                 if (operationalAdjacency == null) {
2042                     operationalAdjacency = populator.createOperationalAdjacency(input);
2043                 }
2044                 adjacencies.add(operationalAdjacency);
2045                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
2046                 VpnInterfaceOpDataEntry newVpnIntf =
2047                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
2048                                 aug, dpnId, currVpnIntf.getLportTag().toJava(),
2049                                 currVpnIntf.getGatewayMacAddress());
2050
2051                 writeOperTxn.merge(identifier, newVpnIntf, CREATE_MISSING_PARENTS);
2052             }
2053         } catch (ReadFailedException e) {
2054             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
2055                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
2056         }
2057     }
2058
2059     @Nullable
2060     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
2061         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
2062         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
2063     }
2064
2065     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
2066                                             Uint64 dpnId, TypedWriteTransaction<Operational> writeOperTxn,
2067                                             TypedWriteTransaction<Configuration> writeConfigTxn) {
2068         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
2069         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
2070         try {
2071             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
2072                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
2073             if (optVpnInterface.isPresent()) {
2074                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
2075                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
2076                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2077                         LogicalDatastoreType.OPERATIONAL, path);
2078                 if (optAdjacencies.isPresent()) {
2079                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
2080
2081                     if (adjacencies != null && !adjacencies.isEmpty()) {
2082                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
2083                         for (Adjacency adjacency : adjacencies) {
2084                             if (Objects.equals(adjacency.getIpAddress(), adj.getIpAddress())) {
2085                                 String rd = adjacency.getVrfId();
2086                                 if (adj.getNextHopIpList() != null) {
2087                                     for (String nh : adj.getNextHopIpList()) {
2088                                         deleteExtraRouteFromCurrentAndImportingVpns(
2089                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
2090                                                 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
2091                                     }
2092                                 } else if (adj.isPhysNetworkFunc()) {
2093                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
2094                                             adj.getIpAddress(), adj.getSubnetId());
2095                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
2096                                             writeConfigTxn);
2097                                 }
2098                                 break;
2099                             }
2100
2101                         }
2102                     }
2103                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
2104                             dpnId, adj.getVrfId());
2105                 } else {
2106                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
2107                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
2108                 }
2109             }
2110         } catch (ReadFailedException e) {
2111             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
2112                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
2113         }
2114     }
2115
2116     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
2117                                     String rd, String intfName, TypedWriteTransaction<Configuration> writeConfigTxn,
2118                                     TypedWriteTransaction<Operational> writeOperTx) {
2119         LOG.info("removing extra-route {} for nexthop {} in VPN {} intfName {} rd {}",
2120                 destination, nextHop, vpnName, intfName, rd);
2121         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
2122         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
2123         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
2124             String vpnRd = vpn.getVrfId();
2125             if (vpnRd != null) {
2126                 LOG.info("deleting extra-route {} for nexthop {} in VPN {} intfName {} vpnRd {}",
2127                         destination, nextHop, vpnName, intfName, vpnRd);
2128                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
2129                         writeOperTx);
2130             }
2131         }
2132     }
2133
2134     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, Uint64 dpnId) {
2135         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2136             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
2137             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
2138     }
2139
2140     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
2141         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2142             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
2143     }
2144
2145     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2146                                                 TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2147         if (vpnInterface == null) {
2148             return;
2149         }
2150         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
2151         if (adjs == null) {
2152             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2153                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2154             return;
2155         }
2156         for (Adjacency adj : adjs) {
2157             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2158                 String primaryInterfaceIp = adj.getIpAddress();
2159                 String macAddress = adj.getMacAddress();
2160                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2161
2162                 Uint32 label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
2163                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2164
2165                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2166                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2167                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
2168                         routerInt, label, writeConfigTxn);
2169                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2170                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2171                         macAddress);
2172             } else {
2173                 LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2174                                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}",
2175                         interfaceName, primaryRd, vpnName);
2176             }
2177         }
2178     }
2179
2180     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
2181             TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2182         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
2183         String rd = vpnUtil.getVpnRd(vpnName);
2184         if (adjs != null) {
2185             List<Adjacency> adjsList = adjs.nonnullAdjacency();
2186             for (Adjacency adj : adjsList) {
2187                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2188                     String primaryInterfaceIp = adj.getIpAddress();
2189                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2190                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
2191                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2192                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2193                 }
2194             }
2195         } else {
2196             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2197                     vpnInterface.getName(), rd);
2198         }
2199     }
2200
2201     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2202         final VpnInterfaceKey key = intefaceData.identifier.firstKeyOf(VpnInterface.class);
2203         final String interfaceName = key.getName();
2204         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2205                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2206         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2207                   intefaceData.identifier, vpnName);
2208     }
2209
2210     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2211                                               VpnInterface vpnInterface, String vpnName) {
2212         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2213                .get(vpnName);
2214         if (vpnInterfaces == null) {
2215             vpnInterfaces = new ConcurrentLinkedQueue<>();
2216         }
2217         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2218         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2219         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2220                 vpnInterface.getName(), vpnName);
2221     }
2222
2223     public boolean isVpnInstanceReady(String vpnInstanceName) {
2224         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2225         if (vpnRd == null) {
2226             return false;
2227         }
2228         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2229
2230         return vpnInstanceOpDataEntry != null;
2231     }
2232
2233     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2234         // FIXME: separate out to somehow?
2235         final ReentrantLock lock = JvmGlobalLocks.getLockForString(vpnInstanceName);
2236         lock.lock();
2237         try {
2238             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2239                     unprocessedVpnInterfaces.get(vpnInstanceName);
2240             if (vpnInterfaces != null) {
2241                 while (!vpnInterfaces.isEmpty()) {
2242                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2243                     if (hasVpnInstanceCreatedSuccessfully) {
2244                         processSavedInterface(savedInterface, vpnInstanceName);
2245                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2246                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2247                     } else {
2248                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2249                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2250                     }
2251                 }
2252             } else {
2253                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2254             }
2255         } finally {
2256             lock.unlock();
2257         }
2258     }
2259
2260     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2261             VpnInterface vpnInterface) {
2262         // FIXME: use VpnInstanceNamesKey perhaps? What about nulls?
2263         final String firstVpnName = VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface);
2264         final ReentrantLock lock = JvmGlobalLocks.getLockForString(firstVpnName);
2265         lock.lock();
2266         try {
2267             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2268                     unprocessedVpnInterfaces.get(firstVpnName);
2269             if (vpnInterfaces != null) {
2270                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2271                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2272                             + "unprocessed list", vpnInterface.getName(), firstVpnName);
2273                 }
2274             } else {
2275                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}", firstVpnName);
2276             }
2277         } finally {
2278             lock.unlock();
2279         }
2280     }
2281
2282     public void vpnInstanceIsReady(String vpnInstanceName) {
2283         processSavedInterfaces(vpnInstanceName, true);
2284     }
2285
2286     public void vpnInstanceFailed(String vpnInstanceName) {
2287         processSavedInterfaces(vpnInstanceName, false);
2288     }
2289
2290     private static class UnprocessedVpnInterfaceData {
2291         InstanceIdentifier<VpnInterface> identifier;
2292         VpnInterface vpnInterface;
2293
2294         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2295             this.identifier = identifier;
2296             this.vpnInterface = vpnInterface;
2297         }
2298
2299         @Override
2300         public int hashCode() {
2301             final int prime = 31;
2302             int result = 1;
2303             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2304             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2305             return result;
2306         }
2307
2308         @Override
2309         public boolean equals(Object obj) {
2310             if (this == obj) {
2311                 return true;
2312             }
2313             if (obj == null) {
2314                 return false;
2315             }
2316             if (getClass() != obj.getClass()) {
2317                 return false;
2318             }
2319             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2320             if (identifier == null) {
2321                 if (other.identifier != null) {
2322                     return false;
2323                 }
2324             } else if (!identifier.equals(other.identifier)) {
2325                 return false;
2326             }
2327             if (vpnInterface == null) {
2328                 if (other.vpnInterface != null) {
2329                     return false;
2330                 }
2331             } else if (!vpnInterface.equals(other.vpnInterface)) {
2332                 return false;
2333             }
2334             return true;
2335         }
2336     }
2337
2338     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2339         String primaryRd = vpnUtil.getVpnRd(vpnName);
2340         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2341         if (vpnInstanceOpData == null) {
2342             return;
2343         }
2344         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2345         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2346             return;
2347         }
2348         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2349         vpnToDpnLists.forEach(vpnToDpnList -> {
2350             if (vpnToDpnList.getVpnInterfaces() == null) {
2351                 return;
2352             }
2353             vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2354                 try {
2355                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2356                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2357                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2358                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2359                     if (!vpnInterfaceOptional.isPresent()) {
2360                         return;
2361                     }
2362                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2363                             vpnInterface.getInterfaceName());
2364                     if (configVpnAdjacencies == null) {
2365                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2366                         return;
2367                     }
2368                     List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2369                             .augmentation(AdjacenciesOp.class).nonnullAdjacency();
2370                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2371                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2372                     // in operational DS. These unprocessed adjacencies will be handled below.
2373                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2374                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2375                     configVpnAdjacencies.stream()
2376                         .filter(adjacency -> operationVpnAdjacencies.stream()
2377                                 .noneMatch(operationalAdjacency ->
2378                                     Objects.equals(operationalAdjacency.getIpAddress(), adjacency.getIpAddress())))
2379                         .forEach(adjacency -> {
2380                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2381                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2382                                 () -> {
2383                                     // TODO Deal with sequencing — the config tx must only submitted
2384                                     // if the oper tx goes in
2385                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2386                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2387                                         futures.add(
2388                                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
2389                                                 //set of prefix used, as entry in prefix-to-interface datastore
2390                                                 // is prerequisite for refresh Fib to avoid race condition leading
2391                                                 // to missing remote next hop in bucket actions on bgp-vpn delete
2392                                                 Set<String> prefixListForRefreshFib = new HashSet<>();
2393                                                 ListenableFuture<Void> configTxFuture =
2394                                                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
2395                                                         confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2396                                                             primaryRd, adjacency,
2397                                                             vpnInterfaceOptional.get().getDpnId(),
2398                                                                 operTx, confTx, confTx, prefixListForRefreshFib));
2399                                                 Futures.addCallback(configTxFuture,
2400                                                     new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
2401                                                     MoreExecutors.directExecutor());
2402                                                 futures.add(configTxFuture);
2403                                             }));
2404                                         return futures;
2405                                     } else {
2406                                         return emptyList();
2407                                     }
2408                                 });
2409                         });
2410                 } catch (ReadFailedException e) {
2411                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2412                             vpnName, primaryRd);
2413                 }
2414             });
2415         });
2416     }
2417
2418     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2419         private final String interfaceName;
2420         private final boolean add;
2421         private final String txnDestination;
2422
2423         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2424             this.interfaceName = interfaceName;
2425             this.add = add;
2426             this.txnDestination = transactionDest;
2427         }
2428
2429         @Override
2430         public void onSuccess(Void voidObj) {
2431             if (add) {
2432                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2433                         interfaceName, txnDestination);
2434             } else {
2435                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2436             }
2437         }
2438
2439         @Override
2440         public void onFailure(Throwable throwable) {
2441             if (add) {
2442                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2443                         interfaceName, txnDestination, throwable);
2444             } else {
2445                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2446                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2447             }
2448         }
2449     }
2450
2451     private class VpnInterfaceCallBackHandler implements FutureCallback<Void> {
2452         private final String primaryRd;
2453         private final Set<String> prefixListForRefreshFib;
2454
2455         VpnInterfaceCallBackHandler(String primaryRd, Set<String> prefixListForRefreshFib) {
2456             this.primaryRd = primaryRd;
2457             this.prefixListForRefreshFib = prefixListForRefreshFib;
2458         }
2459
2460         @Override
2461         public void onSuccess(Void voidObj) {
2462             prefixListForRefreshFib.forEach(prefix -> {
2463                 fibManager.refreshVrfEntry(primaryRd, prefix);
2464             });
2465         }
2466
2467         @Override
2468         public void onFailure(Throwable throwable) {
2469             LOG.debug("write Tx config operation failed", throwable);
2470         }
2471     }
2472 }