vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14 import static org.opendaylight.netvirt.vpnmanager.VpnUtil.requireNonNullElse;
  15
  16 import com.google.common.base.Optional;
  17 import com.google.common.base.Preconditions;
  18 import com.google.common.collect.Iterators;
  19 import com.google.common.util.concurrent.FutureCallback;
  20 import com.google.common.util.concurrent.Futures;
  21 import com.google.common.util.concurrent.ListenableFuture;
  22 import com.google.common.util.concurrent.MoreExecutors;
  23 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  24 import java.math.BigInteger;
  25 import java.util.ArrayList;
  26 import java.util.Collections;
  27 import java.util.HashSet;
  28 import java.util.List;
  29 import java.util.Map;
  30 import java.util.Objects;
  31 import java.util.Set;
  32 import java.util.concurrent.ConcurrentHashMap;
  33 import java.util.concurrent.ConcurrentLinkedQueue;
  34 import java.util.concurrent.ExecutionException;
  35 import java.util.function.Consumer;
  36 import java.util.function.Predicate;
  37 import java.util.stream.Collectors;
  38 import javax.annotation.Nullable;
  39 import javax.annotation.PostConstruct;
  40 import javax.annotation.PreDestroy;
  41 import javax.inject.Inject;
  42 import javax.inject.Singleton;
  43 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  44 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  45 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  46 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  47 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  48 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  49 import org.opendaylight.genius.infra.Datastore.Configuration;
  50 import org.opendaylight.genius.infra.Datastore.Operational;
  51 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  52 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  53 import org.opendaylight.genius.infra.TransactionAdapter;
  54 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  55 import org.opendaylight.genius.infra.TypedWriteTransaction;
  56 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  57 import org.opendaylight.genius.mdsalutil.NWUtil;
  58 import org.opendaylight.genius.mdsalutil.NwConstants;
  59 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  60 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  61 import org.opendaylight.infrautils.caches.CacheProvider;
  62 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  63 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  64 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  65 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  66 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  67 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  68 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  69 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  70 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  71 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  72 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  73 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  74 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  75 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  76 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  77 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  78 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  79 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  80 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 119 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType;
 120 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 121 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 122 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 123 import org.slf4j.Logger;
 124 import org.slf4j.LoggerFactory;
 125
 126 @Singleton
 127 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 128
 129     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 130     private static final short DJC_MAX_RETRIES = 3;
 131
 132     private final DataBroker dataBroker;
 133     private final ManagedNewTransactionRunner txRunner;
 134     private final IBgpManager bgpManager;
 135     private final IFibManager fibManager;
 136     private final IMdsalApiManager mdsalManager;
 137     private final IdManagerService idManager;
 138     private final OdlInterfaceRpcService ifaceMgrRpcService;
 139     private final VpnFootprintService vpnFootprintService;
 140     private final IInterfaceManager interfaceManager;
 141     private final IVpnManager vpnManager;
 142     private final ArpResponderHandler arpResponderHandler;
 143     private final JobCoordinator jobCoordinator;
 144     private final VpnUtil vpnUtil;
 145
 146     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 147
 148     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 149             new ConcurrentHashMap<>();
 150
 151     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 152
 153     @Inject
 154     public VpnInterfaceManager(final DataBroker dataBroker,
 155                                final IBgpManager bgpManager,
 156                                final IdManagerService idManager,
 157                                final IMdsalApiManager mdsalManager,
 158                                final IFibManager fibManager,
 159                                final OdlInterfaceRpcService ifaceMgrRpcService,
 160                                final VpnFootprintService vpnFootprintService,
 161                                final IInterfaceManager interfaceManager,
 162                                final IVpnManager vpnManager,
 163                                final ArpResponderHandler arpResponderHandler,
 164                                final JobCoordinator jobCoordinator,
 165                                final CacheProvider cacheProvider,
 166                                final VpnUtil vpnUtil) {
 167         super(VpnInterface.class, VpnInterfaceManager.class);
 168
 169         this.dataBroker = dataBroker;
 170         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 171         this.bgpManager = bgpManager;
 172         this.idManager = idManager;
 173         this.mdsalManager = mdsalManager;
 174         this.fibManager = fibManager;
 175         this.ifaceMgrRpcService = ifaceMgrRpcService;
 176         this.vpnFootprintService = vpnFootprintService;
 177         this.interfaceManager = interfaceManager;
 178         this.vpnManager = vpnManager;
 179         this.arpResponderHandler = arpResponderHandler;
 180         this.jobCoordinator = jobCoordinator;
 181         this.vpnUtil = vpnUtil;
 182
 183         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 184                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 185                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 186     }
 187
 188     public Runnable isNotifyTaskQueued(String intfName) {
 189         return vpnIntfMap.remove(intfName);
 190     }
 191
 192     @PostConstruct
 193     public void start() {
 194         LOG.info("{} start", getClass().getSimpleName());
 195         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 196     }
 197
 198     @Override
 199     @PreDestroy
 200     public void close() {
 201         super.close();
 202         vpnInstanceOpDataEntryCache.close();
 203     }
 204
 205     @Override
 206     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 207         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 208     }
 209
 210     @Override
 211     protected VpnInterfaceManager getDataTreeChangeListener() {
 212         return VpnInterfaceManager.this;
 213     }
 214
 215     @Override
 216     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 217         LOG.trace("Received VpnInterface add event: vpnInterface={}", vpnInterface);
 218         LOG.info("add: intfName {} onto vpnName {}", vpnInterface.getName(),
 219                 VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 220         addVpnInterface(identifier, vpnInterface, null, null);
 221     }
 222
 223     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 224                           final VpnInterface vpnInterface, String vpnName) {
 225         synchronized (vpnName.intern()) {
 226             if (isVpnInstanceReady(vpnName)) {
 227                 return true;
 228             }
 229             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 230             return false;
 231         }
 232     }
 233
 234     // TODO Clean up the exception handling
 235     @SuppressWarnings("checkstyle:IllegalCatch")
 236     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 237                              final @Nullable List<Adjacency> oldAdjs, final @Nullable List<Adjacency> newAdjs) {
 238         for (VpnInstanceNames vpnInterfaceVpnInstance : requireNonNullElse(vpnInterface.getVpnInstanceNames(),
 239                 Collections.<VpnInstanceNames>emptyList())) {
 240             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 241             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 242         }
 243     }
 244
 245     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 246                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 247         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 248         final String interfaceName = key.getName();
 249
 250         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 251             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 252                   vpnName, vpnInterface.getName());
 253             return;
 254         }
 255         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 256              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 257         List<Adjacency> copyOldAdjs = null;
 258         if (oldAdjs != null) {
 259             copyOldAdjs = new ArrayList<>();
 260             copyOldAdjs.addAll(oldAdjs);
 261         }
 262         List<Adjacency> copyNewAdjs = null;
 263         if (newAdjs != null) {
 264             copyNewAdjs = new ArrayList<>();
 265             copyNewAdjs.addAll(newAdjs);
 266         }
 267         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 268     }
 269
 270     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 271                     final VpnInterface vpnInterface, final @Nullable List<Adjacency> oldAdjs,
 272                     final @Nullable List<Adjacency> newAdjs,
 273                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 274         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 275         final String interfaceName = key.getName();
 276         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 277         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 278             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 279             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 280             if (interfaceState != null) {
 281                 try {
 282                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 283                     final int ifIndex = interfaceState.getIfIndex();
 284                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 285                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 286                         // (the inventory tx goes in last)
 287                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 288                         //set of prefix used, as entry in prefix-to-interface datastore
 289                         // is prerequisite for refresh Fib to avoid race condition leading to
 290                         // missing remote next hop in bucket actions on bgp-vpn delete
 291                         Set<String> prefixListForRefreshFib = new HashSet<>();
 292                         ListenableFuture<Void> confFuture =
 293                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 294                                 confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
 295                                     operTx -> futures.add(
 296                                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, invTx -> {
 297                                             LOG.info(
 298                                                 "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 299                                                     + " on dpn {}",
 300                                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 301                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 302                                                 confTx, operTx, invTx, interfaceState, vpnName,
 303                                                 prefixListForRefreshFib);
 304                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 305                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 306                                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 307                                                 if (newAdjs != null) {
 308                                                     for (Adjacency adj : newAdjs) {
 309                                                         if (oldAdjs.contains(adj)) {
 310                                                             oldAdjs.remove(adj);
 311                                                         } else {
 312                                                             if (!isBgpVpnInternetVpn
 313                                                                 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 314                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 315                                                                     primaryRd, adj, dpnId, operTx, confTx, invTx,
 316                                                                     prefixListForRefreshFib);
 317                                                             }
 318                                                         }
 319                                                     }
 320                                                 }
 321                                                 for (Adjacency adj : oldAdjs) {
 322                                                     if (!isBgpVpnInternetVpn
 323                                                         || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 324                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 325                                                             operTx, confTx);
 326                                                     }
 327                                                 }
 328                                             }
 329                                         })))));
 330                         Futures.addCallback(confFuture,
 331                             new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
 332                             MoreExecutors.directExecutor());
 333                         futures.add(confFuture);
 334                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 335                             MoreExecutors.directExecutor());
 336                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 337                             + " processed successfully", interfaceName, vpnName, dpnId);
 338                         return futures;
 339                     });
 340                 } catch (NumberFormatException | IllegalStateException e) {
 341                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 342                             + "interface {}. Interface addition on vpn {} failed", interfaceName,
 343                         vpnName, e);
 344                     return;
 345                 }
 346             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 347                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 348                     () -> {
 349                         ListenableFuture<Void> future =
 350                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
 351                                 createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 352                                     confTx, vpnName);
 353                                 LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 354                                     vpnName, vpnInterface.getDpnId());
 355                             });
 356                         ListenableFutures.addErrorLogging(future, LOG,
 357                             "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 358                         return Collections.singletonList(future);
 359                     });
 360             } else {
 361                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 362                     + " is not available", interfaceName, vpnName);
 363             }
 364         } else {
 365             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 366                     + " as vpn is pending delete", interfaceName, vpnName,
 367                 vpnInterface.getDpnId());
 368         }
 369     }
 370
 371     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 372     // see comments below.
 373     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 374     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 375             final int lportTag, boolean isInterfaceUp,
 376             TypedWriteTransaction<Configuration> writeConfigTxn,
 377             TypedWriteTransaction<Operational> writeOperTxn,
 378             TypedReadWriteTransaction<Configuration> writeInvTxn,
 379             Interface interfaceState, final String vpnName,
 380             Set<String> prefixListForRefreshFib) throws ExecutionException, InterruptedException {
 381         final String interfaceName = vpnInterface.getName();
 382         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 383                 vpnName);
 384         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 385         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 386         if (!isInterfaceUp) {
 387             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 388                      interfaceName, dpId, vpnName);
 389             long vpnId = vpnUtil.getVpnId(vpnName);
 390             if (vpnId == VpnConstants.INVALID_ID) {
 391                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 392                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 393                         dpId);
 394                 return;
 395             }
 396
 397             boolean waitForVpnInterfaceOpRemoval = false;
 398             if (opVpnInterface != null) {
 399                 String opVpnName = opVpnInterface.getVpnInstanceName();
 400                 String primaryInterfaceIp = null;
 401                 if (Objects.equals(opVpnName, vpnName)) {
 402                     // Please check if the primary VRF Entry does not exist for VPNInterface
 403                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 404                     // back to back
 405                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 406                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 407                     if (adjs == null) {
 408                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 409                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 410                                 vpnName);
 411                         return;
 412                     }
 413                     for (Adjacency adj : adjs) {
 414                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 415                             primaryInterfaceIp = adj.getIpAddress();
 416                             break;
 417                         }
 418                     }
 419                     if (primaryInterfaceIp == null) {
 420                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 421                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 422                                 dpId, vpnName);
 423                         return;
 424                     }
 425                     // Get the rd of the vpn instance
 426                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 427                     if (vrf != null) {
 428                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 429                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 430                         return;
 431                     }
 432                     waitForVpnInterfaceOpRemoval = true;
 433                 } else {
 434                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 435                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 436                 }
 437             }
 438             if (!waitForVpnInterfaceOpRemoval) {
 439                 // Add the VPNInterface and quit
 440                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 441                         null/*ipAddressSourceValuePair*/,
 442                         true /* add */);
 443                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 444                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 445                 if (!isBgpVpnInternetVpn) {
 446                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 447                 }
 448                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 449                         dpId, vpnName);
 450                 if (interfaceManager.isExternalInterface(interfaceName)) {
 451                     processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
 452                         NwConstants.ADD_FLOW);
 453                 }
 454                 return;
 455             }
 456
 457             // FIB didn't get a chance yet to clean up this VPNInterface
 458             // Let us give it a chance here !
 459             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 460                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 461             try {
 462                 Runnable notifyTask = new VpnNotifyTask();
 463                 synchronized (notifyTask) {
 464                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 465                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 466                     // published here while holding the lock on it so this path will hit the wait before notify can be
 467                     // invoked.
 468                     vpnIntfMap.put(interfaceName, notifyTask);
 469                     try {
 470                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 471                     } catch (InterruptedException e) {
 472                         // Ignored
 473                     }
 474                 }
 475             } finally {
 476                 vpnIntfMap.remove(interfaceName);
 477             }
 478
 479             if (opVpnInterface != null) {
 480                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 481                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 482                         dpId, vpnName);
 483                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 484                 return;
 485             }
 486             // VPNInterface got removed, proceed with Add
 487             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 488                     interfaceName, dpId, vpnName);
 489             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 490                     null/*ipAddressSourceValuePair*/,
 491                     true /* add */);
 492             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 493                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 494             if (!isBgpVpnInternetVpn) {
 495                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 496             }
 497             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 498                     + " FIB to clean up", interfaceName, dpId, vpnName);
 499             if (interfaceManager.isExternalInterface(interfaceName)) {
 500                 processExternalVpnInterface(interfaceName, vpnName, dpId,
 501                                lportTag, NwConstants.ADD_FLOW);
 502             }
 503         } else {
 504             try {
 505                 // Interface is retained in the DPN, but its Link Up.
 506                 // Advertise prefixes again for this interface to BGP
 507                 InstanceIdentifier<VpnInterface> identifier =
 508                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 509                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 510                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 511                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 512                 // Perform similar operation as interface add event for extraroutes.
 513                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 514                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 515                         LogicalDatastoreType.CONFIGURATION, path);
 516                 if (!optAdjacencies.isPresent()) {
 517                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 518                     return;
 519                 }
 520                 List<Adjacency> adjacencies = requireNonNullElse(optAdjacencies.get().getAdjacency(), emptyList());
 521                 for (Adjacency adjacency : adjacencies) {
 522                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 523                         continue;
 524                     }
 525                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 526                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 527                         continue;
 528                     }
 529                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 530                             dpId, writeOperTxn, writeConfigTxn, writeInvTxn, prefixListForRefreshFib);
 531                 }
 532             } catch (ReadFailedException e) {
 533                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 534                         interfaceName, vpnName, primaryRd, dpId);
 535             }
 536         }
 537     }
 538
 539     private void processExternalVpnInterface(String interfaceName, String vpnName, BigInteger dpId,
 540         int lportTag, int addOrRemove) {
 541         Uuid extNetworkId;
 542         try {
 543             // vpn instance of ext-net interface is the network-id
 544             extNetworkId = new Uuid(vpnName);
 545         } catch (IllegalArgumentException e) {
 546             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 547                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 548             return;
 549         }
 550
 551         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 552         if (routerIds == null || routerIds.isEmpty()) {
 553             LOG.info("processExternalVpnInterface: No router is associated with {}."
 554                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 555                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 556             return;
 557         }
 558
 559         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 560                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 561         for (Uuid routerId : routerIds) {
 562             String routerName = routerId.getValue();
 563             BigInteger primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 564             if (Objects.equals(primarySwitch, dpId)) {
 565                 Routers router = vpnUtil.getExternalRouter(routerName);
 566                 if (router != null) {
 567                     if (addOrRemove == NwConstants.ADD_FLOW) {
 568                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 569                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 570                                 dpId, interfaceName, lportTag);
 571                     } else {
 572                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 573                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 574                                 dpId, interfaceName, lportTag);
 575                     }
 576                 } else {
 577                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 578                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 579                             interfaceName, dpId, vpnName);
 580                 }
 581             }
 582         }
 583     }
 584
 585     // TODO Clean up the exception handling
 586     @SuppressWarnings("checkstyle:IllegalCatch")
 587     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 588                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 589                                                   String vpnName, String interfaceName) {
 590         if (rd == null) {
 591             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 592                   interfaceName, dpnId, vpnName);
 593             return;
 594         } else {
 595             if (rd.equals(vpnName)) {
 596                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 597                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 598                 return;
 599             }
 600         }
 601         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 602                 interfaceName, dpnId, vpnName, rd);
 603
 604         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 605         if (nextHopIp == null) {
 606             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 607                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 608                     rd, vpnName);
 609             return;
 610         }
 611
 612         try {
 613             //Read NextHops
 614             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 615             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 616                     LogicalDatastoreType.OPERATIONAL, path);
 617             if (adjacencies.isPresent()) {
 618                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 619                 if (nextHops != null && !nextHops.isEmpty()) {
 620                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 621                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 622                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 623                     long l3vni = vpnInstanceOpData.getL3vni();
 624                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 625                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 626                     for (Adjacency nextHop : nextHops) {
 627                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 628                             continue;
 629                         }
 630                         String gatewayMac = null;
 631                         long label = 0;
 632                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 633                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 634                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 635                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 636                                     .get();
 637                         } else {
 638                             label = nextHop.getLabel();
 639                         }
 640                         try {
 641                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 642                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 643                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 644                                     encapType, (int)label, l3vni, 0 /*l2vni*/,
 645                                     gatewayMac);
 646                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 647                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 648                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 649                         } catch (Exception e) {
 650                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 651                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 652                                     interfaceName, dpnId, e);
 653                         }
 654                     }
 655                 }
 656             }
 657         } catch (ReadFailedException e) {
 658             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 659                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 660         }
 661     }
 662
 663     // TODO Clean up the exception handling
 664     @SuppressWarnings("checkstyle:IllegalCatch")
 665     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 666                         String vpnName, String interfaceName, TypedWriteTransaction<Configuration> writeConfigTxn,
 667                         TypedWriteTransaction<Operational> writeOperTx) {
 668         //Read NextHops
 669         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 670         String rd = vpnUtil.getVpnRd(interfaceName);
 671         if (rd == null) {
 672             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 673                 interfaceName, vpnName);
 674             return;
 675         } else {
 676             if (rd.equals(vpnName)) {
 677                 LOG.info(
 678                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 679                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 680                 return;
 681             }
 682         }
 683         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 684                vpnName, rd);
 685         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 686         try {
 687             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 688                     path);
 689         } catch (ReadFailedException e) {
 690             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 691                     interfaceName, vpnName);
 692         }
 693         if (adjacencies.isPresent()) {
 694             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 695
 696             if (nextHops != null && !nextHops.isEmpty()) {
 697                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 698                         nextHops, interfaceName, vpnName, rd);
 699                 for (Adjacency nextHop : nextHops) {
 700                     try {
 701                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 702                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 703                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 704                                     interfaceName, vpnName);
 705                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 706                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 707                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 708                                     interfaceName, vpnName);
 709                         } else {
 710                             // Perform similar operation as interface delete event for extraroutes.
 711                             String allocatedRd = nextHop.getVrfId();
 712                             for (String nh : requireNonNullElse(nextHop.getNextHopIpList(),
 713                                     Collections.<String>emptyList())) {
 714                                 deleteExtraRouteFromCurrentAndImportingVpns(
 715                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
 716                                     writeOperTx);
 717                             }
 718                         }
 719                     } catch (Exception e) {
 720                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 721                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 722                     }
 723                 }
 724             }
 725         }
 726     }
 727
 728     @SuppressWarnings("checkstyle:IllegalCatch")
 729     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 730                                                   String primaryRd, String interfaceName, final long vpnId,
 731                                                   TypedWriteTransaction<Configuration> writeConfigTxn,
 732                                                   TypedWriteTransaction<Operational> writeOperTxn,
 733                                                   TypedReadWriteTransaction<Configuration> writeInvTxn,
 734                                                   Interface interfaceState, Set<String> prefixListForRefreshFib)
 735             throws ExecutionException, InterruptedException {
 736         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 737         // Read NextHops
 738         Optional<VpnInterface> vpnInteface = Optional.absent();
 739         try {
 740             vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 741             LogicalDatastoreType.CONFIGURATION, identifier);
 742         } catch (ReadFailedException e) {
 743             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 744                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 745         }
 746         Uuid intfnetworkUuid = null;
 747         NetworkType networkType = null;
 748         Long segmentationId = Long.valueOf(-1);
 749         Adjacencies adjacencies = null;
 750         if (vpnInteface.isPresent()) {
 751             intfnetworkUuid = vpnInteface.get().getNetworkId();
 752             networkType = vpnInteface.get().getNetworkType();
 753             segmentationId = vpnInteface.get().getSegmentationId();
 754             adjacencies = vpnInteface.get().augmentation(Adjacencies.class);
 755             if (adjacencies == null) {
 756                 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 757                         null/*gwMac*/, writeOperTxn);
 758                 return;
 759             }
 760         }
 761         // Get the rd of the vpn instance
 762         String nextHopIp = null;
 763         try {
 764             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 765         } catch (Exception e) {
 766             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 767                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 768         }
 769         List<String> nhList = new ArrayList<>();
 770         if (nextHopIp != null) {
 771             nhList.add(nextHopIp);
 772             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 773                     interfaceName, dpnId, vpnName, nhList);
 774         }
 775         Optional<String> gwMac = Optional.absent();
 776         String vpnInterfaceSubnetGwMacAddress = null;
 777         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 778         Long l3vni = vpnInstanceOpData.getL3vni();
 779         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 780         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 781         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 782         List<Adjacency> nextHops = (adjacencies != null) ? adjacencies.getAdjacency() : emptyList();
 783         List<Adjacency> value = new ArrayList<>();
 784         for (Adjacency nextHop : nextHops) {
 785             String rd = primaryRd;
 786             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 787             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 788                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 789                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 790                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 791                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 792                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 793                 continue;
 794             }
 795             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 796                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 797                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 798                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 799                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 800                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 801
 802                 Prefixes prefixes = (intfnetworkUuid != null)
 803                     ? VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, intfnetworkUuid ,networkType,
 804                             segmentationId, prefixCue) :
 805                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, prefixCue);
 806                 writeOperTxn.merge(VpnUtil.getPrefixToInterfaceIdentifier(
 807                         vpnUtil.getVpnId(vpnName), prefix), prefixes, true);
 808                 final Uuid subnetId = nextHop.getSubnetId();
 809
 810                 String gatewayIp = nextHop.getSubnetGatewayIp();
 811                 if (gatewayIp == null) {
 812                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 813                     if (gatewayIpOptional.isPresent()) {
 814                         gatewayIp = gatewayIpOptional.get();
 815                     }
 816                 }
 817
 818                 if (gatewayIp != null) {
 819                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 820                     if (gwMac.isPresent()) {
 821                         // A valid mac-address is available for this subnet-gateway-ip
 822                         // Use this for programming ARP_RESPONDER table here.  And save this
 823                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 824                         // to populate L3_GW_MAC_TABLE there.
 825                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 826                                 gatewayIp, gwMac.get());
 827                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 828                     } else {
 829                         // A valid mac-address is not available for this subnet-gateway-ip
 830                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 831                         // Save this connected-mac-address as gateway-mac-address for the
 832                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 833                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 834                         if (gwMac.isPresent()) {
 835                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 836                                     NwConstants.ADD_FLOW, gwMac.get());
 837                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 838                                     gatewayIp, gwMac.get());
 839                         } else {
 840                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 841                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 842                                 + "gwIp {}",
 843                                 subnetId, interfaceName, vpnName, gatewayIp);
 844                         }
 845                     }
 846                 } else {
 847                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 848                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 849                         subnetId, interfaceName, vpnName);
 850                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 851                 }
 852                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 853                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 854             } else {
 855                 //Extra route adjacency
 856                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 857                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 858                 synchronized (vpnPrefixKey.intern()) {
 859                     java.util.Optional<String> rdToAllocate = vpnUtil
 860                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 861                                     nextHop.getNextHopIpList().get(0), dpnId);
 862                     if (rdToAllocate.isPresent()) {
 863                         rd = rdToAllocate.get();
 864                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 865                             rd, prefix);
 866                     } else {
 867                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 868                         continue;
 869                     }
 870                 }
 871                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 872                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 873                         interfaceName, dpnId);
 874             }
 875             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 876             // can be different from the gateway-mac-address within the VRFEntry as the
 877             // gateway-mac-address is a superset.
 878             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 879                     : RouteOrigin.STATIC;
 880             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 881                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 882                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 883             Adjacency operationalAdjacency = null;
 884             try {
 885                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 886             } catch (NullPointerException e) {
 887                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 888                     input, e.getMessage());
 889                 return;
 890             }
 891             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 892                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 893                     vpnName, l3vni, origin, interfaceName, operationalAdjacency, encapType, prefixListForRefreshFib,
 894                     writeConfigTxn);
 895             }
 896             value.add(operationalAdjacency);
 897         }
 898
 899         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 900         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 901                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 902
 903         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 904                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 905                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 906
 907         for (Adjacency nextHop : aug.getAdjacency()) {
 908             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 909             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 910                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 911                         : RouteOrigin.STATIC;
 912                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 913                 registeredPopulator.populateFib(input, writeConfigTxn);
 914             }
 915         }
 916     }
 917
 918     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 919                                               long lportTag, String gwMac,
 920                                               TypedWriteTransaction<Operational> writeOperTxn) {
 921         VpnInterfaceOpDataEntry opInterface =
 922               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac);
 923         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 924             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 925         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
 926         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 927                 interfaceName, dpnId, vpnName);
 928     }
 929
 930     // TODO Clean up the exception handling
 931     @SuppressWarnings("checkstyle:IllegalCatch")
 932     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 933                                            StateTunnelList stateTunnelList,
 934                                            TypedWriteTransaction<Configuration> writeConfigTxn,
 935                                            TypedWriteTransaction<Operational> writeOperTxn) {
 936
 937         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
 938         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 939         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 940         List<Adjacency> adjList =
 941             adjacencies != null && adjacencies.getAdjacency() != null ? adjacencies.getAdjacency() : emptyList();
 942         if (adjList.isEmpty()) {
 943             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 944                     vpnInterface, srcDpnId);
 945             return;
 946         }
 947         String prefix = null;
 948         long label = 0;
 949         List<Adjacency> value = new ArrayList<>();
 950         boolean isNextHopAddReqd = false;
 951         String vpnName = vpnInterface.getVpnInstanceName();
 952         long vpnId = vpnUtil.getVpnId(vpnName);
 953         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 954         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 955                 vpnInterface.getName(), vpnInterface.getDpnId(),
 956                 vpnInterface.getVpnInstanceName(), adjList);
 957         for (Adjacency adj : adjList) {
 958             String rd = adj.getVrfId();
 959             rd = rd != null ? rd : vpnName;
 960             prefix = adj.getIpAddress();
 961             label = adj.getLabel();
 962             List<String> nhList = Collections.singletonList(srcTepIp);
 963             List<String> nextHopList = adj.getNextHopIpList();
 964             // If TEP is added , update the nexthop of primary adjacency.
 965             // Secondary adj nexthop is already pointing to primary adj IP address.
 966             if (nextHopList == null || nextHopList.isEmpty()) {
 967                 isNextHopAddReqd = true;
 968             }
 969
 970             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 971                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 972             } else {
 973                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 974                 if (!vrfEntryOptional.isPresent()) {
 975                     continue;
 976                 }
 977                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 978                 if (!nhList.contains(srcTepIp)) {
 979                     nhList.add(srcTepIp);
 980                     isNextHopAddReqd = true;
 981                 }
 982                 value.add(adj);
 983             }
 984
 985             if (isNextHopAddReqd) {
 986                 updateLabelMapper(label, nhList);
 987                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 988                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 989                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 990                 // Update the VRF entry with nextHop
 991                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 992                         label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
 993
 994                 //Get the list of VPN's importing this route(prefix) .
 995                 // Then update the VRF entry with nhList
 996                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
 997                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 998                     String vpnRd = vpn.getVrfId();
 999                     if (vpnRd != null) {
1000                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1001                             srcTepIp, label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1002                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
1003                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
1004                             vpnInterface.getName(), srcDpnId, vpnName,
1005                             vpn.getVpnInstanceName(), vpnRd);
1006                     }
1007                 }
1008                 // Advertise the prefix to BGP only for external vpn
1009                 // since there is a nexthop change.
1010                 try {
1011                     if (!rd.equalsIgnoreCase(vpnName)) {
1012                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
1013                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
1014                                 null /*gatewayMacAddress*/);
1015                     }
1016                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
1017                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
1018                             srcDpnId, vpnName);
1019                 } catch (Exception ex) {
1020                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
1021                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1022                             vpnInterface.getName(), srcDpnId, vpnName, ex);
1023                 }
1024             }
1025         }
1026         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1027         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1028                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1029                 .addAugmentation(AdjacenciesOp.class, aug).build();
1030         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1031                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1032         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1033         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
1034                 vpnInterface.getName(), srcDpnId, vpnName);
1035
1036     }
1037
1038     // TODO Clean up the exception handling
1039     @SuppressWarnings("checkstyle:IllegalCatch")
1040     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1041                                               StateTunnelList stateTunnelList,
1042                                               TypedWriteTransaction<Configuration> writeConfigTxn,
1043                                               TypedWriteTransaction<Operational> writeOperTxn) {
1044
1045         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1046         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1047         String prefix = null;
1048         long label = 0;
1049         boolean isNextHopRemoveReqd = false;
1050         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
1051         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
1052         String vpnName = vpnInterface.getVpnInstanceName();
1053         long vpnId = vpnUtil.getVpnId(vpnName);
1054         String primaryRd = vpnUtil.getVpnRd(vpnName);
1055         if (adjList != null) {
1056             List<Adjacency> value = new ArrayList<>();
1057             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1058                     vpnInterface.getName(), vpnInterface.getDpnId(),
1059                     vpnInterface.getVpnInstanceName(), adjList);
1060             for (Adjacency adj : adjList) {
1061                 List<String> nhList = new ArrayList<>();
1062                 String rd = adj.getVrfId();
1063                 rd = rd != null ? rd : vpnName;
1064                 prefix = adj.getIpAddress();
1065                 List<String> nextHopList = adj.getNextHopIpList();
1066                 label = adj.getLabel();
1067                 if (nextHopList != null && !nextHopList.isEmpty()) {
1068                     isNextHopRemoveReqd = true;
1069                 }
1070                 // If TEP is deleted , remove the nexthop from primary adjacency.
1071                 // Secondary adj nexthop will continue to point to primary adj IP address.
1072                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1073                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1074                 } else {
1075                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1076                     if (!vrfEntryOptional.isPresent()) {
1077                         continue;
1078                     }
1079                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1080                     if (nhList.contains(srcTepIp)) {
1081                         nhList.remove(srcTepIp);
1082                         isNextHopRemoveReqd = true;
1083                     }
1084                     value.add(adj);
1085                 }
1086
1087                 if (isNextHopRemoveReqd) {
1088                     updateLabelMapper(label, nhList);
1089                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1090                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1091                             prefix, nhList, vpnName,
1092                             vpnId, rd, vpnInterface.getName());
1093                     // Update the VRF entry with removed nextHop
1094                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1095                             label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1096
1097                     //Get the list of VPN's importing this route(prefix) .
1098                     // Then update the VRF entry with nhList
1099                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1100                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1101                         String vpnRd = vpn.getVrfId();
1102                         if (vpnRd != null) {
1103                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1104                                 srcTepIp, label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1105                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1106                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1107                                     nhList, label, vpnInterface.getName(), srcDpnId,
1108                                     vpnName,
1109                                     vpn.getVpnInstanceName(), vpnRd);
1110                         }
1111                     }
1112
1113                     // Withdraw prefix from BGP only for external vpn.
1114                     try {
1115                         if (!rd.equalsIgnoreCase(vpnName)) {
1116                             bgpManager.withdrawPrefix(rd, prefix);
1117                         }
1118                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1119                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1120                                 vpnInterface.getName(), srcDpnId,
1121                                 vpnName);
1122                     } catch (Exception ex) {
1123                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1124                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1125                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1126                     }
1127                 }
1128             }
1129             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1130             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1131                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1132                     .addAugmentation(AdjacenciesOp.class, aug).build();
1133             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1134                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1135             writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1136             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1137                          vpnInterface.getName(), srcDpnId, vpnName);
1138         }
1139     }
1140
1141     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1142         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1143
1144         String vpnRd = vpnUtil.getVpnRd(vpnName);
1145         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1146         if (vpnInstanceOpDataEntry == null) {
1147             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1148                     + " to check for vpns exporting the routes", vpnName);
1149             return vpnsToExportRoute;
1150         }
1151
1152         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1153             if (input.getVpnInstanceName() == null) {
1154                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1155                         input.getVrfId());
1156                 return false;
1157             }
1158             return !input.getVpnInstanceName().equals(vpnName);
1159         };
1160
1161         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1162             Iterable<String> commonRTs =
1163                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1164                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1165             return Iterators.size(commonRTs.iterator()) > 0;
1166         };
1167
1168         vpnsToExportRoute =
1169                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1170                         Collectors.toList());
1171         return vpnsToExportRoute;
1172     }
1173
1174     // TODO Clean up the exception handling
1175     @SuppressWarnings("checkstyle:IllegalCatch")
1176     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1177         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1178         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1179             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1180             if (vrfEntries != null) {
1181                 ListenableFutures.addErrorLogging(
1182                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1183                         for (VrfEntry vrfEntry : vrfEntries) {
1184                             try {
1185                                 if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1186                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1187                                     LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1188                                             + " is not a controller managed non intervpn link route. Ignoring.",
1189                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1190                                     continue;
1191                                 }
1192                                 String prefix = vrfEntry.getDestPrefix();
1193                                 String gwMac = vrfEntry.getGatewayMacAddress();
1194                                 requireNonNullElse(vrfEntry.getRoutePaths(),
1195                                     Collections.<RoutePaths>emptyList()).forEach(routePath -> {
1196                                         String nh = routePath.getNexthopAddress();
1197                                         int label = routePath.getLabel().intValue();
1198                                         if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1199                                             vrfEntry.getOrigin()))) {
1200                                             LOG.info(
1201                                                 "handleVpnsExportingRoutesImporting: Importing fib entry rd {}"
1202                                                     + " prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1203                                                 vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1204                                             fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1205                                                 Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1206                                                 0 /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1207                                                 confTx);
1208                                         } else {
1209                                             LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry"
1210                                                     + " rd {} prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1211                                                 vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1212                                             SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1213                                             importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1214                                                 confTx);
1215                                         }
1216                                     });
1217                             } catch (RuntimeException e) {
1218                                 LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1219                                         + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1220                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1221                             }
1222                         }
1223                     }), LOG, "Error handing VPN exporting routes");
1224             } else {
1225                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1226                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1227             }
1228         }
1229     }
1230
1231     @Override
1232     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1233         LOG.trace("Received VpnInterface remove event: vpnInterface={}", vpnInterface);
1234         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1235         final String interfaceName = key.getName();
1236         for (VpnInstanceNames vpnInterfaceVpnInstance : requireNonNullElse(vpnInterface.getVpnInstanceNames(),
1237                 Collections.<VpnInstanceNames>emptyList())) {
1238             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1239             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1240         }
1241     }
1242
1243     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1244                                 final VpnInterface vpnInterface, final String vpnName,
1245                                 final String interfaceName) {
1246         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1247             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1248                 ListenableFuture<Void> future =
1249                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1250                         deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1251                         LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1252                     });
1253                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1254                         vpnInterface.getName(), vpnName);
1255                 return Collections.singletonList(future);
1256             }, DJC_MAX_RETRIES);
1257         } else {
1258             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1259             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1260         }
1261     }
1262
1263     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1264     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1265                                            final VpnInterface vpnInterface, final String vpnName,
1266                                            final String interfaceName, final Interface interfaceState) {
1267         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1268                 vpnName, vpnInterface.getDpnId());
1269         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1270         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1271             () -> {
1272                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1273                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1274                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1275                         writeOperTxn -> futures.add(
1276                             txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeInvTxn -> {
1277                                 LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1278                                         interfaceName, vpnName);
1279                                 BigInteger dpId;
1280                                 int ifIndex;
1281                                 String gwMacAddress;
1282                                 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1283                                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1284                                 Optional<VpnInterfaceOpDataEntry> optVpnInterface;
1285                                 try {
1286                                     optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1287                                             LogicalDatastoreType.OPERATIONAL, interfaceId);
1288                                 } catch (ReadFailedException e) {
1289                                     LOG.error("remove: Failed to read data store for interface {} vpn {}",
1290                                             interfaceName, vpnName);
1291                                     return;
1292                                 }
1293                                 if (interfaceState != null) {
1294                                     try {
1295                                         dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1296                                     } catch (NumberFormatException | IllegalStateException e) {
1297                                         LOG.error("remove: Unable to retrieve dpnId from interface operational"
1298                                                         + " data store for interface {} on dpn {} for vpn {} Fetching"
1299                                                         + " from vpn interface op data store. ", interfaceName,
1300                                                 vpnInterface.getDpnId(), vpnName, e);
1301                                         dpId = BigInteger.ZERO;
1302                                     }
1303                                     ifIndex = interfaceState.getIfIndex();
1304                                     gwMacAddress = interfaceState.getPhysAddress().getValue();
1305                                 } else {
1306                                     LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1307                                             + " from vpn interface op.", interfaceName);
1308                                     if (optVpnInterface.isPresent()) {
1309                                         VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1310                                         dpId = vpnOpInterface.getDpnId();
1311                                         ifIndex = vpnOpInterface.getLportTag().intValue();
1312                                         gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1313                                     } else {
1314                                         LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1315                                                 + " as interfaceState and vpn interface op is not"
1316                                                 + " available", interfaceName, vpnName);
1317                                         return;
1318                                     }
1319                                 }
1320                                 processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1321                                         optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1322                                         writeConfigTxn, writeOperTxn, writeInvTxn);
1323                                 LOG.info(
1324                                         "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1325                                                 + "successfully",
1326                                         interfaceName, vpnInterface.getDpnId(), vpnName);
1327                             })))));
1328                 futures.add(configFuture);
1329                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1330                 return futures;
1331             }, DJC_MAX_RETRIES);
1332     }
1333
1334     protected void processVpnInterfaceDown(BigInteger dpId,
1335                                            String interfaceName,
1336                                            int lportTag,
1337                                            String gwMac,
1338                                            VpnInterfaceOpDataEntry vpnOpInterface,
1339                                            boolean isInterfaceStateDown,
1340                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1341                                            TypedWriteTransaction<Operational> writeOperTxn,
1342                                            TypedReadWriteTransaction<Configuration> writeInvTxn)
1343             throws ExecutionException, InterruptedException {
1344         if (vpnOpInterface == null) {
1345             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1346                     + " as it is not available in operational data store", interfaceName, dpId);
1347             return;
1348         }
1349         final String vpnName = vpnOpInterface.getVpnInstanceName();
1350         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1351                                                     interfaceName, vpnName);
1352         if (!isInterfaceStateDown) {
1353             final long vpnId = vpnUtil.getVpnId(vpnName);
1354             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName,  null);
1355             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1356             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1357                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1358             if (interfaceManager.isExternalInterface(interfaceName)) {
1359                 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1360                     NwConstants.DEL_FLOW);
1361             }
1362             if (!isBgpVpnInternetVpn) {
1363                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1364             }
1365             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1366                     + " successful", interfaceName, dpId, vpnName);
1367         } else {
1368             // Interface is retained in the DPN, but its Link Down.
1369             // Only withdraw the prefixes for this interface from BGP
1370             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1371         }
1372     }
1373
1374     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1375                                           final String vpnName, final long vpnId, String gwMac,
1376                                           TypedWriteTransaction<Configuration> writeConfigTxn,
1377                                           TypedWriteTransaction<Operational> writeOperTxn,
1378                                           TypedReadWriteTransaction<Configuration> writeInvTxn)
1379             throws ExecutionException, InterruptedException {
1380         //Read NextHops
1381         try {
1382             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1383                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1384             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1385             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1386                     LogicalDatastoreType.OPERATIONAL, path);
1387             String primaryRd = vpnUtil.getVpnRd(vpnName);
1388             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1389                     interfaceName, dpnId, vpnName, primaryRd);
1390             if (adjacencies.isPresent() && adjacencies.get().getAdjacency() != null
1391                     && !adjacencies.get().getAdjacency().isEmpty()) {
1392                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1393                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1394                         interfaceName, dpnId, vpnName, nextHops);
1395                 for (Adjacency nextHop : nextHops) {
1396                     if (nextHop.isPhysNetworkFunc()) {
1397                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1398                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1399                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1400                                 null/*writeCfgTxn*/);
1401                     } else {
1402                         String rd = nextHop.getVrfId();
1403                         List<String> nhList;
1404                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1405                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1406                         } else {
1407                             // This is a primary adjacency
1408                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1409                                     : emptyList();
1410                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1411                                     interfaceName, writeInvTxn);
1412                         }
1413                         if (!nhList.isEmpty()) {
1414                             if (Objects.equals(primaryRd, vpnName)) {
1415                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1416                                 //remove from FIB directly
1417                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1418                                         interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1419                             } else {
1420                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1421                                         interfaceName, writeConfigTxn, writeOperTxn);
1422                             }
1423                         } else {
1424                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1425                                             + " interface {}", nextHop.getIpAddress(), rd,
1426                                     nextHop.getAdjacencyType().toString(), interfaceName);
1427                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1428                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1429                         }
1430                     }
1431                     String ip = nextHop.getIpAddress().split("/")[0];
1432                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1433                     if (vpnVipToPort != null) {
1434                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1435                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1436                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1437                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1438                     }
1439                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1440                     if (vpnPortipToPort != null) {
1441                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1442                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1443                                  + "Interface {} ip {} on dpn {} for vpn {}",
1444                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1445                     }
1446                 }
1447             } else {
1448                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1449                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1450                         + " Removing it.", interfaceName, vpnName, dpnId);
1451                 writeOperTxn.delete(identifier);
1452             }
1453         } catch (ReadFailedException e) {
1454             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1455                     interfaceName, dpnId, vpnName);
1456         }
1457     }
1458
1459     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1460                                                             String interfaceName, BigInteger dpnId,
1461                                                             TypedWriteTransaction<Configuration> writeConfigTxn,
1462                                                             TypedWriteTransaction<Operational> writeOperTx) {
1463         return (nh) -> {
1464             String primaryRd = vpnUtil.getVpnRd(vpnName);
1465             String prefix = nextHop.getIpAddress();
1466             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1467             LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1468                     nextHop, vpnName, interfaceName, dpnId);
1469             synchronized (vpnNamePrefixKey.intern()) {
1470                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1471                         prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1472                     //If extra-route is present behind at least one VM, then do not remove or update
1473                     //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1474                     //prefixtointerface DS
1475                     return;
1476                 }
1477                 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1478                         writeConfigTxn);
1479             }
1480             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1481                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1482                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1483         };
1484     }
1485
1486     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1487                                            BigInteger dpnId, String rd, String interfaceName,
1488                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1489                                            TypedWriteTransaction<Operational> writeOperTx) {
1490         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1491                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1492         nhList.forEach((nh) -> {
1493             //IRT: remove routes from other vpns importing it
1494             vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1495                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1496             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1497                 String vpnRd = vpn.getVrfId();
1498                 if (vpnRd != null) {
1499                     fibManager.removeOrUpdateFibEntry(vpnRd,
1500                             nextHop.getIpAddress(), nh, writeConfigTxn);
1501                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1502                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1503                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1504                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1505                 }
1506             }
1507         });
1508     }
1509
1510     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1511                                                  int lportTag, String gwMac, String interfaceName,
1512                                                  TypedReadWriteTransaction<Configuration> writeInvTxn)
1513             throws ExecutionException, InterruptedException {
1514         final Uuid subnetId = nextHop.getSubnetId();
1515         if (nextHop.getSubnetGatewayMacAddress() == null) {
1516             // A valid mac-address was not available for this subnet-gateway-ip
1517             // So a connected-mac-address was used for this subnet and we need
1518             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1519             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1520         }
1521         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1522                 subnetId);
1523     }
1524
1525     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1526                                                   String interfaceName) {
1527         // This is either an extra-route (or) a learned IP via subnet-route
1528         List<String> nhList = null;
1529         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1530         if (nextHopIp == null || nextHopIp.isEmpty()) {
1531             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1532                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1533                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1534                     vpnName);
1535             nhList = emptyList();
1536         } else {
1537             nhList = Collections.singletonList(nextHopIp);
1538         }
1539         return nhList;
1540     }
1541
1542     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1543         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1544         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1545         // else use connected interface
1546         if (gwPort != null && gwPort.isSubnetIp()) {
1547             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1548                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1549             return Optional.of(gwPort.getMacAddress());
1550         }
1551         return Optional.absent();
1552     }
1553
1554     @Override
1555     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1556         final VpnInterface update) {
1557         LOG.trace("Received VpnInterface update event: original={}, update={}", original, update);
1558         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}", update.getName(),
1559                 update.getDpnId(), original.getVpnInstanceNames(), update.getVpnInstanceNames());
1560         final String vpnInterfaceName = update.getName();
1561         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1562         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1563         //handles switching between <internal VPN - external VPN>
1564         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1565             List<ListenableFuture<Void>> futures = new ArrayList<>();
1566             if (handleVpnInstanceUpdateForVpnInterface(identifier, original, update, futures)) {
1567                 LOG.info("update: handled Instance update for VPNInterface {} on dpn {} from oldVpn(s) {} "
1568                                 + "to newVpn(s) {}",
1569                         original.getName(), dpnId,
1570                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1571                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1572                 return emptyList();
1573             }
1574             updateVpnInstanceAdjChange(original, update, vpnInterfaceName, futures);
1575             return futures;
1576         });
1577     }
1578
1579     private boolean handleVpnInstanceUpdateForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1580                                                            VpnInterface original, VpnInterface update,
1581                                                            List<ListenableFuture<Void>> futures) {
1582         boolean isVpnInstanceUpdate = false;
1583         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1584         final String interfaceName = key.getName();
1585         List<String> oldVpnList = VpnUtil.getVpnListForVpnInterface(original);
1586         List<String> oldVpnListCopy = new ArrayList<>();
1587         oldVpnListCopy.addAll(oldVpnList);
1588         List<String> newVpnList = VpnUtil.getVpnListForVpnInterface(update);
1589         List<String> newVpnListCopy = new ArrayList<>();
1590         newVpnListCopy.addAll(newVpnList);
1591
1592         oldVpnList.removeAll(newVpnList);
1593         newVpnList.removeAll(oldVpnListCopy);
1594         //This block will execute only on if there is a change in the VPN Instance.
1595         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1596             /*
1597              * Internet BGP-VPN Instance update with single router:
1598              * ====================================================
1599              * In this case single VPN Interface will be part of maximum 2 VPN Instance only.
1600              *     1st VPN Instance : router VPN or external BGP-VPN.
1601              *     2nd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1602              *
1603              * VPN Instance UPDATE:
1604              * oldVpnList = 0 and newVpnList = 1 (Internet BGP-VPN)
1605              * oldVpnList = 1 and newVpnList = 0 (Internet BGP-VPN)
1606              *
1607              * External BGP-VPN Instance update with single router:
1608              * ====================================================
1609              * In this case single VPN interface will be part of maximum 1 VPN Instance only.
1610              *
1611              * Updated VPN Instance will be always either internal router VPN to
1612              * external BGP-VPN or external BGP-VPN to internal router VPN swap.
1613              *
1614              * VPN Instance UPDATE:
1615              * oldVpnList = 1 and newVpnList = 1 (router VPN to Ext-BGPVPN)
1616              * oldVpnList = 1 and newVpnList = 1 (Ext-BGPVPN to router VPN)
1617              *
1618              * Dual Router VPN Instance Update:
1619              * ================================
1620              * In this case single VPN interface will be part of maximum 3 VPN Instance only.
1621              *
1622              * 1st VPN Instance : router VPN or external BGP-VPN-1.
1623              * 2nd VPN Instance : router VPN or external BGP-VPN-2.
1624              * 3rd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1625              *
1626              * Dual Router --> Associated with common external BGP-VPN Instance.
1627              * 1st router and 2nd router are getting associated with single External BGP-VPN
1628              * 1) add 1st router to external bgpvpn --> oldVpnList=1, newVpnList=1;
1629              * 2) add 2nd router to the same external bgpvpn --> oldVpnList=1, newVpnList=0
1630              * In this case, we need to call removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1631              *
1632              *
1633              */
1634             isVpnInstanceUpdate = true;
1635             if (VpnUtil.isDualRouterVpnUpdate(oldVpnListCopy, newVpnListCopy)) {
1636                 if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1637                         && (oldVpnList.size() == 1 && newVpnList.size() == 0)) {
1638                     //Identify the external BGP-VPN Instance and pass that value as newVpnList
1639                     List<String> externalBgpVpnList = new ArrayList<>();
1640                     for (String newVpnName : newVpnListCopy) {
1641                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1642                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1643                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1644                                 .BgpvpnType.BGPVPNExternal) {
1645                             externalBgpVpnList.add(newVpnName);
1646                             break;
1647                         }
1648                     }
1649                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1650                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList,
1651                             externalBgpVpnList, oldVpnListCopy, futures);
1652
1653                 } else if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1654                         && (oldVpnList.size() == 0 && newVpnList.size() == 1)) {
1655                     //Identify the router VPN Instance and pass that value as oldVpnList
1656                     List<String> routerVpnList = new ArrayList<>();
1657                     for (String newVpnName : newVpnListCopy) {
1658                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1659                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1660                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1661                                 .BgpvpnType.VPN) {
1662                             routerVpnList.add(newVpnName);
1663                             break;
1664                         }
1665                     }
1666                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1667                     updateVpnInstanceChange(identifier, interfaceName, original, update, routerVpnList,
1668                             newVpnList, oldVpnListCopy, futures);
1669
1670                 } else {
1671                     //Handle remaining use cases.
1672                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1673                             oldVpnListCopy, futures);
1674                 }
1675             } else {
1676                 updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1677                         oldVpnListCopy, futures);
1678             }
1679         }
1680         return isVpnInstanceUpdate;
1681     }
1682
1683     private void updateVpnInstanceChange(InstanceIdentifier<VpnInterface> identifier, String interfaceName,
1684                                          VpnInterface original, VpnInterface update, List<String> oldVpnList,
1685                                          List<String> newVpnList, List<String> oldVpnListCopy,
1686                                          List<ListenableFuture<Void>> futures) {
1687         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1688         final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1689                 ? origAdjs.getAdjacency() : new ArrayList<>();
1690         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1691         final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1692                 ? updateAdjs.getAdjacency() : new ArrayList<>();
1693
1694         boolean isOldVpnRemoveCallExecuted = false;
1695         for (String oldVpnName : oldVpnList) {
1696             LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1697                     + "remove from vpnName {} ", interfaceName, oldVpnName);
1698             removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1699             LOG.info("updateVpnInstanceChange: Processed Remove for update on VPNInterface"
1700                             + " {} upon VPN update from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName,
1701                     newVpnList);
1702             isOldVpnRemoveCallExecuted = true;
1703         }
1704         //Wait for previous interface bindings to be removed
1705         if (isOldVpnRemoveCallExecuted && !newVpnList.isEmpty()) {
1706             try {
1707                 Thread.sleep(2000);
1708             } catch (InterruptedException e) {
1709                 //Ignore
1710             }
1711         }
1712         for (String newVpnName : newVpnList) {
1713             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1714             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1715                 LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1716                         + "onto vpnName {} ", interfaceName, newVpnName);
1717                 addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1718                 LOG.info("updateVpnInstanceChange: Processed Add for update on VPNInterface {}"
1719                                 + "from oldVpn(s) {} to newVpn {} ",
1720                         interfaceName, oldVpnListCopy, newVpnName);
1721                 /* This block will execute only if V6 subnet is associated with internet BGP-VPN.
1722                  * Use Case:
1723                  *     In Dual stack network, first V4 subnet only attached to router and router is associated
1724                  *     with internet BGP-VPN(router-gw). At this point VPN interface is having only router vpn instance.
1725                  *     Later V6 subnet is added to router, at this point existing VPN interface will get updated
1726                  *     with Internet BGP-VPN instance(Note: Internet BGP-VPN Instance update in vpn interface
1727                  *     is applicable for only on V6 subnet is added to router). newVpnList = Contains only Internet
1728                  *     BGP-VPN Instance. So we required V6 primary adjacency info needs to be populated onto
1729                  *     router VPN as well as Internet BGP-VPN.
1730                  *
1731                  *     addVpnInterfaceCall() --> It will create V6 Adj onto Internet BGP-VPN only.
1732                  *     updateVpnInstanceAdjChange() --> This method call is needed for second primary V6 Adj
1733                  *                                       update in existing router VPN instance.
1734                  */
1735                 if (vpnUtil.isBgpVpnInternet(newVpnName)) {
1736                     LOG.info("updateVpnInstanceChange: VPN Interface {} with new Adjacency {} in existing "
1737                             + "VPN instance {}", interfaceName, newAdjs, original.getVpnInstanceNames());
1738                     updateVpnInstanceAdjChange(original, update, interfaceName, futures);
1739                 }
1740             }
1741         }
1742     }
1743
1744     private List<ListenableFuture<Void>> updateVpnInstanceAdjChange(VpnInterface original, VpnInterface update,
1745                                                                     String vpnInterfaceName,
1746                                                                     List<ListenableFuture<Void>> futures) {
1747         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1748         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1749                 != null ? origAdjs.getAdjacency() : new ArrayList<>();
1750         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1751         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1752                 != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1753
1754         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1755         for (VpnInstanceNames vpnInterfaceVpnInstance : requireNonNullElse(update.getVpnInstanceNames(),
1756                 Collections.<VpnInstanceNames>emptyList())) {
1757             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1758             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1759             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1760             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1761             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1762                 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1763                 //set of prefix used as entry in prefix-to-interface datastore
1764                 // is prerequisite for refresh Fib to avoid race condition leading to missing remote next hop
1765                 // in bucket actions on bgp-vpn delete
1766                 Set<String> prefixListForRefreshFib = new HashSet<>();
1767                 ListenableFuture<Void> configTxFuture = txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
1768                     confTx -> futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL,
1769                         operTx -> {
1770                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1771                                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1772                             LOG.info("VPN Interface update event-intfName {} onto vpnName {} running config-driven",
1773                                     update.getName(), newVpnName);
1774                             //handle both addition and removal of adjacencies
1775                             // currently, new adjacency may be an extra route
1776                             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1777                             if (!oldAdjs.equals(newAdjs)) {
1778                                 for (Adjacency adj : copyNewAdjs) {
1779                                     if (copyOldAdjs.contains(adj)) {
1780                                         copyOldAdjs.remove(adj);
1781                                     } else {
1782                                         // add new adjacency
1783                                         if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1784                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1785                                                     dpnId, operTx, confTx, confTx, prefixListForRefreshFib);
1786                                         }
1787                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} "
1788                                             + " added to vpn interface {} on vpn {} dpnId {}",
1789                                             adj.getIpAddress(), adj.getNextHopIpList(), adj.getLabel(),
1790                                             adj.getSubnetId(), update.getName(), newVpnName, dpnId);
1791                                     }
1792                                 }
1793                                 for (Adjacency adj : copyOldAdjs) {
1794                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1795                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1796                                             && !adj.isPhysNetworkFunc()) {
1797                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, operTx,
1798                                                 confTx);
1799                                             //remove FIB entry
1800                                             String vpnRd = vpnUtil.getVpnRd(newVpnName);
1801                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1802                                                 + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1803                                             //remove BGP entry
1804                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1805                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1806                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1807                                             }
1808                                         } else {
1809                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1810                                                 operTx, confTx);
1811                                         }
1812                                     }
1813                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1814                                         + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1815                                         adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1816                                 }
1817                             }
1818                         })));
1819                 Futures.addCallback(configTxFuture, new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
1820                     MoreExecutors.directExecutor());
1821                 futures.add(configTxFuture);
1822                 for (ListenableFuture<Void> future : futures) {
1823                     ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1824                             update.getName(), update.getVpnInstanceNames());
1825                 }
1826             } else {
1827                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1828                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1829             }
1830         }
1831         return futures;
1832     }
1833
1834     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1835         try {
1836             Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1837             synchronized (label.toString().intern()) {
1838                 InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1839                         .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1840                 Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1841                         LogicalDatastoreType.OPERATIONAL, lriIid);
1842                 if (opResult.isPresent()) {
1843                     LabelRouteInfo labelRouteInfo =
1844                             new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1845                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1846                             labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1847                 }
1848             }
1849             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1850                     nextHopIpList);
1851         } catch (ReadFailedException e) {
1852             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1853                     nextHopIpList);
1854         } catch (TransactionCommitFailedException e) {
1855             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1856                     nextHopIpList);
1857         }
1858     }
1859
1860     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1861         SubnetRoute route, String parentVpnRd, TypedWriteTransaction<Configuration> writeConfigTxn) {
1862
1863         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1864         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1865                 .addAugmentation(SubnetRoute.class, route).build();
1866         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1867         InstanceIdentifierBuilder<VrfTables> idBuilder =
1868             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1869         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1870         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1871         if (writeConfigTxn != null) {
1872             writeConfigTxn.merge(vrfTableId, vrfTableNew, CREATE_MISSING_PARENTS);
1873         } else {
1874             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1875         }
1876         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1877                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1878     }
1879
1880     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1881                                            Adjacency adj, BigInteger dpnId,
1882                                            TypedWriteTransaction<Operational> writeOperTxn,
1883                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1884                                            TypedReadWriteTransaction<Configuration> writeInvTxn,
1885                                            Set<String> prefixListForRefreshFib)
1886             throws ExecutionException, InterruptedException {
1887         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1888         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1889         try {
1890             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1891                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1892             if (optVpnInterface.isPresent()) {
1893                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1894                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1895                 String vpnName = currVpnIntf.getVpnInstanceName();
1896                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1897                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1898                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1899                         LogicalDatastoreType.OPERATIONAL, adjPath);
1900                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1901                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1902                 long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1903                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1904                 List<Adjacency> adjacencies = new ArrayList<>();
1905                 if (optAdjacencies.isPresent() && optAdjacencies.get().getAdjacency() != null) {
1906                     adjacencies.addAll(optAdjacencies.get().getAdjacency());
1907                 }
1908                 long vpnId = vpnUtil.getVpnId(vpnName);
1909                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1910                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1911                 Adjacency operationalAdjacency = null;
1912                 //Handling dual stack neutron port primary adjacency
1913                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1914                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1915                             currVpnIntf.getName(), vpnName);
1916                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1917                             currVpnIntf.getName());
1918                     if (interfaceState != null) {
1919                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1920                             currVpnIntf.getName(), vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState,
1921                             prefixListForRefreshFib);
1922                     }
1923                 }
1924                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1925                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1926                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.LearntIp ? RouteOrigin.DYNAMIC
1927                             : RouteOrigin.STATIC;
1928                     String nh = adj.getNextHopIpList().get(0);
1929                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1930                     synchronized (vpnPrefixKey.intern()) {
1931                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1932                                 vpnId, null, prefix, vpnName, nh, dpnId);
1933                         if (rdToAllocate.isPresent()) {
1934                             input.setRd(rdToAllocate.get());
1935                             operationalAdjacency = populator.createOperationalAdjacency(input);
1936                             int label = operationalAdjacency.getLabel().intValue();
1937                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1938                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
1939                                     currVpnIntf.getName(), operationalAdjacency, encapType,
1940                                     prefixListForRefreshFib, writeConfigTxn);
1941                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1942                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1943                                     vpnName, label, currVpnIntf.getName(), dpnId);
1944                         } else {
1945                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
1946                                     vpnName, prefix);
1947                             return;
1948                         }
1949                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1950                         // Keeping the MPLS check for now.
1951                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1952                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1953                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1954                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
1955                             vpnsToImportRoute.forEach(vpn -> {
1956                                 if (vpn.getVrfId() != null) {
1957                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(), vpnId, prefix,
1958                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
1959                                             .ifPresent(
1960                                                 rds -> vpnManager.addExtraRoute(
1961                                                         vpnUtil.getVpnName(vpn.getVpnId()), adj.getIpAddress(),
1962                                                         nh, rds, currVpnIntf.getVpnInstanceName(), l3vni,
1963                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(), opAdjacency,
1964                                                         encapType, prefixListForRefreshFib, writeConfigTxn));
1965                                 }
1966                             });
1967                         }
1968                     }
1969                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1970                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1971                             currVpnIntf.getName(), vpnName);
1972
1973                     InstanceIdentifier<VpnInterface> vpnIfaceConfigidentifier = VpnUtil
1974                             .getVpnInterfaceIdentifier(currVpnIntf.getName());
1975                     Optional<VpnInterface> vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1976                             LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier);
1977                     Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1978                             Prefixes.PrefixCue.PhysNetFunc);
1979                     if (vpnIntefaceConfig.isPresent()) {
1980                         pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1981                                 vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(),
1982                                 vpnIntefaceConfig.get().getSegmentationId(), Prefixes.PrefixCue.PhysNetFunc);
1983                     }
1984
1985                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1986
1987                     writeOperTxn.merge(
1988                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
1989                                     prefix), pnfPrefix, true);
1990
1991                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1992                             adj.getIpAddress(), emptyList(), null /* EncapType */, 0 /* label */,
1993                             0 /*l3vni*/, null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1994
1995                     input.setRd(adj.getVrfId());
1996                 }
1997                 if (operationalAdjacency == null) {
1998                     operationalAdjacency = populator.createOperationalAdjacency(input);
1999                 }
2000                 adjacencies.add(operationalAdjacency);
2001                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
2002                 VpnInterfaceOpDataEntry newVpnIntf =
2003                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
2004                                 aug, dpnId, currVpnIntf.getLportTag(),
2005                                 currVpnIntf.getGatewayMacAddress());
2006
2007                 writeOperTxn.merge(identifier, newVpnIntf, CREATE_MISSING_PARENTS);
2008             }
2009         } catch (ReadFailedException e) {
2010             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
2011                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
2012         }
2013     }
2014
2015     @Nullable
2016     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
2017         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
2018         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
2019     }
2020
2021     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
2022                                             BigInteger dpnId, TypedWriteTransaction<Operational> writeOperTxn,
2023                                             TypedWriteTransaction<Configuration> writeConfigTxn) {
2024         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
2025         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
2026         try {
2027             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
2028                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
2029             if (optVpnInterface.isPresent()) {
2030                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
2031                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
2032                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2033                         LogicalDatastoreType.OPERATIONAL, path);
2034                 if (optAdjacencies.isPresent()) {
2035                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
2036
2037                     if (adjacencies != null && !adjacencies.isEmpty()) {
2038                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
2039                         for (Adjacency adjacency : adjacencies) {
2040                             if (Objects.equals(adjacency.getIpAddress(), adj.getIpAddress())) {
2041                                 String rd = adjacency.getVrfId();
2042                                 if (adj.getNextHopIpList() != null) {
2043                                     for (String nh : adj.getNextHopIpList()) {
2044                                         deleteExtraRouteFromCurrentAndImportingVpns(
2045                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
2046                                                 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
2047                                     }
2048                                 } else if (adj.isPhysNetworkFunc()) {
2049                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
2050                                             adj.getIpAddress(), adj.getSubnetId());
2051                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
2052                                             writeConfigTxn);
2053                                 }
2054                                 break;
2055                             }
2056
2057                         }
2058                     }
2059                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
2060                             dpnId, adj.getVrfId());
2061                 } else {
2062                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
2063                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
2064                 }
2065             }
2066         } catch (ReadFailedException e) {
2067             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
2068                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
2069         }
2070     }
2071
2072     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
2073                                     String rd, String intfName, TypedWriteTransaction<Configuration> writeConfigTxn,
2074                                     TypedWriteTransaction<Operational> writeOperTx) {
2075         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
2076         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
2077         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
2078             String vpnRd = vpn.getVrfId();
2079             if (vpnRd != null) {
2080                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
2081                         writeOperTx);
2082             }
2083         }
2084     }
2085
2086     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
2087         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2088             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
2089             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
2090     }
2091
2092     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
2093         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2094             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
2095     }
2096
2097     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2098                                                 TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2099         if (vpnInterface == null) {
2100             return;
2101         }
2102         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
2103         if (adjs == null) {
2104             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2105                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2106             return;
2107         }
2108         for (Adjacency adj : adjs) {
2109             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2110                 String primaryInterfaceIp = adj.getIpAddress();
2111                 String macAddress = adj.getMacAddress();
2112                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2113
2114                 long label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
2115                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2116
2117                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2118                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2119                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
2120                         routerInt, label, writeConfigTxn);
2121                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2122                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2123                         macAddress);
2124             } else {
2125                 LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2126                                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}",
2127                         interfaceName, primaryRd, vpnName);
2128             }
2129         }
2130     }
2131
2132     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
2133             TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2134         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
2135         String rd = vpnUtil.getVpnRd(vpnName);
2136         if (adjs != null) {
2137             List<Adjacency> adjsList = requireNonNullElse(adjs.getAdjacency(), emptyList());
2138             for (Adjacency adj : adjsList) {
2139                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2140                     String primaryInterfaceIp = adj.getIpAddress();
2141                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2142                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
2143                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2144                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2145                     return;
2146                 }
2147             }
2148         } else {
2149             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2150                     vpnInterface.getName(), rd);
2151         }
2152     }
2153
2154     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2155         final VpnInterfaceKey key = intefaceData.identifier.firstKeyOf(VpnInterface.class);
2156         final String interfaceName = key.getName();
2157         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2158                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2159         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2160                   intefaceData.identifier, vpnName);
2161     }
2162
2163     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2164                                               VpnInterface vpnInterface, String vpnName) {
2165         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2166                .get(vpnName);
2167         if (vpnInterfaces == null) {
2168             vpnInterfaces = new ConcurrentLinkedQueue<>();
2169         }
2170         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2171         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2172         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2173                 vpnInterface.getName(), vpnName);
2174     }
2175
2176     public boolean isVpnInstanceReady(String vpnInstanceName) {
2177         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2178         if (vpnRd == null) {
2179             return false;
2180         }
2181         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2182
2183         return vpnInstanceOpDataEntry != null;
2184     }
2185
2186     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2187         synchronized (vpnInstanceName.intern()) {
2188             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2189                     unprocessedVpnInterfaces.get(vpnInstanceName);
2190             if (vpnInterfaces != null) {
2191                 while (!vpnInterfaces.isEmpty()) {
2192                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2193                     if (hasVpnInstanceCreatedSuccessfully) {
2194                         processSavedInterface(savedInterface, vpnInstanceName);
2195                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2196                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2197                     } else {
2198                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2199                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2200                     }
2201                 }
2202             } else {
2203                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2204             }
2205         }
2206     }
2207
2208     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2209             VpnInterface vpnInterface) {
2210         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2211             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2212                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2213             if (vpnInterfaces != null) {
2214                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2215                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2216                             + "unprocessed list", vpnInterface.getName(),
2217                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2218                 }
2219             } else {
2220                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2221                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2222             }
2223         }
2224     }
2225
2226     public void vpnInstanceIsReady(String vpnInstanceName) {
2227         processSavedInterfaces(vpnInstanceName, true);
2228     }
2229
2230     public void vpnInstanceFailed(String vpnInstanceName) {
2231         processSavedInterfaces(vpnInstanceName, false);
2232     }
2233
2234     private static class UnprocessedVpnInterfaceData {
2235         InstanceIdentifier<VpnInterface> identifier;
2236         VpnInterface vpnInterface;
2237
2238         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2239             this.identifier = identifier;
2240             this.vpnInterface = vpnInterface;
2241         }
2242
2243         @Override
2244         public int hashCode() {
2245             final int prime = 31;
2246             int result = 1;
2247             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2248             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2249             return result;
2250         }
2251
2252         @Override
2253         public boolean equals(Object obj) {
2254             if (this == obj) {
2255                 return true;
2256             }
2257             if (obj == null) {
2258                 return false;
2259             }
2260             if (getClass() != obj.getClass()) {
2261                 return false;
2262             }
2263             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2264             if (identifier == null) {
2265                 if (other.identifier != null) {
2266                     return false;
2267                 }
2268             } else if (!identifier.equals(other.identifier)) {
2269                 return false;
2270             }
2271             if (vpnInterface == null) {
2272                 if (other.vpnInterface != null) {
2273                     return false;
2274                 }
2275             } else if (!vpnInterface.equals(other.vpnInterface)) {
2276                 return false;
2277             }
2278             return true;
2279         }
2280     }
2281
2282     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2283         String primaryRd = vpnUtil.getVpnRd(vpnName);
2284         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2285         if (vpnInstanceOpData == null) {
2286             return;
2287         }
2288         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2289         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2290             return;
2291         }
2292         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2293         vpnToDpnLists.forEach(vpnToDpnList -> {
2294             if (vpnToDpnList.getVpnInterfaces() == null) {
2295                 return;
2296             }
2297             vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2298                 try {
2299                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2300                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2301                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2302                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2303                     if (!vpnInterfaceOptional.isPresent()) {
2304                         return;
2305                     }
2306                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2307                             vpnInterface.getInterfaceName());
2308                     if (configVpnAdjacencies == null) {
2309                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2310                         return;
2311                     }
2312                     List<Adjacency> operationVpnAdjacencies = requireNonNullElse(vpnInterfaceOptional.get()
2313                             .augmentation(AdjacenciesOp.class).getAdjacency(), emptyList());
2314                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2315                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2316                     // in operational DS. These unprocessed adjacencies will be handled below.
2317                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2318                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2319                     configVpnAdjacencies.stream()
2320                         .filter(adjacency -> operationVpnAdjacencies.stream()
2321                                 .noneMatch(operationalAdjacency ->
2322                                     Objects.equals(operationalAdjacency.getIpAddress(), adjacency.getIpAddress())))
2323                         .forEach(adjacency -> {
2324                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2325                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2326                                 () -> {
2327                                     // TODO Deal with sequencing — the config tx must only submitted
2328                                     // if the oper tx goes in
2329                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2330                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2331                                         futures.add(
2332                                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
2333                                                 //set of prefix used, as entry in prefix-to-interface datastore
2334                                                 // is prerequisite for refresh Fib to avoid race condition leading
2335                                                 // to missing remote next hop in bucket actions on bgp-vpn delete
2336                                                 Set<String> prefixListForRefreshFib = new HashSet<>();
2337                                                 ListenableFuture<Void> configTxFuture =
2338                                                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
2339                                                         confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2340                                                             primaryRd, adjacency, vpnInterfaceOptional.get().getDpnId(),
2341                                                                 operTx, confTx, confTx, prefixListForRefreshFib));
2342                                                 Futures.addCallback(configTxFuture,
2343                                                     new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
2344                                                     MoreExecutors.directExecutor());
2345                                                 futures.add(configTxFuture);
2346                                             }));
2347                                         return futures;
2348                                     } else {
2349                                         return emptyList();
2350                                     }
2351                                 });
2352                         });
2353                 } catch (ReadFailedException e) {
2354                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2355                             vpnName, primaryRd);
2356                 }
2357             });
2358         });
2359     }
2360
2361     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2362         private final String interfaceName;
2363         private final boolean add;
2364         private final String txnDestination;
2365
2366         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2367             this.interfaceName = interfaceName;
2368             this.add = add;
2369             this.txnDestination = transactionDest;
2370         }
2371
2372         @Override
2373         public void onSuccess(Void voidObj) {
2374             if (add) {
2375                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2376                         interfaceName, txnDestination);
2377             } else {
2378                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2379             }
2380         }
2381
2382         @Override
2383         public void onFailure(Throwable throwable) {
2384             if (add) {
2385                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2386                         interfaceName, txnDestination, throwable);
2387             } else {
2388                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2389                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2390             }
2391         }
2392     }
2393
2394     private class VpnInterfaceCallBackHandler implements FutureCallback<Void> {
2395         private final String primaryRd;
2396         private final Set<String> prefixListForRefreshFib;
2397
2398         VpnInterfaceCallBackHandler(String primaryRd, Set<String> prefixListForRefreshFib) {
2399             this.primaryRd = primaryRd;
2400             this.prefixListForRefreshFib = prefixListForRefreshFib;
2401         }
2402
2403         @Override
2404         public void onSuccess(Void voidObj) {
2405             prefixListForRefreshFib.forEach(prefix -> {
2406                 fibManager.refreshVrfEntry(primaryRd, prefix);
2407             });
2408         }
2409
2410         @Override
2411         public void onFailure(Throwable throwable) {
2412             LOG.debug("write Tx config operation failed {}", throwable);
2413         }
2414     }
2415 }