vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14 import static org.opendaylight.netvirt.vpnmanager.VpnUtil.requireNonNullElse;
  15
  16 import com.google.common.base.Optional;
  17 import com.google.common.base.Preconditions;
  18 import com.google.common.collect.Iterators;
  19 import com.google.common.util.concurrent.FutureCallback;
  20 import com.google.common.util.concurrent.Futures;
  21 import com.google.common.util.concurrent.ListenableFuture;
  22 import com.google.common.util.concurrent.MoreExecutors;
  23 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  24 import java.math.BigInteger;
  25 import java.util.ArrayList;
  26 import java.util.Collections;
  27 import java.util.List;
  28 import java.util.Map;
  29 import java.util.Objects;
  30 import java.util.concurrent.ConcurrentHashMap;
  31 import java.util.concurrent.ConcurrentLinkedQueue;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.function.Consumer;
  34 import java.util.function.Predicate;
  35 import java.util.stream.Collectors;
  36 import javax.annotation.Nullable;
  37 import javax.annotation.PostConstruct;
  38 import javax.annotation.PreDestroy;
  39 import javax.inject.Inject;
  40 import javax.inject.Singleton;
  41 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  42 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  43 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  44 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  45 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  46 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  47 import org.opendaylight.genius.infra.Datastore.Configuration;
  48 import org.opendaylight.genius.infra.Datastore.Operational;
  49 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  50 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  51 import org.opendaylight.genius.infra.TransactionAdapter;
  52 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  53 import org.opendaylight.genius.infra.TypedWriteTransaction;
  54 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  55 import org.opendaylight.genius.mdsalutil.NWUtil;
  56 import org.opendaylight.genius.mdsalutil.NwConstants;
  57 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  58 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  59 import org.opendaylight.infrautils.caches.CacheProvider;
  60 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  61 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  62 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  63 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  64 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  65 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  66 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  67 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  68 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  69 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  70 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  71 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  72 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  73 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  74 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  75 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  76 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  77 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  78 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 119 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 120 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 121 import org.slf4j.Logger;
 122 import org.slf4j.LoggerFactory;
 123
 124 @Singleton
 125 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 126
 127     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 128     private static final short DJC_MAX_RETRIES = 3;
 129
 130     private final DataBroker dataBroker;
 131     private final ManagedNewTransactionRunner txRunner;
 132     private final IBgpManager bgpManager;
 133     private final IFibManager fibManager;
 134     private final IMdsalApiManager mdsalManager;
 135     private final IdManagerService idManager;
 136     private final OdlInterfaceRpcService ifaceMgrRpcService;
 137     private final VpnFootprintService vpnFootprintService;
 138     private final IInterfaceManager interfaceManager;
 139     private final IVpnManager vpnManager;
 140     private final ArpResponderHandler arpResponderHandler;
 141     private final JobCoordinator jobCoordinator;
 142     private final VpnUtil vpnUtil;
 143
 144     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 145
 146     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 147             new ConcurrentHashMap<>();
 148
 149     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 150
 151     @Inject
 152     public VpnInterfaceManager(final DataBroker dataBroker,
 153                                final IBgpManager bgpManager,
 154                                final IdManagerService idManager,
 155                                final IMdsalApiManager mdsalManager,
 156                                final IFibManager fibManager,
 157                                final OdlInterfaceRpcService ifaceMgrRpcService,
 158                                final VpnFootprintService vpnFootprintService,
 159                                final IInterfaceManager interfaceManager,
 160                                final IVpnManager vpnManager,
 161                                final ArpResponderHandler arpResponderHandler,
 162                                final JobCoordinator jobCoordinator,
 163                                final CacheProvider cacheProvider,
 164                                final VpnUtil vpnUtil) {
 165         super(VpnInterface.class, VpnInterfaceManager.class);
 166
 167         this.dataBroker = dataBroker;
 168         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 169         this.bgpManager = bgpManager;
 170         this.idManager = idManager;
 171         this.mdsalManager = mdsalManager;
 172         this.fibManager = fibManager;
 173         this.ifaceMgrRpcService = ifaceMgrRpcService;
 174         this.vpnFootprintService = vpnFootprintService;
 175         this.interfaceManager = interfaceManager;
 176         this.vpnManager = vpnManager;
 177         this.arpResponderHandler = arpResponderHandler;
 178         this.jobCoordinator = jobCoordinator;
 179         this.vpnUtil = vpnUtil;
 180
 181         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 182                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 183                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 184     }
 185
 186     public Runnable isNotifyTaskQueued(String intfName) {
 187         return vpnIntfMap.remove(intfName);
 188     }
 189
 190     @PostConstruct
 191     public void start() {
 192         LOG.info("{} start", getClass().getSimpleName());
 193         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 194     }
 195
 196     @Override
 197     @PreDestroy
 198     public void close() {
 199         super.close();
 200         vpnInstanceOpDataEntryCache.close();
 201     }
 202
 203     @Override
 204     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 205         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 206     }
 207
 208     @Override
 209     protected VpnInterfaceManager getDataTreeChangeListener() {
 210         return VpnInterfaceManager.this;
 211     }
 212
 213     @Override
 214     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 215         LOG.trace("Received VpnInterface add event: vpnInterface={}", vpnInterface);
 216         LOG.info("add: intfName {} onto vpnName {}", vpnInterface.getName(),
 217                 VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 218         addVpnInterface(identifier, vpnInterface, null, null);
 219     }
 220
 221     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 222                           final VpnInterface vpnInterface, String vpnName) {
 223         synchronized (vpnName.intern()) {
 224             if (isVpnInstanceReady(vpnName)) {
 225                 return true;
 226             }
 227             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 228             return false;
 229         }
 230     }
 231
 232     // TODO Clean up the exception handling
 233     @SuppressWarnings("checkstyle:IllegalCatch")
 234     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 235                              final @Nullable List<Adjacency> oldAdjs, final @Nullable List<Adjacency> newAdjs) {
 236         for (VpnInstanceNames vpnInterfaceVpnInstance : requireNonNullElse(vpnInterface.getVpnInstanceNames(),
 237                 Collections.<VpnInstanceNames>emptyList())) {
 238             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 239             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 240         }
 241     }
 242
 243     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 244                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 245         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 246         final String interfaceName = key.getName();
 247
 248         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 249             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 250                   vpnName, vpnInterface.getName());
 251             return;
 252         }
 253         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 254              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 255         List<Adjacency> copyOldAdjs = null;
 256         if (oldAdjs != null) {
 257             copyOldAdjs = new ArrayList<>();
 258             copyOldAdjs.addAll(oldAdjs);
 259         }
 260         List<Adjacency> copyNewAdjs = null;
 261         if (newAdjs != null) {
 262             copyNewAdjs = new ArrayList<>();
 263             copyNewAdjs.addAll(newAdjs);
 264         }
 265         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 266     }
 267
 268     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 269                     final VpnInterface vpnInterface, final @Nullable List<Adjacency> oldAdjs,
 270                     final @Nullable List<Adjacency> newAdjs,
 271                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 272         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 273         final String interfaceName = key.getName();
 274         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 275         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 276             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 277             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 278             if (interfaceState != null) {
 279                 try {
 280                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 281                     final int ifIndex = interfaceState.getIfIndex();
 282                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 283                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 284                         // (the inventory tx goes in last)
 285                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 286                         ListenableFuture<Void> confFuture =
 287                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 288                                 confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
 289                                     operTx -> futures.add(
 290                                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, invTx -> {
 291                                             LOG.info(
 292                                                 "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 293                                                     + " on dpn {}",
 294                                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 295                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 296                                                 confTx, operTx, invTx, interfaceState, vpnName);
 297                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 298                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 299                                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 300                                                 if (newAdjs != null) {
 301                                                     for (Adjacency adj : newAdjs) {
 302                                                         if (oldAdjs.contains(adj)) {
 303                                                             oldAdjs.remove(adj);
 304                                                         } else {
 305                                                             if (!isBgpVpnInternetVpn
 306                                                                 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 307                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 308                                                                     primaryRd, adj, dpnId, operTx, confTx, invTx);
 309                                                             }
 310                                                         }
 311                                                     }
 312                                                 }
 313                                                 for (Adjacency adj : oldAdjs) {
 314                                                     if (!isBgpVpnInternetVpn
 315                                                         || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 316                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 317                                                             operTx, confTx);
 318                                                     }
 319                                                 }
 320                                             }
 321                                         })))));
 322                         futures.add(confFuture);
 323                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 324                             MoreExecutors.directExecutor());
 325                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 326                             + " processed successfully", interfaceName, vpnName, dpnId);
 327                         return futures;
 328                     });
 329                 } catch (NumberFormatException | IllegalStateException e) {
 330                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 331                             + "interface {}. Interface addition on vpn {} failed", interfaceName,
 332                         vpnName, e);
 333                     return;
 334                 }
 335             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 336                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 337                     () -> {
 338                         ListenableFuture<Void> future =
 339                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
 340                                 createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 341                                     confTx, vpnName);
 342                                 LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 343                                     vpnName, vpnInterface.getDpnId());
 344                             });
 345                         ListenableFutures.addErrorLogging(future, LOG,
 346                             "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 347                         return Collections.singletonList(future);
 348                     });
 349             } else {
 350                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 351                     + " is not available", interfaceName, vpnName);
 352             }
 353         } else {
 354             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 355                     + " as vpn is pending delete", interfaceName, vpnName,
 356                 vpnInterface.getDpnId());
 357         }
 358     }
 359
 360     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 361     // see comments below.
 362     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 363     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 364             final int lportTag, boolean isInterfaceUp,
 365             TypedWriteTransaction<Configuration> writeConfigTxn,
 366             TypedWriteTransaction<Operational> writeOperTxn,
 367             TypedReadWriteTransaction<Configuration> writeInvTxn,
 368             Interface interfaceState,
 369             final String vpnName) throws ExecutionException, InterruptedException {
 370         final String interfaceName = vpnInterface.getName();
 371         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 372                 vpnName);
 373         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 374         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 375         if (!isInterfaceUp) {
 376             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 377                      interfaceName, dpId, vpnName);
 378             long vpnId = vpnUtil.getVpnId(vpnName);
 379             if (vpnId == VpnConstants.INVALID_ID) {
 380                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 381                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 382                         dpId);
 383                 return;
 384             }
 385
 386             boolean waitForVpnInterfaceOpRemoval = false;
 387             if (opVpnInterface != null) {
 388                 String opVpnName = opVpnInterface.getVpnInstanceName();
 389                 String primaryInterfaceIp = null;
 390                 if (Objects.equals(opVpnName, vpnName)) {
 391                     // Please check if the primary VRF Entry does not exist for VPNInterface
 392                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 393                     // back to back
 394                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 395                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 396                     if (adjs == null) {
 397                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 398                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 399                                 vpnName);
 400                         return;
 401                     }
 402                     for (Adjacency adj : adjs) {
 403                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 404                             primaryInterfaceIp = adj.getIpAddress();
 405                             break;
 406                         }
 407                     }
 408                     if (primaryInterfaceIp == null) {
 409                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 410                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 411                                 dpId, vpnName);
 412                         return;
 413                     }
 414                     // Get the rd of the vpn instance
 415                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 416                     if (vrf != null) {
 417                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 418                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 419                         return;
 420                     }
 421                     waitForVpnInterfaceOpRemoval = true;
 422                 } else {
 423                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 424                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 425                 }
 426             }
 427             if (!waitForVpnInterfaceOpRemoval) {
 428                 // Add the VPNInterface and quit
 429                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 430                         null/*ipAddressSourceValuePair*/,
 431                         true /* add */);
 432                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 433                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 434                 if (!isBgpVpnInternetVpn) {
 435                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 436                 }
 437                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 438                         dpId, vpnName);
 439                 if (interfaceManager.isExternalInterface(interfaceName)) {
 440                     processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
 441                         NwConstants.ADD_FLOW);
 442                 }
 443                 return;
 444             }
 445
 446             // FIB didn't get a chance yet to clean up this VPNInterface
 447             // Let us give it a chance here !
 448             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 449                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 450             try {
 451                 Runnable notifyTask = new VpnNotifyTask();
 452                 synchronized (notifyTask) {
 453                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 454                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 455                     // published here while holding the lock on it so this path will hit the wait before notify can be
 456                     // invoked.
 457                     vpnIntfMap.put(interfaceName, notifyTask);
 458                     try {
 459                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 460                     } catch (InterruptedException e) {
 461                         // Ignored
 462                     }
 463                 }
 464             } finally {
 465                 vpnIntfMap.remove(interfaceName);
 466             }
 467
 468             if (opVpnInterface != null) {
 469                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 470                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 471                         dpId, vpnName);
 472                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 473                 return;
 474             }
 475             // VPNInterface got removed, proceed with Add
 476             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 477                     interfaceName, dpId, vpnName);
 478             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 479                     null/*ipAddressSourceValuePair*/,
 480                     true /* add */);
 481             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 482                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 483             if (!isBgpVpnInternetVpn) {
 484                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 485             }
 486             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 487                     + " FIB to clean up", interfaceName, dpId, vpnName);
 488             if (interfaceManager.isExternalInterface(interfaceName)) {
 489                 processExternalVpnInterface(interfaceName, vpnName, dpId,
 490                                lportTag, NwConstants.ADD_FLOW);
 491             }
 492         } else {
 493             try {
 494                 // Interface is retained in the DPN, but its Link Up.
 495                 // Advertise prefixes again for this interface to BGP
 496                 InstanceIdentifier<VpnInterface> identifier =
 497                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 498                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 499                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 500                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 501                 // Perform similar operation as interface add event for extraroutes.
 502                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 503                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 504                         LogicalDatastoreType.CONFIGURATION, path);
 505                 if (!optAdjacencies.isPresent()) {
 506                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 507                     return;
 508                 }
 509                 List<Adjacency> adjacencies = requireNonNullElse(optAdjacencies.get().getAdjacency(), emptyList());
 510                 for (Adjacency adjacency : adjacencies) {
 511                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 512                         continue;
 513                     }
 514                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 515                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 516                         continue;
 517                     }
 518                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 519                             dpId, writeOperTxn, writeConfigTxn, writeInvTxn);
 520                 }
 521             } catch (ReadFailedException e) {
 522                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 523                         interfaceName, vpnName, primaryRd, dpId);
 524             }
 525         }
 526     }
 527
 528     private void processExternalVpnInterface(String interfaceName, String vpnName, BigInteger dpId,
 529         int lportTag, int addOrRemove) {
 530         Uuid extNetworkId;
 531         try {
 532             // vpn instance of ext-net interface is the network-id
 533             extNetworkId = new Uuid(vpnName);
 534         } catch (IllegalArgumentException e) {
 535             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 536                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 537             return;
 538         }
 539
 540         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 541         if (routerIds == null || routerIds.isEmpty()) {
 542             LOG.info("processExternalVpnInterface: No router is associated with {}."
 543                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 544                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 545             return;
 546         }
 547
 548         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 549                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 550         for (Uuid routerId : routerIds) {
 551             String routerName = routerId.getValue();
 552             BigInteger primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 553             if (Objects.equals(primarySwitch, dpId)) {
 554                 Routers router = vpnUtil.getExternalRouter(routerName);
 555                 if (router != null) {
 556                     if (addOrRemove == NwConstants.ADD_FLOW) {
 557                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 558                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 559                                 dpId, interfaceName, lportTag);
 560                     } else {
 561                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 562                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 563                                 dpId, interfaceName, lportTag);
 564                     }
 565                 } else {
 566                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 567                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 568                             interfaceName, dpId, vpnName);
 569                 }
 570             }
 571         }
 572     }
 573
 574     // TODO Clean up the exception handling
 575     @SuppressWarnings("checkstyle:IllegalCatch")
 576     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 577                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 578                                                   String vpnName, String interfaceName) {
 579         if (rd == null) {
 580             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 581                   interfaceName, dpnId, vpnName);
 582             return;
 583         } else {
 584             if (rd.equals(vpnName)) {
 585                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 586                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 587                 return;
 588             }
 589         }
 590         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 591                 interfaceName, dpnId, vpnName, rd);
 592
 593         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 594         if (nextHopIp == null) {
 595             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 596                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 597                     rd, vpnName);
 598             return;
 599         }
 600
 601         try {
 602             //Read NextHops
 603             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 604             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 605                     LogicalDatastoreType.OPERATIONAL, path);
 606             if (adjacencies.isPresent()) {
 607                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 608                 if (nextHops != null && !nextHops.isEmpty()) {
 609                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 610                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 611                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 612                     long l3vni = vpnInstanceOpData.getL3vni();
 613                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 614                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 615                     for (Adjacency nextHop : nextHops) {
 616                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 617                             continue;
 618                         }
 619                         String gatewayMac = null;
 620                         long label = 0;
 621                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 622                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 623                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 624                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 625                                     .get();
 626                         } else {
 627                             label = nextHop.getLabel();
 628                         }
 629                         try {
 630                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 631                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 632                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 633                                     encapType, (int)label, l3vni, 0 /*l2vni*/,
 634                                     gatewayMac);
 635                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 636                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 637                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 638                         } catch (Exception e) {
 639                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 640                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 641                                     interfaceName, dpnId, e);
 642                         }
 643                     }
 644                 }
 645             }
 646         } catch (ReadFailedException e) {
 647             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 648                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 649         }
 650     }
 651
 652     // TODO Clean up the exception handling
 653     @SuppressWarnings("checkstyle:IllegalCatch")
 654     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 655                         String vpnName, String interfaceName, TypedWriteTransaction<Configuration> writeConfigTxn,
 656                         TypedWriteTransaction<Operational> writeOperTx) {
 657         //Read NextHops
 658         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 659         String rd = vpnUtil.getVpnRd(interfaceName);
 660         if (rd == null) {
 661             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 662                 interfaceName, vpnName);
 663             return;
 664         } else {
 665             if (rd.equals(vpnName)) {
 666                 LOG.info(
 667                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 668                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 669                 return;
 670             }
 671         }
 672         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 673                vpnName, rd);
 674         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 675         try {
 676             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 677                     path);
 678         } catch (ReadFailedException e) {
 679             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 680                     interfaceName, vpnName);
 681         }
 682         if (adjacencies.isPresent()) {
 683             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 684
 685             if (nextHops != null && !nextHops.isEmpty()) {
 686                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 687                         nextHops, interfaceName, vpnName, rd);
 688                 for (Adjacency nextHop : nextHops) {
 689                     try {
 690                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 691                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 692                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 693                                     interfaceName, vpnName);
 694                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 695                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 696                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 697                                     interfaceName, vpnName);
 698                         } else {
 699                             // Perform similar operation as interface delete event for extraroutes.
 700                             String allocatedRd = nextHop.getVrfId();
 701                             for (String nh : requireNonNullElse(nextHop.getNextHopIpList(),
 702                                     Collections.<String>emptyList())) {
 703                                 deleteExtraRouteFromCurrentAndImportingVpns(
 704                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
 705                                     writeOperTx);
 706                             }
 707                         }
 708                     } catch (Exception e) {
 709                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 710                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 711                     }
 712                 }
 713             }
 714         }
 715     }
 716
 717     @SuppressWarnings("checkstyle:IllegalCatch")
 718     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 719                                                   String primaryRd, String interfaceName, final long vpnId,
 720                                                   TypedWriteTransaction<Configuration> writeConfigTxn,
 721                                                   TypedWriteTransaction<Operational> writeOperTxn,
 722                                                   TypedReadWriteTransaction<Configuration> writeInvTxn,
 723                                                   Interface interfaceState)
 724             throws ExecutionException, InterruptedException {
 725         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 726         // Read NextHops
 727         Optional<VpnInterface> vpnInteface = Optional.absent();
 728         try {
 729             vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 730             LogicalDatastoreType.CONFIGURATION, identifier);
 731         } catch (ReadFailedException e) {
 732             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 733                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 734         }
 735         Uuid intfnetworkUuid = null;
 736         NetworkType networkType = null;
 737         Long segmentationId = Long.valueOf(-1);
 738         Adjacencies adjacencies = null;
 739         if (vpnInteface.isPresent()) {
 740             intfnetworkUuid = vpnInteface.get().getNetworkId();
 741             networkType = vpnInteface.get().getNetworkType();
 742             segmentationId = vpnInteface.get().getSegmentationId();
 743             adjacencies = vpnInteface.get().augmentation(Adjacencies.class);
 744             if (adjacencies == null) {
 745                 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 746                         null/*gwMac*/, writeOperTxn);
 747                 return;
 748             }
 749         }
 750         // Get the rd of the vpn instance
 751         String nextHopIp = null;
 752         try {
 753             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 754         } catch (Exception e) {
 755             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 756                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 757         }
 758         List<String> nhList = new ArrayList<>();
 759         if (nextHopIp != null) {
 760             nhList.add(nextHopIp);
 761             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 762                     interfaceName, dpnId, vpnName, nhList);
 763         }
 764         Optional<String> gwMac = Optional.absent();
 765         String vpnInterfaceSubnetGwMacAddress = null;
 766         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 767         Long l3vni = vpnInstanceOpData.getL3vni();
 768         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 769         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 770         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 771         List<Adjacency> nextHops = (adjacencies != null) ? adjacencies.getAdjacency() : emptyList();
 772         List<Adjacency> value = new ArrayList<>();
 773         for (Adjacency nextHop : nextHops) {
 774             String rd = primaryRd;
 775             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 776             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 777                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 778                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 779                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 780                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 781                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 782                 continue;
 783             }
 784             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 785                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 786                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 787                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 788                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 789                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 790
 791                 Prefixes prefixes = (intfnetworkUuid != null)
 792                     ? VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, intfnetworkUuid ,networkType,
 793                             segmentationId, prefixCue) :
 794                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, prefixCue);
 795                 writeOperTxn.merge(VpnUtil.getPrefixToInterfaceIdentifier(
 796                         vpnUtil.getVpnId(vpnName), prefix), prefixes, true);
 797                 final Uuid subnetId = nextHop.getSubnetId();
 798
 799                 String gatewayIp = nextHop.getSubnetGatewayIp();
 800                 if (gatewayIp == null) {
 801                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 802                     if (gatewayIpOptional.isPresent()) {
 803                         gatewayIp = gatewayIpOptional.get();
 804                     }
 805                 }
 806
 807                 if (gatewayIp != null) {
 808                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 809                     if (gwMac.isPresent()) {
 810                         // A valid mac-address is available for this subnet-gateway-ip
 811                         // Use this for programming ARP_RESPONDER table here.  And save this
 812                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 813                         // to populate L3_GW_MAC_TABLE there.
 814                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 815                                 gatewayIp, gwMac.get());
 816                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 817                     } else {
 818                         // A valid mac-address is not available for this subnet-gateway-ip
 819                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 820                         // Save this connected-mac-address as gateway-mac-address for the
 821                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 822                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 823                         if (gwMac.isPresent()) {
 824                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 825                                     NwConstants.ADD_FLOW, gwMac.get());
 826                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 827                                     gatewayIp, gwMac.get());
 828                         } else {
 829                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 830                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 831                                 + "gwIp {}",
 832                                 subnetId, interfaceName, vpnName, gatewayIp);
 833                         }
 834                     }
 835                 } else {
 836                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 837                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 838                         subnetId, interfaceName, vpnName);
 839                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 840                 }
 841                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 842                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 843             } else {
 844                 //Extra route adjacency
 845                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 846                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 847                 synchronized (vpnPrefixKey.intern()) {
 848                     java.util.Optional<String> rdToAllocate = vpnUtil
 849                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 850                                     nextHop.getNextHopIpList().get(0), dpnId);
 851                     if (rdToAllocate.isPresent()) {
 852                         rd = rdToAllocate.get();
 853                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 854                             rd, prefix);
 855                     } else {
 856                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 857                         continue;
 858                     }
 859                 }
 860                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 861                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 862                         interfaceName, dpnId);
 863             }
 864             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 865             // can be different from the gateway-mac-address within the VRFEntry as the
 866             // gateway-mac-address is a superset.
 867             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 868                     : RouteOrigin.STATIC;
 869             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 870                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 871                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 872             Adjacency operationalAdjacency = null;
 873             try {
 874                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 875             } catch (NullPointerException e) {
 876                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 877                     input, e.getMessage());
 878                 return;
 879             }
 880             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 881                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 882                         vpnName, l3vni, origin,
 883                         interfaceName, operationalAdjacency, encapType, writeConfigTxn);
 884             }
 885             value.add(operationalAdjacency);
 886         }
 887
 888         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 889         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 890                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 891
 892         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 893                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 894                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 895
 896         for (Adjacency nextHop : aug.getAdjacency()) {
 897             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 898             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 899                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 900                         : RouteOrigin.STATIC;
 901                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 902                 registeredPopulator.populateFib(input, writeConfigTxn);
 903             }
 904         }
 905     }
 906
 907     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 908                                               long lportTag, String gwMac,
 909                                               TypedWriteTransaction<Operational> writeOperTxn) {
 910         VpnInterfaceOpDataEntry opInterface =
 911               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac);
 912         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 913             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 914         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
 915         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 916                 interfaceName, dpnId, vpnName);
 917     }
 918
 919     // TODO Clean up the exception handling
 920     @SuppressWarnings("checkstyle:IllegalCatch")
 921     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 922                                            StateTunnelList stateTunnelList,
 923                                            TypedWriteTransaction<Configuration> writeConfigTxn,
 924                                            TypedWriteTransaction<Operational> writeOperTxn) {
 925
 926         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
 927         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 928         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 929         List<Adjacency> adjList =
 930             adjacencies != null && adjacencies.getAdjacency() != null ? adjacencies.getAdjacency() : emptyList();
 931         if (adjList.isEmpty()) {
 932             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 933                     vpnInterface, srcDpnId);
 934             return;
 935         }
 936         String prefix = null;
 937         long label = 0;
 938         List<Adjacency> value = new ArrayList<>();
 939         boolean isNextHopAddReqd = false;
 940         String vpnName = vpnInterface.getVpnInstanceName();
 941         long vpnId = vpnUtil.getVpnId(vpnName);
 942         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 943         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 944                 vpnInterface.getName(), vpnInterface.getDpnId(),
 945                 vpnInterface.getVpnInstanceName(), adjList);
 946         for (Adjacency adj : adjList) {
 947             String rd = adj.getVrfId();
 948             rd = rd != null ? rd : vpnName;
 949             prefix = adj.getIpAddress();
 950             label = adj.getLabel();
 951             List<String> nhList = Collections.singletonList(srcTepIp);
 952             List<String> nextHopList = adj.getNextHopIpList();
 953             // If TEP is added , update the nexthop of primary adjacency.
 954             // Secondary adj nexthop is already pointing to primary adj IP address.
 955             if (nextHopList == null || nextHopList.isEmpty()) {
 956                 isNextHopAddReqd = true;
 957             }
 958
 959             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 960                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 961             } else {
 962                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 963                 if (!vrfEntryOptional.isPresent()) {
 964                     continue;
 965                 }
 966                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 967                 if (!nhList.contains(srcTepIp)) {
 968                     nhList.add(srcTepIp);
 969                     isNextHopAddReqd = true;
 970                 }
 971                 value.add(adj);
 972             }
 973
 974             if (isNextHopAddReqd) {
 975                 updateLabelMapper(label, nhList);
 976                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 977                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 978                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 979                 // Update the VRF entry with nextHop
 980                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 981                         label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
 982
 983                 //Get the list of VPN's importing this route(prefix) .
 984                 // Then update the VRF entry with nhList
 985                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
 986                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 987                     String vpnRd = vpn.getVrfId();
 988                     if (vpnRd != null) {
 989                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
 990                             srcTepIp, label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
 991                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 992                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
 993                             vpnInterface.getName(), srcDpnId, vpnName,
 994                             vpn.getVpnInstanceName(), vpnRd);
 995                     }
 996                 }
 997                 // Advertise the prefix to BGP only for external vpn
 998                 // since there is a nexthop change.
 999                 try {
1000                     if (!rd.equalsIgnoreCase(vpnName)) {
1001                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
1002                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
1003                                 null /*gatewayMacAddress*/);
1004                     }
1005                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
1006                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
1007                             srcDpnId, vpnName);
1008                 } catch (Exception ex) {
1009                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
1010                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1011                             vpnInterface.getName(), srcDpnId, vpnName, ex);
1012                 }
1013             }
1014         }
1015         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1016         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1017                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1018                 .addAugmentation(AdjacenciesOp.class, aug).build();
1019         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1020                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1021         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1022         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
1023                 vpnInterface.getName(), srcDpnId, vpnName);
1024
1025     }
1026
1027     // TODO Clean up the exception handling
1028     @SuppressWarnings("checkstyle:IllegalCatch")
1029     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1030                                               StateTunnelList stateTunnelList,
1031                                               TypedWriteTransaction<Configuration> writeConfigTxn,
1032                                               TypedWriteTransaction<Operational> writeOperTxn) {
1033
1034         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1035         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1036         String prefix = null;
1037         long label = 0;
1038         boolean isNextHopRemoveReqd = false;
1039         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
1040         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
1041         String vpnName = vpnInterface.getVpnInstanceName();
1042         long vpnId = vpnUtil.getVpnId(vpnName);
1043         String primaryRd = vpnUtil.getVpnRd(vpnName);
1044         if (adjList != null) {
1045             List<Adjacency> value = new ArrayList<>();
1046             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1047                     vpnInterface.getName(), vpnInterface.getDpnId(),
1048                     vpnInterface.getVpnInstanceName(), adjList);
1049             for (Adjacency adj : adjList) {
1050                 List<String> nhList = new ArrayList<>();
1051                 String rd = adj.getVrfId();
1052                 rd = rd != null ? rd : vpnName;
1053                 prefix = adj.getIpAddress();
1054                 List<String> nextHopList = adj.getNextHopIpList();
1055                 label = adj.getLabel();
1056                 if (nextHopList != null && !nextHopList.isEmpty()) {
1057                     isNextHopRemoveReqd = true;
1058                 }
1059                 // If TEP is deleted , remove the nexthop from primary adjacency.
1060                 // Secondary adj nexthop will continue to point to primary adj IP address.
1061                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1062                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1063                 } else {
1064                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1065                     if (!vrfEntryOptional.isPresent()) {
1066                         continue;
1067                     }
1068                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1069                     if (nhList.contains(srcTepIp)) {
1070                         nhList.remove(srcTepIp);
1071                         isNextHopRemoveReqd = true;
1072                     }
1073                     value.add(adj);
1074                 }
1075
1076                 if (isNextHopRemoveReqd) {
1077                     updateLabelMapper(label, nhList);
1078                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1079                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1080                             prefix, nhList, vpnName,
1081                             vpnId, rd, vpnInterface.getName());
1082                     // Update the VRF entry with removed nextHop
1083                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1084                             label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1085
1086                     //Get the list of VPN's importing this route(prefix) .
1087                     // Then update the VRF entry with nhList
1088                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1089                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1090                         String vpnRd = vpn.getVrfId();
1091                         if (vpnRd != null) {
1092                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1093                                 srcTepIp, label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1094                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1095                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1096                                     nhList, label, vpnInterface.getName(), srcDpnId,
1097                                     vpnName,
1098                                     vpn.getVpnInstanceName(), vpnRd);
1099                         }
1100                     }
1101
1102                     // Withdraw prefix from BGP only for external vpn.
1103                     try {
1104                         if (!rd.equalsIgnoreCase(vpnName)) {
1105                             bgpManager.withdrawPrefix(rd, prefix);
1106                         }
1107                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1108                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1109                                 vpnInterface.getName(), srcDpnId,
1110                                 vpnName);
1111                     } catch (Exception ex) {
1112                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1113                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1114                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1115                     }
1116                 }
1117             }
1118             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1119             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1120                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1121                     .addAugmentation(AdjacenciesOp.class, aug).build();
1122             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1123                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1124             writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1125             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1126                          vpnInterface.getName(), srcDpnId, vpnName);
1127         }
1128     }
1129
1130     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1131         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1132
1133         String vpnRd = vpnUtil.getVpnRd(vpnName);
1134         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1135         if (vpnInstanceOpDataEntry == null) {
1136             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1137                     + " to check for vpns exporting the routes", vpnName);
1138             return vpnsToExportRoute;
1139         }
1140
1141         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1142             if (input.getVpnInstanceName() == null) {
1143                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1144                         input.getVrfId());
1145                 return false;
1146             }
1147             return !input.getVpnInstanceName().equals(vpnName);
1148         };
1149
1150         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1151             Iterable<String> commonRTs =
1152                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1153                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1154             return Iterators.size(commonRTs.iterator()) > 0;
1155         };
1156
1157         vpnsToExportRoute =
1158                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1159                         Collectors.toList());
1160         return vpnsToExportRoute;
1161     }
1162
1163     // TODO Clean up the exception handling
1164     @SuppressWarnings("checkstyle:IllegalCatch")
1165     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1166         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1167         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1168             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1169             if (vrfEntries != null) {
1170                 ListenableFutures.addErrorLogging(
1171                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1172                         for (VrfEntry vrfEntry : vrfEntries) {
1173                             try {
1174                                 if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1175                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1176                                     LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1177                                             + " is not a controller managed non intervpn link route. Ignoring.",
1178                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1179                                     continue;
1180                                 }
1181                                 String prefix = vrfEntry.getDestPrefix();
1182                                 String gwMac = vrfEntry.getGatewayMacAddress();
1183                                 requireNonNullElse(vrfEntry.getRoutePaths(),
1184                                     Collections.<RoutePaths>emptyList()).forEach(routePath -> {
1185                                         String nh = routePath.getNexthopAddress();
1186                                         int label = routePath.getLabel().intValue();
1187                                         if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1188                                             vrfEntry.getOrigin()))) {
1189                                             LOG.info(
1190                                                 "handleVpnsExportingRoutesImporting: Importing fib entry rd {}"
1191                                                     + " prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1192                                                 vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1193                                             fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1194                                                 Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1195                                                 0 /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1196                                                 confTx);
1197                                         } else {
1198                                             LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry"
1199                                                     + " rd {} prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1200                                                 vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1201                                             SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1202                                             importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1203                                                 confTx);
1204                                         }
1205                                     });
1206                             } catch (RuntimeException e) {
1207                                 LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1208                                         + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1209                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1210                             }
1211                         }
1212                     }), LOG, "Error handing VPN exporting routes");
1213             } else {
1214                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1215                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1216             }
1217         }
1218     }
1219
1220     @Override
1221     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1222         LOG.trace("Received VpnInterface remove event: vpnInterface={}", vpnInterface);
1223         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1224         final String interfaceName = key.getName();
1225         for (VpnInstanceNames vpnInterfaceVpnInstance : requireNonNullElse(vpnInterface.getVpnInstanceNames(),
1226                 Collections.<VpnInstanceNames>emptyList())) {
1227             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1228             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1229         }
1230     }
1231
1232     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1233                                 final VpnInterface vpnInterface, final String vpnName,
1234                                 final String interfaceName) {
1235         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1236             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1237                 ListenableFuture<Void> future =
1238                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1239                         deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1240                         LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1241                     });
1242                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1243                         vpnInterface.getName(), vpnName);
1244                 return Collections.singletonList(future);
1245             }, DJC_MAX_RETRIES);
1246         } else {
1247             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1248             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1249         }
1250     }
1251
1252     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1253     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1254                                            final VpnInterface vpnInterface, final String vpnName,
1255                                            final String interfaceName, final Interface interfaceState) {
1256         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1257                 vpnName, vpnInterface.getDpnId());
1258         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1259         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1260             () -> {
1261                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1262                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1263                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1264                         writeOperTxn -> futures.add(
1265                             txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeInvTxn -> {
1266                                 LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1267                                         interfaceName, vpnName);
1268                                 BigInteger dpId;
1269                                 int ifIndex;
1270                                 String gwMacAddress;
1271                                 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1272                                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1273                                 Optional<VpnInterfaceOpDataEntry> optVpnInterface;
1274                                 try {
1275                                     optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1276                                             LogicalDatastoreType.OPERATIONAL, interfaceId);
1277                                 } catch (ReadFailedException e) {
1278                                     LOG.error("remove: Failed to read data store for interface {} vpn {}",
1279                                             interfaceName, vpnName);
1280                                     return;
1281                                 }
1282                                 if (interfaceState != null) {
1283                                     try {
1284                                         dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1285                                     } catch (NumberFormatException | IllegalStateException e) {
1286                                         LOG.error("remove: Unable to retrieve dpnId from interface operational"
1287                                                         + " data store for interface {} on dpn {} for vpn {} Fetching"
1288                                                         + " from vpn interface op data store. ", interfaceName,
1289                                                 vpnInterface.getDpnId(), vpnName, e);
1290                                         dpId = BigInteger.ZERO;
1291                                     }
1292                                     ifIndex = interfaceState.getIfIndex();
1293                                     gwMacAddress = interfaceState.getPhysAddress().getValue();
1294                                 } else {
1295                                     LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1296                                             + " from vpn interface op.", interfaceName);
1297                                     if (optVpnInterface.isPresent()) {
1298                                         VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1299                                         dpId = vpnOpInterface.getDpnId();
1300                                         ifIndex = vpnOpInterface.getLportTag().intValue();
1301                                         gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1302                                     } else {
1303                                         LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1304                                                 + " as interfaceState and vpn interface op is not"
1305                                                 + " available", interfaceName, vpnName);
1306                                         return;
1307                                     }
1308                                 }
1309                                 processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1310                                         optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1311                                         writeConfigTxn, writeOperTxn, writeInvTxn);
1312                                 LOG.info(
1313                                         "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1314                                                 + "successfully",
1315                                         interfaceName, vpnInterface.getDpnId(), vpnName);
1316                             })))));
1317                 futures.add(configFuture);
1318                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1319                 return futures;
1320             }, DJC_MAX_RETRIES);
1321     }
1322
1323     protected void processVpnInterfaceDown(BigInteger dpId,
1324                                            String interfaceName,
1325                                            int lportTag,
1326                                            String gwMac,
1327                                            VpnInterfaceOpDataEntry vpnOpInterface,
1328                                            boolean isInterfaceStateDown,
1329                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1330                                            TypedWriteTransaction<Operational> writeOperTxn,
1331                                            TypedReadWriteTransaction<Configuration> writeInvTxn)
1332             throws ExecutionException, InterruptedException {
1333         if (vpnOpInterface == null) {
1334             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1335                     + " as it is not available in operational data store", interfaceName, dpId);
1336             return;
1337         }
1338         final String vpnName = vpnOpInterface.getVpnInstanceName();
1339         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1340                                                     interfaceName, vpnName);
1341         if (!isInterfaceStateDown) {
1342             final long vpnId = vpnUtil.getVpnId(vpnName);
1343             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName,  null);
1344             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1345             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1346                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1347             if (interfaceManager.isExternalInterface(interfaceName)) {
1348                 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1349                     NwConstants.DEL_FLOW);
1350             }
1351             if (!isBgpVpnInternetVpn) {
1352                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1353             }
1354             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1355                     + " successful", interfaceName, dpId, vpnName);
1356         } else {
1357             // Interface is retained in the DPN, but its Link Down.
1358             // Only withdraw the prefixes for this interface from BGP
1359             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1360         }
1361     }
1362
1363     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1364                                           final String vpnName, final long vpnId, String gwMac,
1365                                           TypedWriteTransaction<Configuration> writeConfigTxn,
1366                                           TypedWriteTransaction<Operational> writeOperTxn,
1367                                           TypedReadWriteTransaction<Configuration> writeInvTxn)
1368             throws ExecutionException, InterruptedException {
1369         //Read NextHops
1370         try {
1371             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1372                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1373             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1374             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1375                     LogicalDatastoreType.OPERATIONAL, path);
1376             String primaryRd = vpnUtil.getVpnRd(vpnName);
1377             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1378                     interfaceName, dpnId, vpnName, primaryRd);
1379             if (adjacencies.isPresent() && adjacencies.get().getAdjacency() != null
1380                     && !adjacencies.get().getAdjacency().isEmpty()) {
1381                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1382                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1383                         interfaceName, dpnId, vpnName, nextHops);
1384                 for (Adjacency nextHop : nextHops) {
1385                     if (nextHop.isPhysNetworkFunc()) {
1386                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1387                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1388                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1389                                 null/*writeCfgTxn*/);
1390                     } else {
1391                         String rd = nextHop.getVrfId();
1392                         List<String> nhList;
1393                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1394                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1395                         } else {
1396                             // This is a primary adjacency
1397                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1398                                     : emptyList();
1399                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1400                                     interfaceName, writeInvTxn);
1401                         }
1402                         if (!nhList.isEmpty()) {
1403                             if (Objects.equals(rd, vpnName)) {
1404                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1405                                 //remove from FIB directly
1406                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1407                                         interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1408                             } else {
1409                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1410                                         interfaceName, writeConfigTxn, writeOperTxn);
1411                             }
1412                         } else {
1413                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1414                                             + " interface {}", nextHop.getIpAddress(), rd,
1415                                     nextHop.getAdjacencyType().toString(), interfaceName);
1416                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1417                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1418                         }
1419                     }
1420                     String ip = nextHop.getIpAddress().split("/")[0];
1421                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1422                     if (vpnVipToPort != null) {
1423                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1424                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1425                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1426                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1427                     }
1428                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1429                     if (vpnPortipToPort != null) {
1430                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1431                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1432                                  + "Interface {} ip {} on dpn {} for vpn {}",
1433                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1434                     }
1435                 }
1436             } else {
1437                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1438                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1439                         + " Removing it.", interfaceName, vpnName, dpnId);
1440                 writeOperTxn.delete(identifier);
1441             }
1442         } catch (ReadFailedException e) {
1443             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1444                     interfaceName, dpnId, vpnName);
1445         }
1446     }
1447
1448     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1449                                                             String interfaceName, BigInteger dpnId,
1450                                                             TypedWriteTransaction<Configuration> writeConfigTxn,
1451                                                             TypedWriteTransaction<Operational> writeOperTx) {
1452         return (nh) -> {
1453             String primaryRd = vpnUtil.getVpnRd(vpnName);
1454             String prefix = nextHop.getIpAddress();
1455             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1456             LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1457                     nextHop, vpnName, interfaceName, dpnId);
1458             synchronized (vpnNamePrefixKey.intern()) {
1459                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1460                         prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1461                     //If extra-route is present behind at least one VM, then do not remove or update
1462                     //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1463                     //prefixtointerface DS
1464                     return;
1465                 }
1466                 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1467                         writeConfigTxn);
1468             }
1469             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1470                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1471                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1472         };
1473     }
1474
1475     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1476                                            BigInteger dpnId, String rd, String interfaceName,
1477                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1478                                            TypedWriteTransaction<Operational> writeOperTx) {
1479         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1480                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1481         nhList.forEach((nh) -> {
1482             //IRT: remove routes from other vpns importing it
1483             vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1484                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1485             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1486                 String vpnRd = vpn.getVrfId();
1487                 if (vpnRd != null) {
1488                     fibManager.removeOrUpdateFibEntry(vpnRd,
1489                             nextHop.getIpAddress(), nh, writeConfigTxn);
1490                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1491                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1492                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1493                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1494                 }
1495             }
1496         });
1497     }
1498
1499     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1500                                                  int lportTag, String gwMac, String interfaceName,
1501                                                  TypedReadWriteTransaction<Configuration> writeInvTxn)
1502             throws ExecutionException, InterruptedException {
1503         final Uuid subnetId = nextHop.getSubnetId();
1504         if (nextHop.getSubnetGatewayMacAddress() == null) {
1505             // A valid mac-address was not available for this subnet-gateway-ip
1506             // So a connected-mac-address was used for this subnet and we need
1507             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1508             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1509         }
1510         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1511                 subnetId);
1512     }
1513
1514     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1515                                                   String interfaceName) {
1516         // This is either an extra-route (or) a learned IP via subnet-route
1517         List<String> nhList = null;
1518         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1519         if (nextHopIp == null || nextHopIp.isEmpty()) {
1520             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1521                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1522                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1523                     vpnName);
1524             nhList = emptyList();
1525         } else {
1526             nhList = Collections.singletonList(nextHopIp);
1527         }
1528         return nhList;
1529     }
1530
1531     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1532         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1533         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1534         // else use connected interface
1535         if (gwPort != null && gwPort.isSubnetIp()) {
1536             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1537                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1538             return Optional.of(gwPort.getMacAddress());
1539         }
1540         return Optional.absent();
1541     }
1542
1543     @Override
1544     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1545         final VpnInterface update) {
1546         LOG.trace("Received VpnInterface update event: original={}, update={}", original, update);
1547         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}", update.getName(),
1548                 update.getDpnId(), original.getVpnInstanceNames(), update.getVpnInstanceNames());
1549         final String vpnInterfaceName = update.getName();
1550         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1551         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1552         //handles switching between <internal VPN - external VPN>
1553         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1554             List<ListenableFuture<Void>> futures = new ArrayList<>();
1555             if (handleVpnInstanceUpdateForVpnInterface(identifier, original, update, futures)) {
1556                 LOG.info("update: handled Instance update for VPNInterface {} on dpn {} from oldVpn(s) {} "
1557                                 + "to newVpn(s) {}",
1558                         original.getName(), dpnId,
1559                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1560                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1561                 return emptyList();
1562             }
1563             updateVpnInstanceAdjChange(original, update, vpnInterfaceName, futures);
1564             return futures;
1565         });
1566     }
1567
1568     private boolean handleVpnInstanceUpdateForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1569                                                            VpnInterface original, VpnInterface update,
1570                                                            List<ListenableFuture<Void>> futures) {
1571         boolean isVpnInstanceUpdate = Boolean.FALSE;
1572         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1573         final String interfaceName = key.getName();
1574         List<String> oldVpnList = vpnUtil.getVpnListForVpnInterface(original);
1575         List<String> oldVpnListCopy = new ArrayList<>();
1576         oldVpnListCopy.addAll(oldVpnList);
1577         List<String> newVpnList = vpnUtil.getVpnListForVpnInterface(update);
1578         List<String> newVpnListCopy = new ArrayList<>();
1579         newVpnListCopy.addAll(newVpnList);
1580
1581         oldVpnList.removeAll(newVpnList);
1582         newVpnList.removeAll(oldVpnListCopy);
1583         //This block will execute only on if there is a change in the VPN Instance.
1584         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1585             /*
1586              * Internet BGP-VPN Instance update with single router:
1587              * ====================================================
1588              * In this case single VPN Interface will be part of maximum 2 VPN Instance only.
1589              *     1st VPN Instance : router VPN or external BGP-VPN.
1590              *     2nd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1591              *
1592              * VPN Instance UPDATE:
1593              * oldVpnList = 0 and newVpnList = 1 (Internet BGP-VPN)
1594              * oldVpnList = 1 and newVpnList = 0 (Internet BGP-VPN)
1595              *
1596              * External BGP-VPN Instance update with single router:
1597              * ====================================================
1598              * In this case single VPN interface will be part of maximum 1 VPN Instance only.
1599              *
1600              * Updated VPN Instance will be always either internal router VPN to
1601              * external BGP-VPN or external BGP-VPN to internal router VPN swap.
1602              *
1603              * VPN Instance UPDATE:
1604              * oldVpnList = 1 and newVpnList = 1 (router VPN to Ext-BGPVPN)
1605              * oldVpnList = 1 and newVpnList = 1 (Ext-BGPVPN to router VPN)
1606              *
1607              * Dual Router VPN Instance Update:
1608              * ================================
1609              * In this case single VPN interface will be part of maximum 3 VPN Instance only.
1610              *
1611              * 1st VPN Instance : router VPN or external BGP-VPN-1.
1612              * 2nd VPN Instance : router VPN or external BGP-VPN-2.
1613              * 3rd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1614              *
1615              * Dual Router --> Associated with common external BGP-VPN Instance.
1616              * 1st router and 2nd router are getting associated with single External BGP-VPN
1617              * 1) add 1st router to external bgpvpn --> oldVpnList=1, newVpnList=1;
1618              * 2) add 2nd router to the same external bgpvpn --> oldVpnList=1, newVpnList=0
1619              * In this case, we need to call removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1620              *
1621              *
1622              */
1623             isVpnInstanceUpdate = Boolean.TRUE;
1624             if (VpnUtil.isDualRouterVpnUpdate(oldVpnListCopy, newVpnListCopy)) {
1625                 if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1626                         && (oldVpnList.size() == 1 && newVpnList.size() == 0)) {
1627                     //Identify the external BGP-VPN Instance and pass that value as newVpnList
1628                     List<String> externalBgpVpnList = new ArrayList<>();
1629                     for (String newVpnName : newVpnListCopy) {
1630                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1631                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1632                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1633                                 .BgpvpnType.BGPVPNExternal) {
1634                             externalBgpVpnList.add(newVpnName);
1635                             break;
1636                         }
1637                     }
1638                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1639                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList,
1640                             externalBgpVpnList, oldVpnListCopy, futures);
1641
1642                 } else if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1643                         && (oldVpnList.size() == 0 && newVpnList.size() == 1)) {
1644                     //Identify the router VPN Instance and pass that value as oldVpnList
1645                     List<String> routerVpnList = new ArrayList<>();
1646                     for (String newVpnName : newVpnListCopy) {
1647                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1648                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1649                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1650                                 .BgpvpnType.VPN) {
1651                             routerVpnList.add(newVpnName);
1652                             break;
1653                         }
1654                     }
1655                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1656                     updateVpnInstanceChange(identifier, interfaceName, original, update, routerVpnList,
1657                             newVpnList, oldVpnListCopy, futures);
1658
1659                 } else {
1660                     //Handle remaining use cases.
1661                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1662                             oldVpnListCopy, futures);
1663                 }
1664             } else {
1665                 updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1666                         oldVpnListCopy, futures);
1667             }
1668         }
1669         return isVpnInstanceUpdate;
1670     }
1671
1672     private void updateVpnInstanceChange(InstanceIdentifier<VpnInterface> identifier, String interfaceName,
1673                                          VpnInterface original, VpnInterface update, List<String> oldVpnList,
1674                                          List<String> newVpnList, List<String> oldVpnListCopy,
1675                                          List<ListenableFuture<Void>> futures) {
1676         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1677         final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1678                 ? origAdjs.getAdjacency() : new ArrayList<>();
1679         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1680         final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1681                 ? updateAdjs.getAdjacency() : new ArrayList<>();
1682
1683         boolean isOldVpnRemoveCallExecuted = false;
1684         for (String oldVpnName : oldVpnList) {
1685             LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1686                     + "remove from vpnName {} ", interfaceName, oldVpnName);
1687             removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1688             LOG.info("updateVpnInstanceChange: Processed Remove for update on VPNInterface"
1689                             + " {} upon VPN update from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName,
1690                     newVpnList);
1691             isOldVpnRemoveCallExecuted = true;
1692         }
1693         //Wait for previous interface bindings to be removed
1694         if (isOldVpnRemoveCallExecuted && !newVpnList.isEmpty()) {
1695             try {
1696                 Thread.sleep(2000);
1697             } catch (InterruptedException e) {
1698                 //Ignore
1699             }
1700         }
1701         for (String newVpnName : newVpnList) {
1702             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1703             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1704                 LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1705                         + "onto vpnName {} ", interfaceName, newVpnName);
1706                 addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1707                 LOG.info("updateVpnInstanceChange: Processed Add for update on VPNInterface {}"
1708                                 + "from oldVpn(s) {} to newVpn {} ",
1709                         interfaceName, oldVpnListCopy, newVpnName);
1710                 /* This block will execute only if V6 subnet is associated with internet BGP-VPN.
1711                  * Use Case:
1712                  *     In Dual stack network, first V4 subnet only attached to router and router is associated
1713                  *     with internet BGP-VPN(router-gw). At this point VPN interface is having only router vpn instance.
1714                  *     Later V6 subnet is added to router, at this point existing VPN interface will get updated
1715                  *     with Internet BGP-VPN instance(Note: Internet BGP-VPN Instance update in vpn interface
1716                  *     is applicable for only on V6 subnet is added to router). newVpnList = Contains only Internet
1717                  *     BGP-VPN Instance. So we required V6 primary adjacency info needs to be populated onto
1718                  *     router VPN as well as Internet BGP-VPN.
1719                  *
1720                  *     addVpnInterfaceCall() --> It will create V6 Adj onto Internet BGP-VPN only.
1721                  *     updateVpnInstanceAdjChange() --> This method call is needed for second primary V6 Adj
1722                  *                                       update in existing router VPN instance.
1723                  */
1724                 if (vpnUtil.isBgpVpnInternet(newVpnName)) {
1725                     LOG.info("updateVpnInstanceChange: VPN Interface {} with new Adjacency {} in existing "
1726                             + "VPN instance {}", interfaceName, newAdjs, original.getVpnInstanceNames());
1727                     updateVpnInstanceAdjChange(original, update, interfaceName, futures);
1728                 }
1729             }
1730         }
1731     }
1732
1733     private List<ListenableFuture<Void>> updateVpnInstanceAdjChange(VpnInterface original, VpnInterface update,
1734                                                                     String vpnInterfaceName,
1735                                                                     List<ListenableFuture<Void>> futures) {
1736         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1737         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1738                 != null ? origAdjs.getAdjacency() : new ArrayList<>();
1739         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1740         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1741                 != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1742
1743         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1744         for (VpnInstanceNames vpnInterfaceVpnInstance : requireNonNullElse(update.getVpnInstanceNames(),
1745                 Collections.<VpnInstanceNames>emptyList())) {
1746             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1747             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1748             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1749             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1750             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1751                 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1752                 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
1753                     futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
1754                         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1755                                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1756                         LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven",
1757                                 update.getName(), newVpnName);
1758                         //handle both addition and removal of adjacencies
1759                         //currently, new adjacency may be an extra route
1760                         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1761                         if (!oldAdjs.equals(newAdjs)) {
1762                             for (Adjacency adj : copyNewAdjs) {
1763                                 if (copyOldAdjs.contains(adj)) {
1764                                     copyOldAdjs.remove(adj);
1765                                 } else {
1766                                     // add new adjacency
1767                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1768                                         addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1769                                                 dpnId, operTx, confTx, confTx);
1770                                     }
1771                                     LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to"
1772                                                     + " vpn interface {} on vpn {} dpnId {}",
1773                                             adj.getIpAddress(), adj.getNextHopIpList(),
1774                                             adj.getLabel(), adj.getSubnetId(), update.getName(),
1775                                             newVpnName, dpnId);
1776                                 }
1777                             }
1778                             for (Adjacency adj : copyOldAdjs) {
1779                                 if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1780                                     if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1781                                             && !adj.isPhysNetworkFunc()) {
1782                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1783                                                 operTx, confTx);
1784                                         //remove FIB entry
1785                                         String vpnRd = vpnUtil.getVpnRd(newVpnName);
1786                                         LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1787                                                 + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1788                                         //remove BGP entry
1789                                         fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1790                                         if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1791                                             bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1792                                         }
1793                                     } else {
1794                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1795                                                 operTx, confTx);
1796                                     }
1797                                 }
1798                                 LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1799                                                 + " vpn interface {} on vpn {}", adj.getIpAddress(), adj
1800                                                 .getNextHopIpList(),
1801                                         adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1802                             }
1803                         }
1804                     }));
1805                 }));
1806                 for (ListenableFuture<Void> future : futures) {
1807                     ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1808                             update.getName(), update.getVpnInstanceNames());
1809                 }
1810             } else {
1811                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1812                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1813             }
1814         }
1815         return futures;
1816     }
1817
1818     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1819         try {
1820             Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1821             synchronized (label.toString().intern()) {
1822                 InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1823                         .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1824                 Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1825                         LogicalDatastoreType.OPERATIONAL, lriIid);
1826                 if (opResult.isPresent()) {
1827                     LabelRouteInfo labelRouteInfo =
1828                             new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1829                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1830                             labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1831                 }
1832             }
1833             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1834                     nextHopIpList);
1835         } catch (ReadFailedException e) {
1836             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1837                     nextHopIpList);
1838         } catch (TransactionCommitFailedException e) {
1839             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1840                     nextHopIpList);
1841         }
1842     }
1843
1844     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1845         SubnetRoute route, String parentVpnRd, TypedWriteTransaction<Configuration> writeConfigTxn) {
1846
1847         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1848         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1849                 .addAugmentation(SubnetRoute.class, route).build();
1850         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1851         InstanceIdentifierBuilder<VrfTables> idBuilder =
1852             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1853         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1854         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1855         if (writeConfigTxn != null) {
1856             writeConfigTxn.merge(vrfTableId, vrfTableNew, CREATE_MISSING_PARENTS);
1857         } else {
1858             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1859         }
1860         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1861                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1862     }
1863
1864     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1865                                            Adjacency adj, BigInteger dpnId,
1866                                            TypedWriteTransaction<Operational> writeOperTxn,
1867                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1868                                            TypedReadWriteTransaction<Configuration> writeInvTxn)
1869             throws ExecutionException, InterruptedException {
1870         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1871         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1872         try {
1873             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1874                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1875             if (optVpnInterface.isPresent()) {
1876                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1877                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1878                 String vpnName = currVpnIntf.getVpnInstanceName();
1879                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1880                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1881                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1882                         LogicalDatastoreType.OPERATIONAL, adjPath);
1883                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1884                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1885                 long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1886                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1887                 List<Adjacency> adjacencies = new ArrayList<>();
1888                 if (optAdjacencies.isPresent() && optAdjacencies.get().getAdjacency() != null) {
1889                     adjacencies.addAll(optAdjacencies.get().getAdjacency());
1890                 }
1891                 long vpnId = vpnUtil.getVpnId(vpnName);
1892                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1893                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1894                 Adjacency operationalAdjacency = null;
1895                 //Handling dual stack neutron port primary adjacency
1896                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1897                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1898                             currVpnIntf.getName(), vpnName);
1899                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1900                             currVpnIntf.getName());
1901                     if (interfaceState != null) {
1902                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1903                                 currVpnIntf.getName(),
1904                                 vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
1905                     }
1906                 }
1907                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1908                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1909                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
1910                             : RouteOrigin.STATIC;
1911                     String nh = adj.getNextHopIpList().get(0);
1912                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1913                     synchronized (vpnPrefixKey.intern()) {
1914                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1915                                 vpnId, null, prefix, vpnName, nh, dpnId);
1916                         if (rdToAllocate.isPresent()) {
1917                             input.setRd(rdToAllocate.get());
1918                             operationalAdjacency = populator.createOperationalAdjacency(input);
1919                             int label = operationalAdjacency.getLabel().intValue();
1920                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1921                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
1922                                     currVpnIntf.getName(), operationalAdjacency, encapType, writeConfigTxn);
1923                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1924                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1925                                     vpnName, label, currVpnIntf.getName(), dpnId);
1926                         } else {
1927                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
1928                                     vpnName, prefix);
1929                             return;
1930                         }
1931                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1932                         // Keeping the MPLS check for now.
1933                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1934                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1935                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1936                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
1937                             vpnsToImportRoute.forEach(vpn -> {
1938                                 if (vpn.getVrfId() != null) {
1939                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(), vpnId, prefix,
1940                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
1941                                             .ifPresent(
1942                                                 rds -> vpnManager.addExtraRoute(
1943                                                         vpnUtil.getVpnName(vpn.getVpnId()),
1944                                                         adj.getIpAddress(), nh, rds,
1945                                                         currVpnIntf.getVpnInstanceName(), l3vni,
1946                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(),
1947                                                         opAdjacency, encapType, writeConfigTxn));
1948                                 }
1949                             });
1950                         }
1951                     }
1952                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1953                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1954                             currVpnIntf.getName(), vpnName);
1955
1956                     InstanceIdentifier<VpnInterface> vpnIfaceConfigidentifier = VpnUtil
1957                             .getVpnInterfaceIdentifier(currVpnIntf.getName());
1958                     Optional<VpnInterface> vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1959                             LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier);
1960                     Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1961                             Prefixes.PrefixCue.PhysNetFunc);
1962                     if (vpnIntefaceConfig.isPresent()) {
1963                         pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1964                                 vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(),
1965                                 vpnIntefaceConfig.get().getSegmentationId(), Prefixes.PrefixCue.PhysNetFunc);
1966                     }
1967
1968                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1969
1970                     writeOperTxn.merge(
1971                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
1972                                     prefix), pnfPrefix, true);
1973
1974                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1975                             adj.getIpAddress(), emptyList(), null /* EncapType */, 0 /* label */,
1976                             0 /*l3vni*/, null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1977
1978                     input.setRd(adj.getVrfId());
1979                 }
1980                 if (operationalAdjacency == null) {
1981                     operationalAdjacency = populator.createOperationalAdjacency(input);
1982                 }
1983                 adjacencies.add(operationalAdjacency);
1984                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1985                 VpnInterfaceOpDataEntry newVpnIntf =
1986                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
1987                                 aug, dpnId, currVpnIntf.getLportTag(),
1988                                 currVpnIntf.getGatewayMacAddress());
1989
1990                 writeOperTxn.merge(identifier, newVpnIntf, CREATE_MISSING_PARENTS);
1991             }
1992         } catch (ReadFailedException e) {
1993             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
1994                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
1995         }
1996     }
1997
1998     @Nullable
1999     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
2000         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
2001         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
2002     }
2003
2004     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
2005                                             BigInteger dpnId, TypedWriteTransaction<Operational> writeOperTxn,
2006                                             TypedWriteTransaction<Configuration> writeConfigTxn) {
2007         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
2008         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
2009         try {
2010             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
2011                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
2012             if (optVpnInterface.isPresent()) {
2013                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
2014                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
2015                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2016                         LogicalDatastoreType.OPERATIONAL, path);
2017                 if (optAdjacencies.isPresent()) {
2018                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
2019
2020                     if (adjacencies != null && !adjacencies.isEmpty()) {
2021                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
2022                         for (Adjacency adjacency : adjacencies) {
2023                             if (Objects.equals(adjacency.getIpAddress(), adj.getIpAddress())) {
2024                                 String rd = adjacency.getVrfId();
2025                                 if (adj.getNextHopIpList() != null) {
2026                                     for (String nh : adj.getNextHopIpList()) {
2027                                         deleteExtraRouteFromCurrentAndImportingVpns(
2028                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
2029                                                 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
2030                                     }
2031                                 } else if (adj.isPhysNetworkFunc()) {
2032                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
2033                                             adj.getIpAddress(), adj.getSubnetId());
2034                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
2035                                             writeConfigTxn);
2036                                 }
2037                                 break;
2038                             }
2039
2040                         }
2041                     }
2042                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
2043                             dpnId, adj.getVrfId());
2044                 } else {
2045                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
2046                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
2047                 }
2048             }
2049         } catch (ReadFailedException e) {
2050             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
2051                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
2052         }
2053     }
2054
2055     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
2056                                     String rd, String intfName, TypedWriteTransaction<Configuration> writeConfigTxn,
2057                                     TypedWriteTransaction<Operational> writeOperTx) {
2058         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
2059         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
2060         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
2061             String vpnRd = vpn.getVrfId();
2062             if (vpnRd != null) {
2063                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
2064                         writeOperTx);
2065             }
2066         }
2067     }
2068
2069     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
2070         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2071             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
2072             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
2073     }
2074
2075     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
2076         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2077             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
2078     }
2079
2080     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2081                                                 TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2082         if (vpnInterface == null) {
2083             return;
2084         }
2085         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
2086         if (adjs == null) {
2087             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2088                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2089             return;
2090         }
2091         for (Adjacency adj : adjs) {
2092             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2093                 String primaryInterfaceIp = adj.getIpAddress();
2094                 String macAddress = adj.getMacAddress();
2095                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2096
2097                 long label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
2098                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2099
2100                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2101                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2102                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
2103                         routerInt, label, writeConfigTxn);
2104                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2105                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2106                         macAddress);
2107                 return;
2108             }
2109         }
2110         LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2111                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}", interfaceName,
2112                 primaryRd, vpnName);
2113     }
2114
2115     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
2116             TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2117         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
2118         String rd = vpnUtil.getVpnRd(vpnName);
2119         if (adjs != null) {
2120             List<Adjacency> adjsList = requireNonNullElse(adjs.getAdjacency(), emptyList());
2121             for (Adjacency adj : adjsList) {
2122                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2123                     String primaryInterfaceIp = adj.getIpAddress();
2124                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2125                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
2126                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2127                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2128                     return;
2129                 }
2130             }
2131         } else {
2132             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2133                     vpnInterface.getName(), rd);
2134         }
2135     }
2136
2137     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2138         final VpnInterfaceKey key = intefaceData.identifier.firstKeyOf(VpnInterface.class);
2139         final String interfaceName = key.getName();
2140         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2141                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2142         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2143                   intefaceData.identifier, vpnName);
2144     }
2145
2146     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2147                                               VpnInterface vpnInterface, String vpnName) {
2148         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2149                .get(vpnName);
2150         if (vpnInterfaces == null) {
2151             vpnInterfaces = new ConcurrentLinkedQueue<>();
2152         }
2153         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2154         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2155         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2156                 vpnInterface.getName(), vpnName);
2157     }
2158
2159     public boolean isVpnInstanceReady(String vpnInstanceName) {
2160         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2161         if (vpnRd == null) {
2162             return false;
2163         }
2164         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2165
2166         return vpnInstanceOpDataEntry != null;
2167     }
2168
2169     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2170         synchronized (vpnInstanceName.intern()) {
2171             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2172                     unprocessedVpnInterfaces.get(vpnInstanceName);
2173             if (vpnInterfaces != null) {
2174                 while (!vpnInterfaces.isEmpty()) {
2175                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2176                     if (hasVpnInstanceCreatedSuccessfully) {
2177                         processSavedInterface(savedInterface, vpnInstanceName);
2178                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2179                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2180                     } else {
2181                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2182                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2183                     }
2184                 }
2185             } else {
2186                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2187             }
2188         }
2189     }
2190
2191     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2192             VpnInterface vpnInterface) {
2193         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2194             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2195                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2196             if (vpnInterfaces != null) {
2197                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2198                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2199                             + "unprocessed list", vpnInterface.getName(),
2200                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2201                 }
2202             } else {
2203                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2204                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2205             }
2206         }
2207     }
2208
2209     public void vpnInstanceIsReady(String vpnInstanceName) {
2210         processSavedInterfaces(vpnInstanceName, true);
2211     }
2212
2213     public void vpnInstanceFailed(String vpnInstanceName) {
2214         processSavedInterfaces(vpnInstanceName, false);
2215     }
2216
2217     private static class UnprocessedVpnInterfaceData {
2218         InstanceIdentifier<VpnInterface> identifier;
2219         VpnInterface vpnInterface;
2220
2221         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2222             this.identifier = identifier;
2223             this.vpnInterface = vpnInterface;
2224         }
2225
2226         @Override
2227         public int hashCode() {
2228             final int prime = 31;
2229             int result = 1;
2230             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2231             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2232             return result;
2233         }
2234
2235         @Override
2236         public boolean equals(Object obj) {
2237             if (this == obj) {
2238                 return true;
2239             }
2240             if (obj == null) {
2241                 return false;
2242             }
2243             if (getClass() != obj.getClass()) {
2244                 return false;
2245             }
2246             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2247             if (identifier == null) {
2248                 if (other.identifier != null) {
2249                     return false;
2250                 }
2251             } else if (!identifier.equals(other.identifier)) {
2252                 return false;
2253             }
2254             if (vpnInterface == null) {
2255                 if (other.vpnInterface != null) {
2256                     return false;
2257                 }
2258             } else if (!vpnInterface.equals(other.vpnInterface)) {
2259                 return false;
2260             }
2261             return true;
2262         }
2263     }
2264
2265     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2266         String primaryRd = vpnUtil.getVpnRd(vpnName);
2267         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2268         if (vpnInstanceOpData == null) {
2269             return;
2270         }
2271         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2272         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2273             return;
2274         }
2275         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2276         vpnToDpnLists.forEach(vpnToDpnList -> {
2277             if (vpnToDpnList.getVpnInterfaces() == null) {
2278                 return;
2279             }
2280             vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2281                 try {
2282                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2283                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2284                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2285                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2286                     if (!vpnInterfaceOptional.isPresent()) {
2287                         return;
2288                     }
2289                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2290                             vpnInterface.getInterfaceName());
2291                     if (configVpnAdjacencies == null) {
2292                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2293                         return;
2294                     }
2295                     List<Adjacency> operationVpnAdjacencies = requireNonNullElse(vpnInterfaceOptional.get()
2296                             .augmentation(AdjacenciesOp.class).getAdjacency(), emptyList());
2297                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2298                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2299                     // in operational DS. These unprocessed adjacencies will be handled below.
2300                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2301                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2302                     configVpnAdjacencies.stream()
2303                         .filter(adjacency -> operationVpnAdjacencies.stream()
2304                                 .noneMatch(operationalAdjacency ->
2305                                     Objects.equals(operationalAdjacency.getIpAddress(), adjacency.getIpAddress())))
2306                         .forEach(adjacency -> {
2307                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2308                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2309                                 () -> {
2310                                     // TODO Deal with sequencing — the config tx must only submitted
2311                                     // if the oper tx goes in
2312                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2313                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2314                                         futures.add(
2315                                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx ->
2316                                                 futures.add(
2317                                                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
2318                                                         confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2319                                                             primaryRd, adjacency, vpnInterfaceOptional.get()
2320                                                                 .getDpnId(), operTx, confTx, confTx)))));
2321                                         return futures;
2322                                     } else {
2323                                         return emptyList();
2324                                     }
2325                                 });
2326                         });
2327                 } catch (ReadFailedException e) {
2328                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2329                             vpnName, primaryRd);
2330                 }
2331             });
2332         });
2333     }
2334
2335     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2336         private final String interfaceName;
2337         private final boolean add;
2338         private final String txnDestination;
2339
2340         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2341             this.interfaceName = interfaceName;
2342             this.add = add;
2343             this.txnDestination = transactionDest;
2344         }
2345
2346         @Override
2347         public void onSuccess(Void voidObj) {
2348             if (add) {
2349                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2350                         interfaceName, txnDestination);
2351             } else {
2352                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2353             }
2354         }
2355
2356         @Override
2357         public void onFailure(Throwable throwable) {
2358             if (add) {
2359                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2360                         interfaceName, txnDestination, throwable);
2361             } else {
2362                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2363                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2364             }
2365         }
2366     }
2367 }