vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14 import static org.opendaylight.netvirt.vpnmanager.VpnUtil.requireNonNullElse;
  15
  16 import com.google.common.base.Optional;
  17 import com.google.common.base.Preconditions;
  18 import com.google.common.collect.Iterators;
  19 import com.google.common.util.concurrent.FutureCallback;
  20 import com.google.common.util.concurrent.Futures;
  21 import com.google.common.util.concurrent.ListenableFuture;
  22 import com.google.common.util.concurrent.MoreExecutors;
  23 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  24 import java.math.BigInteger;
  25 import java.util.ArrayList;
  26 import java.util.Collections;
  27 import java.util.List;
  28 import java.util.Map;
  29 import java.util.Objects;
  30 import java.util.concurrent.ConcurrentHashMap;
  31 import java.util.concurrent.ConcurrentLinkedQueue;
  32 import java.util.concurrent.ExecutionException;
  33 import java.util.function.Consumer;
  34 import java.util.function.Predicate;
  35 import java.util.stream.Collectors;
  36 import javax.annotation.Nullable;
  37 import javax.annotation.PostConstruct;
  38 import javax.annotation.PreDestroy;
  39 import javax.inject.Inject;
  40 import javax.inject.Singleton;
  41 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  42 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  43 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  44 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  45 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  46 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  47 import org.opendaylight.genius.infra.Datastore.Configuration;
  48 import org.opendaylight.genius.infra.Datastore.Operational;
  49 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  50 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  51 import org.opendaylight.genius.infra.TransactionAdapter;
  52 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  53 import org.opendaylight.genius.infra.TypedWriteTransaction;
  54 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  55 import org.opendaylight.genius.mdsalutil.NWUtil;
  56 import org.opendaylight.genius.mdsalutil.NwConstants;
  57 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  58 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  59 import org.opendaylight.infrautils.caches.CacheProvider;
  60 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  61 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  62 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  63 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  64 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  65 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  66 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  67 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  68 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  69 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  70 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  71 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  72 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  73 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  74 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  75 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  76 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  77 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  78 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentrybase.RoutePaths;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 119 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 120 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 121 import org.slf4j.Logger;
 122 import org.slf4j.LoggerFactory;
 123
 124 @Singleton
 125 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 126
 127     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 128     private static final short DJC_MAX_RETRIES = 3;
 129
 130     private final DataBroker dataBroker;
 131     private final ManagedNewTransactionRunner txRunner;
 132     private final IBgpManager bgpManager;
 133     private final IFibManager fibManager;
 134     private final IMdsalApiManager mdsalManager;
 135     private final IdManagerService idManager;
 136     private final OdlInterfaceRpcService ifaceMgrRpcService;
 137     private final VpnFootprintService vpnFootprintService;
 138     private final IInterfaceManager interfaceManager;
 139     private final IVpnManager vpnManager;
 140     private final ArpResponderHandler arpResponderHandler;
 141     private final JobCoordinator jobCoordinator;
 142     private final VpnUtil vpnUtil;
 143
 144     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 145
 146     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 147             new ConcurrentHashMap<>();
 148
 149     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 150
 151     @Inject
 152     public VpnInterfaceManager(final DataBroker dataBroker,
 153                                final IBgpManager bgpManager,
 154                                final IdManagerService idManager,
 155                                final IMdsalApiManager mdsalManager,
 156                                final IFibManager fibManager,
 157                                final OdlInterfaceRpcService ifaceMgrRpcService,
 158                                final VpnFootprintService vpnFootprintService,
 159                                final IInterfaceManager interfaceManager,
 160                                final IVpnManager vpnManager,
 161                                final ArpResponderHandler arpResponderHandler,
 162                                final JobCoordinator jobCoordinator,
 163                                final CacheProvider cacheProvider,
 164                                final VpnUtil vpnUtil) {
 165         super(VpnInterface.class, VpnInterfaceManager.class);
 166
 167         this.dataBroker = dataBroker;
 168         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 169         this.bgpManager = bgpManager;
 170         this.idManager = idManager;
 171         this.mdsalManager = mdsalManager;
 172         this.fibManager = fibManager;
 173         this.ifaceMgrRpcService = ifaceMgrRpcService;
 174         this.vpnFootprintService = vpnFootprintService;
 175         this.interfaceManager = interfaceManager;
 176         this.vpnManager = vpnManager;
 177         this.arpResponderHandler = arpResponderHandler;
 178         this.jobCoordinator = jobCoordinator;
 179         this.vpnUtil = vpnUtil;
 180
 181         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 182                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 183                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 184     }
 185
 186     public Runnable isNotifyTaskQueued(String intfName) {
 187         return vpnIntfMap.remove(intfName);
 188     }
 189
 190     @PostConstruct
 191     public void start() {
 192         LOG.info("{} start", getClass().getSimpleName());
 193         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 194     }
 195
 196     @Override
 197     @PreDestroy
 198     public void close() {
 199         super.close();
 200         vpnInstanceOpDataEntryCache.close();
 201     }
 202
 203     @Override
 204     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 205         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 206     }
 207
 208     @Override
 209     protected VpnInterfaceManager getDataTreeChangeListener() {
 210         return VpnInterfaceManager.this;
 211     }
 212
 213     @Override
 214     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 215         LOG.info("add: intfName {} onto vpnName {}",
 216                  vpnInterface.getName(),
 217                  VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 218         addVpnInterface(identifier, vpnInterface, null, null);
 219     }
 220
 221     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 222                           final VpnInterface vpnInterface, String vpnName) {
 223         synchronized (vpnName.intern()) {
 224             if (isVpnInstanceReady(vpnName)) {
 225                 return true;
 226             }
 227             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 228             return false;
 229         }
 230     }
 231
 232     // TODO Clean up the exception handling
 233     @SuppressWarnings("checkstyle:IllegalCatch")
 234     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 235                              final @Nullable List<Adjacency> oldAdjs, final @Nullable List<Adjacency> newAdjs) {
 236         for (VpnInstanceNames vpnInterfaceVpnInstance : requireNonNullElse(vpnInterface.getVpnInstanceNames(),
 237                 Collections.<VpnInstanceNames>emptyList())) {
 238             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 239             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 240         }
 241     }
 242
 243     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 244                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 245         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 246         final String interfaceName = key.getName();
 247
 248         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 249             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 250                   vpnName, vpnInterface.getName());
 251             return;
 252         }
 253         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 254              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 255         List<Adjacency> copyOldAdjs = null;
 256         if (oldAdjs != null) {
 257             copyOldAdjs = new ArrayList<>();
 258             copyOldAdjs.addAll(oldAdjs);
 259         }
 260         List<Adjacency> copyNewAdjs = null;
 261         if (newAdjs != null) {
 262             copyNewAdjs = new ArrayList<>();
 263             copyNewAdjs.addAll(newAdjs);
 264         }
 265         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 266     }
 267
 268     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 269                     final VpnInterface vpnInterface, final @Nullable List<Adjacency> oldAdjs,
 270                     final @Nullable List<Adjacency> newAdjs,
 271                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 272         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 273         final String interfaceName = key.getName();
 274         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 275         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 276             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 277             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 278             if (interfaceState != null) {
 279                 try {
 280                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 281                     final int ifIndex = interfaceState.getIfIndex();
 282                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 283                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 284                         // (the inventory tx goes in last)
 285                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 286                         ListenableFuture<Void> confFuture =
 287                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 288                                 confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
 289                                     operTx -> futures.add(
 290                                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, invTx -> {
 291                                             LOG.info(
 292                                                 "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 293                                                     + " on dpn {}",
 294                                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 295                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 296                                                 confTx, operTx, invTx, interfaceState, vpnName);
 297                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 298                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 299                                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 300                                                 if (newAdjs != null) {
 301                                                     for (Adjacency adj : newAdjs) {
 302                                                         if (oldAdjs.contains(adj)) {
 303                                                             oldAdjs.remove(adj);
 304                                                         } else {
 305                                                             if (!isBgpVpnInternetVpn
 306                                                                 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 307                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 308                                                                     primaryRd, adj, dpnId, operTx, confTx, invTx);
 309                                                             }
 310                                                         }
 311                                                     }
 312                                                 }
 313                                                 for (Adjacency adj : oldAdjs) {
 314                                                     if (!isBgpVpnInternetVpn
 315                                                         || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 316                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 317                                                             operTx, confTx);
 318                                                     }
 319                                                 }
 320                                             }
 321                                         })))));
 322                         futures.add(confFuture);
 323                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 324                             MoreExecutors.directExecutor());
 325                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 326                             + " processed successfully", interfaceName, vpnName, dpnId);
 327                         return futures;
 328                     });
 329                 } catch (NumberFormatException | IllegalStateException e) {
 330                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 331                             + "interface {}. Interface addition on vpn {} failed", interfaceName,
 332                         vpnName, e);
 333                     return;
 334                 }
 335             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 336                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 337                     () -> {
 338                         ListenableFuture<Void> future =
 339                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
 340                                 createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 341                                     confTx, vpnName);
 342                                 LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 343                                     vpnName, vpnInterface.getDpnId());
 344                             });
 345                         ListenableFutures.addErrorLogging(future, LOG,
 346                             "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 347                         return Collections.singletonList(future);
 348                     });
 349             } else {
 350                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 351                     + " is not available", interfaceName, vpnName);
 352             }
 353         } else {
 354             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 355                     + " as vpn is pending delete", interfaceName, vpnName,
 356                 vpnInterface.getDpnId());
 357         }
 358     }
 359
 360     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 361     // see comments below.
 362     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 363     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 364             final int lportTag, boolean isInterfaceUp,
 365             TypedWriteTransaction<Configuration> writeConfigTxn,
 366             TypedWriteTransaction<Operational> writeOperTxn,
 367             TypedReadWriteTransaction<Configuration> writeInvTxn,
 368             Interface interfaceState,
 369             final String vpnName) throws ExecutionException, InterruptedException {
 370         final String interfaceName = vpnInterface.getName();
 371         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 372                 vpnName);
 373         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 374         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 375         if (!isInterfaceUp) {
 376             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 377                      interfaceName, dpId, vpnName);
 378             long vpnId = vpnUtil.getVpnId(vpnName);
 379             if (vpnId == VpnConstants.INVALID_ID) {
 380                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 381                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 382                         dpId);
 383                 return;
 384             }
 385
 386             boolean waitForVpnInterfaceOpRemoval = false;
 387             if (opVpnInterface != null) {
 388                 String opVpnName = opVpnInterface.getVpnInstanceName();
 389                 String primaryInterfaceIp = null;
 390                 if (Objects.equals(opVpnName, vpnName)) {
 391                     // Please check if the primary VRF Entry does not exist for VPNInterface
 392                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 393                     // back to back
 394                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 395                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 396                     if (adjs == null) {
 397                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 398                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 399                                 vpnName);
 400                         return;
 401                     }
 402                     for (Adjacency adj : adjs) {
 403                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 404                             primaryInterfaceIp = adj.getIpAddress();
 405                             break;
 406                         }
 407                     }
 408                     if (primaryInterfaceIp == null) {
 409                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 410                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 411                                 dpId, vpnName);
 412                         return;
 413                     }
 414                     // Get the rd of the vpn instance
 415                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 416                     if (vrf != null) {
 417                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 418                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 419                         return;
 420                     }
 421                     waitForVpnInterfaceOpRemoval = true;
 422                 } else {
 423                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 424                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 425                 }
 426             }
 427             if (!waitForVpnInterfaceOpRemoval) {
 428                 // Add the VPNInterface and quit
 429                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 430                         null/*ipAddressSourceValuePair*/,
 431                         true /* add */);
 432                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 433                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 434                 if (!isBgpVpnInternetVpn) {
 435                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 436                 }
 437                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 438                         dpId, vpnName);
 439                 if (interfaceManager.isExternalInterface(interfaceName)) {
 440                     processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
 441                         NwConstants.ADD_FLOW);
 442                 }
 443                 return;
 444             }
 445
 446             // FIB didn't get a chance yet to clean up this VPNInterface
 447             // Let us give it a chance here !
 448             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 449                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 450             try {
 451                 Runnable notifyTask = new VpnNotifyTask();
 452                 synchronized (notifyTask) {
 453                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 454                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 455                     // published here while holding the lock on it so this path will hit the wait before notify can be
 456                     // invoked.
 457                     vpnIntfMap.put(interfaceName, notifyTask);
 458                     try {
 459                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 460                     } catch (InterruptedException e) {
 461                         // Ignored
 462                     }
 463                 }
 464             } finally {
 465                 vpnIntfMap.remove(interfaceName);
 466             }
 467
 468             if (opVpnInterface != null) {
 469                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 470                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 471                         dpId, vpnName);
 472                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 473                 return;
 474             }
 475             // VPNInterface got removed, proceed with Add
 476             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 477                     interfaceName, dpId, vpnName);
 478             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 479                     null/*ipAddressSourceValuePair*/,
 480                     true /* add */);
 481             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 482                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 483             if (!isBgpVpnInternetVpn) {
 484                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 485             }
 486             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 487                     + " FIB to clean up", interfaceName, dpId, vpnName);
 488             if (interfaceManager.isExternalInterface(interfaceName)) {
 489                 processExternalVpnInterface(interfaceName, vpnName, dpId,
 490                                lportTag, NwConstants.ADD_FLOW);
 491             }
 492         } else {
 493             try {
 494                 // Interface is retained in the DPN, but its Link Up.
 495                 // Advertise prefixes again for this interface to BGP
 496                 InstanceIdentifier<VpnInterface> identifier =
 497                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 498                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 499                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 500                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 501                 // Perform similar operation as interface add event for extraroutes.
 502                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 503                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 504                         LogicalDatastoreType.CONFIGURATION, path);
 505                 if (!optAdjacencies.isPresent()) {
 506                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 507                     return;
 508                 }
 509                 List<Adjacency> adjacencies = requireNonNullElse(optAdjacencies.get().getAdjacency(), emptyList());
 510                 for (Adjacency adjacency : adjacencies) {
 511                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 512                         continue;
 513                     }
 514                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 515                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 516                         continue;
 517                     }
 518                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 519                             dpId, writeOperTxn, writeConfigTxn, writeInvTxn);
 520                 }
 521             } catch (ReadFailedException e) {
 522                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 523                         interfaceName, vpnName, primaryRd, dpId);
 524             }
 525         }
 526     }
 527
 528     private void processExternalVpnInterface(String interfaceName, String vpnName, BigInteger dpId,
 529         int lportTag, int addOrRemove) {
 530         Uuid extNetworkId;
 531         try {
 532             // vpn instance of ext-net interface is the network-id
 533             extNetworkId = new Uuid(vpnName);
 534         } catch (IllegalArgumentException e) {
 535             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 536                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 537             return;
 538         }
 539
 540         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 541         if (routerIds == null || routerIds.isEmpty()) {
 542             LOG.info("processExternalVpnInterface: No router is associated with {}."
 543                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 544                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 545             return;
 546         }
 547
 548         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 549                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 550         for (Uuid routerId : routerIds) {
 551             String routerName = routerId.getValue();
 552             BigInteger primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 553             if (Objects.equals(primarySwitch, dpId)) {
 554                 Routers router = vpnUtil.getExternalRouter(routerName);
 555                 if (router != null) {
 556                     if (addOrRemove == NwConstants.ADD_FLOW) {
 557                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 558                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 559                                 dpId, interfaceName, lportTag);
 560                     } else {
 561                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 562                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 563                                 dpId, interfaceName, lportTag);
 564                     }
 565                 } else {
 566                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 567                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 568                             interfaceName, dpId, vpnName);
 569                 }
 570             }
 571         }
 572     }
 573
 574     // TODO Clean up the exception handling
 575     @SuppressWarnings("checkstyle:IllegalCatch")
 576     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 577                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 578                                                   String vpnName, String interfaceName) {
 579         if (rd == null) {
 580             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 581                   interfaceName, dpnId, vpnName);
 582             return;
 583         } else {
 584             if (rd.equals(vpnName)) {
 585                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 586                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 587                 return;
 588             }
 589         }
 590         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 591                 interfaceName, dpnId, vpnName, rd);
 592
 593         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 594         if (nextHopIp == null) {
 595             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 596                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 597                     rd, vpnName);
 598             return;
 599         }
 600
 601         try {
 602             //Read NextHops
 603             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 604             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 605                     LogicalDatastoreType.OPERATIONAL, path);
 606             if (adjacencies.isPresent()) {
 607                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 608                 if (nextHops != null && !nextHops.isEmpty()) {
 609                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 610                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 611                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 612                     long l3vni = vpnInstanceOpData.getL3vni();
 613                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 614                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 615                     for (Adjacency nextHop : nextHops) {
 616                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 617                             continue;
 618                         }
 619                         String gatewayMac = null;
 620                         long label = 0;
 621                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 622                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 623                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 624                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 625                                     .get();
 626                         } else {
 627                             label = nextHop.getLabel();
 628                         }
 629                         try {
 630                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 631                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 632                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 633                                     encapType, (int)label, l3vni, 0 /*l2vni*/,
 634                                     gatewayMac);
 635                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 636                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 637                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 638                         } catch (Exception e) {
 639                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 640                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 641                                     interfaceName, dpnId, e);
 642                         }
 643                     }
 644                 }
 645             }
 646         } catch (ReadFailedException e) {
 647             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 648                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 649         }
 650     }
 651
 652     // TODO Clean up the exception handling
 653     @SuppressWarnings("checkstyle:IllegalCatch")
 654     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 655                         String vpnName, String interfaceName, TypedWriteTransaction<Configuration> writeConfigTxn,
 656                         TypedWriteTransaction<Operational> writeOperTx) {
 657         //Read NextHops
 658         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 659         String rd = vpnUtil.getVpnRd(interfaceName);
 660         if (rd == null) {
 661             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 662                 interfaceName, vpnName);
 663             return;
 664         } else {
 665             if (rd.equals(vpnName)) {
 666                 LOG.info(
 667                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 668                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 669                 return;
 670             }
 671         }
 672         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 673                vpnName, rd);
 674         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 675         try {
 676             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 677                     path);
 678         } catch (ReadFailedException e) {
 679             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 680                     interfaceName, vpnName);
 681         }
 682         if (adjacencies.isPresent()) {
 683             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 684
 685             if (nextHops != null && !nextHops.isEmpty()) {
 686                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 687                         nextHops, interfaceName, vpnName, rd);
 688                 for (Adjacency nextHop : nextHops) {
 689                     try {
 690                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 691                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 692                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 693                                     interfaceName, vpnName);
 694                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 695                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 696                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 697                                     interfaceName, vpnName);
 698                         } else {
 699                             // Perform similar operation as interface delete event for extraroutes.
 700                             String allocatedRd = nextHop.getVrfId();
 701                             for (String nh : requireNonNullElse(nextHop.getNextHopIpList(),
 702                                     Collections.<String>emptyList())) {
 703                                 deleteExtraRouteFromCurrentAndImportingVpns(
 704                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
 705                                     writeOperTx);
 706                             }
 707                         }
 708                     } catch (Exception e) {
 709                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 710                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 711                     }
 712                 }
 713             }
 714         }
 715     }
 716
 717     @SuppressWarnings("checkstyle:IllegalCatch")
 718     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 719                                                   String primaryRd, String interfaceName, final long vpnId,
 720                                                   TypedWriteTransaction<Configuration> writeConfigTxn,
 721                                                   TypedWriteTransaction<Operational> writeOperTxn,
 722                                                   TypedReadWriteTransaction<Configuration> writeInvTxn,
 723                                                   Interface interfaceState)
 724             throws ExecutionException, InterruptedException {
 725         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 726         // Read NextHops
 727         Optional<VpnInterface> vpnInteface = Optional.absent();
 728         try {
 729             vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 730             LogicalDatastoreType.CONFIGURATION, identifier);
 731         } catch (ReadFailedException e) {
 732             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 733                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 734         }
 735         Uuid intfnetworkUuid = null;
 736         NetworkType networkType = null;
 737         Long segmentationId = Long.valueOf(-1);
 738         Adjacencies adjacencies = null;
 739         if (vpnInteface.isPresent()) {
 740             intfnetworkUuid = vpnInteface.get().getNetworkId();
 741             networkType = vpnInteface.get().getNetworkType();
 742             segmentationId = vpnInteface.get().getSegmentationId();
 743             adjacencies = vpnInteface.get().augmentation(Adjacencies.class);
 744             if (adjacencies == null) {
 745                 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 746                         null/*gwMac*/, writeOperTxn);
 747                 return;
 748             }
 749         }
 750         // Get the rd of the vpn instance
 751         String nextHopIp = null;
 752         try {
 753             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 754         } catch (Exception e) {
 755             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 756                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 757         }
 758         List<String> nhList = new ArrayList<>();
 759         if (nextHopIp != null) {
 760             nhList.add(nextHopIp);
 761             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 762                     interfaceName, dpnId, vpnName, nhList);
 763         }
 764         Optional<String> gwMac = Optional.absent();
 765         String vpnInterfaceSubnetGwMacAddress = null;
 766         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 767         Long l3vni = vpnInstanceOpData.getL3vni();
 768         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 769         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 770         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 771         List<Adjacency> nextHops = (adjacencies != null) ? adjacencies.getAdjacency() : emptyList();
 772         List<Adjacency> value = new ArrayList<>();
 773         for (Adjacency nextHop : nextHops) {
 774             String rd = primaryRd;
 775             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 776             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 777                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 778                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 779                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 780                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 781                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 782                 continue;
 783             }
 784             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 785                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 786                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 787                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 788                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 789                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 790
 791                 Prefixes prefixes = (intfnetworkUuid != null)
 792                     ? VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, intfnetworkUuid ,networkType,
 793                             segmentationId, prefixCue) :
 794                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, prefixCue);
 795                 writeOperTxn.merge(VpnUtil.getPrefixToInterfaceIdentifier(
 796                         vpnUtil.getVpnId(vpnName), prefix), prefixes, true);
 797                 final Uuid subnetId = nextHop.getSubnetId();
 798
 799                 String gatewayIp = nextHop.getSubnetGatewayIp();
 800                 if (gatewayIp == null) {
 801                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 802                     if (gatewayIpOptional.isPresent()) {
 803                         gatewayIp = gatewayIpOptional.get();
 804                     }
 805                 }
 806
 807                 if (gatewayIp != null) {
 808                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 809                     if (gwMac.isPresent()) {
 810                         // A valid mac-address is available for this subnet-gateway-ip
 811                         // Use this for programming ARP_RESPONDER table here.  And save this
 812                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 813                         // to populate L3_GW_MAC_TABLE there.
 814                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 815                                 gatewayIp, gwMac.get());
 816                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 817                     } else {
 818                         // A valid mac-address is not available for this subnet-gateway-ip
 819                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 820                         // Save this connected-mac-address as gateway-mac-address for the
 821                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 822                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 823                         if (gwMac.isPresent()) {
 824                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 825                                     NwConstants.ADD_FLOW, gwMac.get());
 826                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 827                                     gatewayIp, gwMac.get());
 828                         } else {
 829                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 830                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 831                                 + "gwIp {}",
 832                                 subnetId, interfaceName, vpnName, gatewayIp);
 833                         }
 834                     }
 835                 } else {
 836                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 837                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 838                         subnetId, interfaceName, vpnName);
 839                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 840                 }
 841                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 842                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 843             } else {
 844                 //Extra route adjacency
 845                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 846                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 847                 synchronized (vpnPrefixKey.intern()) {
 848                     java.util.Optional<String> rdToAllocate = vpnUtil
 849                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 850                                     nextHop.getNextHopIpList().get(0), dpnId);
 851                     if (rdToAllocate.isPresent()) {
 852                         rd = rdToAllocate.get();
 853                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 854                             rd, prefix);
 855                     } else {
 856                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 857                         continue;
 858                     }
 859                 }
 860                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 861                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 862                         interfaceName, dpnId);
 863             }
 864             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 865             // can be different from the gateway-mac-address within the VRFEntry as the
 866             // gateway-mac-address is a superset.
 867             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 868                     : RouteOrigin.STATIC;
 869             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 870                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 871                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 872             Adjacency operationalAdjacency = null;
 873             try {
 874                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 875             } catch (NullPointerException e) {
 876                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 877                     input, e.getMessage());
 878                 return;
 879             }
 880             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 881                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 882                         vpnName, l3vni, origin,
 883                         interfaceName, operationalAdjacency, encapType, writeConfigTxn);
 884             }
 885             value.add(operationalAdjacency);
 886         }
 887
 888         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 889         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 890                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 891
 892         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 893                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 894                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 895
 896         for (Adjacency nextHop : aug.getAdjacency()) {
 897             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 898             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 899                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 900                         : RouteOrigin.STATIC;
 901                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 902                 registeredPopulator.populateFib(input, writeConfigTxn);
 903             }
 904         }
 905     }
 906
 907     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 908                                               long lportTag, String gwMac,
 909                                               TypedWriteTransaction<Operational> writeOperTxn) {
 910         VpnInterfaceOpDataEntry opInterface =
 911               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac);
 912         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 913             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 914         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
 915         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 916                 interfaceName, dpnId, vpnName);
 917     }
 918
 919     // TODO Clean up the exception handling
 920     @SuppressWarnings("checkstyle:IllegalCatch")
 921     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 922                                            StateTunnelList stateTunnelList,
 923                                            TypedWriteTransaction<Configuration> writeConfigTxn,
 924                                            TypedWriteTransaction<Operational> writeOperTxn) {
 925
 926         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
 927         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 928         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 929         List<Adjacency> adjList =
 930             adjacencies != null && adjacencies.getAdjacency() != null ? adjacencies.getAdjacency() : emptyList();
 931         if (adjList.isEmpty()) {
 932             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 933                     vpnInterface, srcDpnId);
 934             return;
 935         }
 936         String prefix = null;
 937         long label = 0;
 938         List<Adjacency> value = new ArrayList<>();
 939         boolean isNextHopAddReqd = false;
 940         String vpnName = vpnInterface.getVpnInstanceName();
 941         long vpnId = vpnUtil.getVpnId(vpnName);
 942         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 943         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 944                 vpnInterface.getName(), vpnInterface.getDpnId(),
 945                 vpnInterface.getVpnInstanceName(), adjList);
 946         for (Adjacency adj : adjList) {
 947             String rd = adj.getVrfId();
 948             rd = rd != null ? rd : vpnName;
 949             prefix = adj.getIpAddress();
 950             label = adj.getLabel();
 951             List<String> nhList = Collections.singletonList(srcTepIp);
 952             List<String> nextHopList = adj.getNextHopIpList();
 953             // If TEP is added , update the nexthop of primary adjacency.
 954             // Secondary adj nexthop is already pointing to primary adj IP address.
 955             if (nextHopList == null || nextHopList.isEmpty()) {
 956                 isNextHopAddReqd = true;
 957             }
 958
 959             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 960                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 961             } else {
 962                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 963                 if (!vrfEntryOptional.isPresent()) {
 964                     continue;
 965                 }
 966                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 967                 if (!nhList.contains(srcTepIp)) {
 968                     nhList.add(srcTepIp);
 969                     isNextHopAddReqd = true;
 970                 }
 971                 value.add(adj);
 972             }
 973
 974             if (isNextHopAddReqd) {
 975                 updateLabelMapper(label, nhList);
 976                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 977                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 978                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 979                 // Update the VRF entry with nextHop
 980                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 981                         label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
 982
 983                 //Get the list of VPN's importing this route(prefix) .
 984                 // Then update the VRF entry with nhList
 985                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
 986                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 987                     String vpnRd = vpn.getVrfId();
 988                     if (vpnRd != null) {
 989                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
 990                             srcTepIp, label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
 991                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 992                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
 993                             vpnInterface.getName(), srcDpnId, vpnName,
 994                             vpn.getVpnInstanceName(), vpnRd);
 995                     }
 996                 }
 997                 // Advertise the prefix to BGP only for external vpn
 998                 // since there is a nexthop change.
 999                 try {
1000                     if (!rd.equalsIgnoreCase(vpnName)) {
1001                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
1002                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
1003                                 null /*gatewayMacAddress*/);
1004                     }
1005                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
1006                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
1007                             srcDpnId, vpnName);
1008                 } catch (Exception ex) {
1009                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
1010                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1011                             vpnInterface.getName(), srcDpnId, vpnName, ex);
1012                 }
1013             }
1014         }
1015         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1016         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1017                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1018                 .addAugmentation(AdjacenciesOp.class, aug).build();
1019         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1020                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1021         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1022         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
1023                 vpnInterface.getName(), srcDpnId, vpnName);
1024
1025     }
1026
1027     // TODO Clean up the exception handling
1028     @SuppressWarnings("checkstyle:IllegalCatch")
1029     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1030                                               StateTunnelList stateTunnelList,
1031                                               TypedWriteTransaction<Configuration> writeConfigTxn,
1032                                               TypedWriteTransaction<Operational> writeOperTxn) {
1033
1034         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1035         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1036         String prefix = null;
1037         long label = 0;
1038         boolean isNextHopRemoveReqd = false;
1039         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
1040         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
1041         String vpnName = vpnInterface.getVpnInstanceName();
1042         long vpnId = vpnUtil.getVpnId(vpnName);
1043         String primaryRd = vpnUtil.getVpnRd(vpnName);
1044         if (adjList != null) {
1045             List<Adjacency> value = new ArrayList<>();
1046             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1047                     vpnInterface.getName(), vpnInterface.getDpnId(),
1048                     vpnInterface.getVpnInstanceName(), adjList);
1049             for (Adjacency adj : adjList) {
1050                 List<String> nhList = new ArrayList<>();
1051                 String rd = adj.getVrfId();
1052                 rd = rd != null ? rd : vpnName;
1053                 prefix = adj.getIpAddress();
1054                 List<String> nextHopList = adj.getNextHopIpList();
1055                 label = adj.getLabel();
1056                 if (nextHopList != null && !nextHopList.isEmpty()) {
1057                     isNextHopRemoveReqd = true;
1058                 }
1059                 // If TEP is deleted , remove the nexthop from primary adjacency.
1060                 // Secondary adj nexthop will continue to point to primary adj IP address.
1061                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1062                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1063                 } else {
1064                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1065                     if (!vrfEntryOptional.isPresent()) {
1066                         continue;
1067                     }
1068                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1069                     if (nhList.contains(srcTepIp)) {
1070                         nhList.remove(srcTepIp);
1071                         isNextHopRemoveReqd = true;
1072                     }
1073                     value.add(adj);
1074                 }
1075
1076                 if (isNextHopRemoveReqd) {
1077                     updateLabelMapper(label, nhList);
1078                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1079                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1080                             prefix, nhList, vpnName,
1081                             vpnId, rd, vpnInterface.getName());
1082                     // Update the VRF entry with removed nextHop
1083                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1084                             label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1085
1086                     //Get the list of VPN's importing this route(prefix) .
1087                     // Then update the VRF entry with nhList
1088                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1089                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1090                         String vpnRd = vpn.getVrfId();
1091                         if (vpnRd != null) {
1092                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1093                                 srcTepIp, label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1094                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1095                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1096                                     nhList, label, vpnInterface.getName(), srcDpnId,
1097                                     vpnName,
1098                                     vpn.getVpnInstanceName(), vpnRd);
1099                         }
1100                     }
1101
1102                     // Withdraw prefix from BGP only for external vpn.
1103                     try {
1104                         if (!rd.equalsIgnoreCase(vpnName)) {
1105                             bgpManager.withdrawPrefix(rd, prefix);
1106                         }
1107                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1108                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1109                                 vpnInterface.getName(), srcDpnId,
1110                                 vpnName);
1111                     } catch (Exception ex) {
1112                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1113                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1114                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1115                     }
1116                 }
1117             }
1118             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1119             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1120                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1121                     .addAugmentation(AdjacenciesOp.class, aug).build();
1122             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1123                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1124             writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1125             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1126                          vpnInterface.getName(), srcDpnId, vpnName);
1127         }
1128     }
1129
1130     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1131         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1132
1133         String vpnRd = vpnUtil.getVpnRd(vpnName);
1134         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1135         if (vpnInstanceOpDataEntry == null) {
1136             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1137                     + " to check for vpns exporting the routes", vpnName);
1138             return vpnsToExportRoute;
1139         }
1140
1141         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1142             if (input.getVpnInstanceName() == null) {
1143                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1144                         input.getVrfId());
1145                 return false;
1146             }
1147             return !input.getVpnInstanceName().equals(vpnName);
1148         };
1149
1150         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1151             Iterable<String> commonRTs =
1152                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1153                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1154             return Iterators.size(commonRTs.iterator()) > 0;
1155         };
1156
1157         vpnsToExportRoute =
1158                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1159                         Collectors.toList());
1160         return vpnsToExportRoute;
1161     }
1162
1163     // TODO Clean up the exception handling
1164     @SuppressWarnings("checkstyle:IllegalCatch")
1165     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1166         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1167         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1168             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1169             if (vrfEntries != null) {
1170                 ListenableFutures.addErrorLogging(
1171                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1172                         for (VrfEntry vrfEntry : vrfEntries) {
1173                             try {
1174                                 if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1175                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1176                                     LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1177                                             + " is not a controller managed non intervpn link route. Ignoring.",
1178                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1179                                     continue;
1180                                 }
1181                                 String prefix = vrfEntry.getDestPrefix();
1182                                 String gwMac = vrfEntry.getGatewayMacAddress();
1183                                 requireNonNullElse(vrfEntry.getRoutePaths(),
1184                                     Collections.<RoutePaths>emptyList()).forEach(routePath -> {
1185                                         String nh = routePath.getNexthopAddress();
1186                                         int label = routePath.getLabel().intValue();
1187                                         if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1188                                             vrfEntry.getOrigin()))) {
1189                                             LOG.info(
1190                                                 "handleVpnsExportingRoutesImporting: Importing fib entry rd {}"
1191                                                     + " prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1192                                                 vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1193                                             fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1194                                                 Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1195                                                 0 /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1196                                                 confTx);
1197                                         } else {
1198                                             LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry"
1199                                                     + " rd {} prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1200                                                 vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1201                                             SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1202                                             importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1203                                                 confTx);
1204                                         }
1205                                     });
1206                             } catch (RuntimeException e) {
1207                                 LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1208                                         + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1209                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1210                             }
1211                         }
1212                     }), LOG, "Error handing VPN exporting routes");
1213             } else {
1214                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1215                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1216             }
1217         }
1218     }
1219
1220     @Override
1221     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1222         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1223         final String interfaceName = key.getName();
1224         for (VpnInstanceNames vpnInterfaceVpnInstance : requireNonNullElse(vpnInterface.getVpnInstanceNames(),
1225                 Collections.<VpnInstanceNames>emptyList())) {
1226             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1227             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1228         }
1229     }
1230
1231     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1232                                 final VpnInterface vpnInterface, final String vpnName,
1233                                 final String interfaceName) {
1234         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1235             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1236                 ListenableFuture<Void> future =
1237                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1238                         deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1239                         LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1240                     });
1241                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1242                         vpnInterface.getName(), vpnName);
1243                 return Collections.singletonList(future);
1244             }, DJC_MAX_RETRIES);
1245         } else {
1246             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1247             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1248         }
1249     }
1250
1251     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1252     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1253                                            final VpnInterface vpnInterface, final String vpnName,
1254                                            final String interfaceName, final Interface interfaceState) {
1255         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1256                 vpnName, vpnInterface.getDpnId());
1257         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1258         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1259             () -> {
1260                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1261                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1262                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1263                         writeOperTxn -> futures.add(
1264                             txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeInvTxn -> {
1265                                 LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1266                                         interfaceName, vpnName);
1267                                 BigInteger dpId;
1268                                 int ifIndex;
1269                                 String gwMacAddress;
1270                                 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1271                                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1272                                 Optional<VpnInterfaceOpDataEntry> optVpnInterface;
1273                                 try {
1274                                     optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1275                                             LogicalDatastoreType.OPERATIONAL, interfaceId);
1276                                 } catch (ReadFailedException e) {
1277                                     LOG.error("remove: Failed to read data store for interface {} vpn {}",
1278                                             interfaceName, vpnName);
1279                                     return;
1280                                 }
1281                                 if (interfaceState != null) {
1282                                     try {
1283                                         dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1284                                     } catch (NumberFormatException | IllegalStateException e) {
1285                                         LOG.error("remove: Unable to retrieve dpnId from interface operational"
1286                                                         + " data store for interface {} on dpn {} for vpn {} Fetching"
1287                                                         + " from vpn interface op data store. ", interfaceName,
1288                                                 vpnInterface.getDpnId(), vpnName, e);
1289                                         dpId = BigInteger.ZERO;
1290                                     }
1291                                     ifIndex = interfaceState.getIfIndex();
1292                                     gwMacAddress = interfaceState.getPhysAddress().getValue();
1293                                 } else {
1294                                     LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1295                                             + " from vpn interface op.", interfaceName);
1296                                     if (optVpnInterface.isPresent()) {
1297                                         VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1298                                         dpId = vpnOpInterface.getDpnId();
1299                                         ifIndex = vpnOpInterface.getLportTag().intValue();
1300                                         gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1301                                     } else {
1302                                         LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1303                                                 + " as interfaceState and vpn interface op is not"
1304                                                 + " available", interfaceName, vpnName);
1305                                         return;
1306                                     }
1307                                 }
1308                                 processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1309                                         optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1310                                         writeConfigTxn, writeOperTxn, writeInvTxn);
1311                                 LOG.info(
1312                                         "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1313                                                 + "successfully",
1314                                         interfaceName, vpnInterface.getDpnId(), vpnName);
1315                             })))));
1316                 futures.add(configFuture);
1317                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1318                 return futures;
1319             }, DJC_MAX_RETRIES);
1320     }
1321
1322     protected void processVpnInterfaceDown(BigInteger dpId,
1323                                            String interfaceName,
1324                                            int lportTag,
1325                                            String gwMac,
1326                                            VpnInterfaceOpDataEntry vpnOpInterface,
1327                                            boolean isInterfaceStateDown,
1328                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1329                                            TypedWriteTransaction<Operational> writeOperTxn,
1330                                            TypedReadWriteTransaction<Configuration> writeInvTxn)
1331             throws ExecutionException, InterruptedException {
1332         if (vpnOpInterface == null) {
1333             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1334                     + " as it is not available in operational data store", interfaceName, dpId);
1335             return;
1336         }
1337         final String vpnName = vpnOpInterface.getVpnInstanceName();
1338         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1339                                                     interfaceName, vpnName);
1340         if (!isInterfaceStateDown) {
1341             final long vpnId = vpnUtil.getVpnId(vpnName);
1342             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName,  null);
1343             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1344             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1345                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1346             if (interfaceManager.isExternalInterface(interfaceName)) {
1347                 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1348                     NwConstants.DEL_FLOW);
1349             }
1350             if (!isBgpVpnInternetVpn) {
1351                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1352             }
1353             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1354                     + " successful", interfaceName, dpId, vpnName);
1355         } else {
1356             // Interface is retained in the DPN, but its Link Down.
1357             // Only withdraw the prefixes for this interface from BGP
1358             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1359         }
1360     }
1361
1362     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1363                                           final String vpnName, final long vpnId, String gwMac,
1364                                           TypedWriteTransaction<Configuration> writeConfigTxn,
1365                                           TypedWriteTransaction<Operational> writeOperTxn,
1366                                           TypedReadWriteTransaction<Configuration> writeInvTxn)
1367             throws ExecutionException, InterruptedException {
1368         //Read NextHops
1369         try {
1370             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1371                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1372             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1373             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1374                     LogicalDatastoreType.OPERATIONAL, path);
1375             String primaryRd = vpnUtil.getVpnRd(vpnName);
1376             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1377                     interfaceName, dpnId, vpnName, primaryRd);
1378             if (adjacencies.isPresent() && adjacencies.get().getAdjacency() != null
1379                     && !adjacencies.get().getAdjacency().isEmpty()) {
1380                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1381                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1382                         interfaceName, dpnId, vpnName, nextHops);
1383                 for (Adjacency nextHop : nextHops) {
1384                     if (nextHop.isPhysNetworkFunc()) {
1385                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1386                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1387                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1388                                 null/*writeCfgTxn*/);
1389                     } else {
1390                         String rd = nextHop.getVrfId();
1391                         List<String> nhList;
1392                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1393                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1394                         } else {
1395                             // This is a primary adjacency
1396                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1397                                     : emptyList();
1398                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1399                                     interfaceName, writeInvTxn);
1400                         }
1401                         if (!nhList.isEmpty()) {
1402                             if (Objects.equals(rd, vpnName)) {
1403                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1404                                 //remove from FIB directly
1405                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1406                                         interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1407                             } else {
1408                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1409                                         interfaceName, writeConfigTxn, writeOperTxn);
1410                             }
1411                         } else {
1412                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1413                                             + " interface {}", nextHop.getIpAddress(), rd,
1414                                     nextHop.getAdjacencyType().toString(), interfaceName);
1415                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1416                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1417                         }
1418                     }
1419                     String ip = nextHop.getIpAddress().split("/")[0];
1420                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1421                     if (vpnVipToPort != null) {
1422                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1423                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1424                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1425                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1426                     }
1427                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1428                     if (vpnPortipToPort != null) {
1429                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1430                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1431                                  + "Interface {} ip {} on dpn {} for vpn {}",
1432                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1433                     }
1434                 }
1435             } else {
1436                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1437                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1438                         + " Removing it.", interfaceName, vpnName, dpnId);
1439                 writeOperTxn.delete(identifier);
1440             }
1441         } catch (ReadFailedException e) {
1442             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1443                     interfaceName, dpnId, vpnName);
1444         }
1445     }
1446
1447     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1448                                                             String interfaceName, BigInteger dpnId,
1449                                                             TypedWriteTransaction<Configuration> writeConfigTxn,
1450                                                             TypedWriteTransaction<Operational> writeOperTx) {
1451         return (nh) -> {
1452             String primaryRd = vpnUtil.getVpnRd(vpnName);
1453             String prefix = nextHop.getIpAddress();
1454             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1455             LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1456                     nextHop, vpnName, interfaceName, dpnId);
1457             synchronized (vpnNamePrefixKey.intern()) {
1458                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1459                         prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1460                     //If extra-route is present behind at least one VM, then do not remove or update
1461                     //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1462                     //prefixtointerface DS
1463                     return;
1464                 }
1465                 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1466                         writeConfigTxn);
1467             }
1468             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1469                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1470                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1471         };
1472     }
1473
1474     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1475                                            BigInteger dpnId, String rd, String interfaceName,
1476                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1477                                            TypedWriteTransaction<Operational> writeOperTx) {
1478         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1479                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1480         nhList.forEach((nh) -> {
1481             //IRT: remove routes from other vpns importing it
1482             vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1483                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1484             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1485                 String vpnRd = vpn.getVrfId();
1486                 if (vpnRd != null) {
1487                     fibManager.removeOrUpdateFibEntry(vpnRd,
1488                             nextHop.getIpAddress(), nh, writeConfigTxn);
1489                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1490                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1491                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1492                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1493                 }
1494             }
1495         });
1496     }
1497
1498     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1499                                                  int lportTag, String gwMac, String interfaceName,
1500                                                  TypedReadWriteTransaction<Configuration> writeInvTxn)
1501             throws ExecutionException, InterruptedException {
1502         final Uuid subnetId = nextHop.getSubnetId();
1503         if (nextHop.getSubnetGatewayMacAddress() == null) {
1504             // A valid mac-address was not available for this subnet-gateway-ip
1505             // So a connected-mac-address was used for this subnet and we need
1506             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1507             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1508         }
1509         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1510                 subnetId);
1511     }
1512
1513     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1514                                                   String interfaceName) {
1515         // This is either an extra-route (or) a learned IP via subnet-route
1516         List<String> nhList = null;
1517         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1518         if (nextHopIp == null || nextHopIp.isEmpty()) {
1519             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1520                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1521                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1522                     vpnName);
1523             nhList = emptyList();
1524         } else {
1525             nhList = Collections.singletonList(nextHopIp);
1526         }
1527         return nhList;
1528     }
1529
1530     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1531         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1532         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1533         // else use connected interface
1534         if (gwPort != null && gwPort.isSubnetIp()) {
1535             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1536                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1537             return Optional.of(gwPort.getMacAddress());
1538         }
1539         return Optional.absent();
1540     }
1541
1542     @Override
1543     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1544         final VpnInterface update) {
1545         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}" ,update.getName(),
1546                 update.getDpnId(), original.getVpnInstanceNames(),
1547                 update.getVpnInstanceNames());
1548         final String vpnInterfaceName = update.getName();
1549         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1550         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1551         //handles switching between <internal VPN - external VPN>
1552         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1553             List<ListenableFuture<Void>> futures = new ArrayList<>();
1554             if (handleVpnInstanceUpdateForVpnInterface(identifier, original, update, futures)) {
1555                 LOG.info("update: handled Instance update for VPNInterface {} on dpn {} from oldVpn(s) {} "
1556                                 + "to newVpn(s) {}",
1557                         original.getName(), dpnId,
1558                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1559                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1560                 return emptyList();
1561             }
1562             updateVpnInstanceAdjChange(original, update, vpnInterfaceName, futures);
1563             return futures;
1564         });
1565     }
1566
1567     private boolean handleVpnInstanceUpdateForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1568                                                            VpnInterface original, VpnInterface update,
1569                                                            List<ListenableFuture<Void>> futures) {
1570         boolean isVpnInstanceUpdate = Boolean.FALSE;
1571         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1572         final String interfaceName = key.getName();
1573         List<String> oldVpnList = vpnUtil.getVpnListForVpnInterface(original);
1574         List<String> oldVpnListCopy = new ArrayList<>();
1575         oldVpnListCopy.addAll(oldVpnList);
1576         List<String> newVpnList = vpnUtil.getVpnListForVpnInterface(update);
1577         List<String> newVpnListCopy = new ArrayList<>();
1578         newVpnListCopy.addAll(newVpnList);
1579
1580         oldVpnList.removeAll(newVpnList);
1581         newVpnList.removeAll(oldVpnListCopy);
1582         //This block will execute only on if there is a change in the VPN Instance.
1583         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1584             /*
1585              * Internet BGP-VPN Instance update with single router:
1586              * ====================================================
1587              * In this case single VPN Interface will be part of maximum 2 VPN Instance only.
1588              *     1st VPN Instance : router VPN or external BGP-VPN.
1589              *     2nd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1590              *
1591              * VPN Instance UPDATE:
1592              * oldVpnList = 0 and newVpnList = 1 (Internet BGP-VPN)
1593              * oldVpnList = 1 and newVpnList = 0 (Internet BGP-VPN)
1594              *
1595              * External BGP-VPN Instance update with single router:
1596              * ====================================================
1597              * In this case single VPN interface will be part of maximum 1 VPN Instance only.
1598              *
1599              * Updated VPN Instance will be always either internal router VPN to
1600              * external BGP-VPN or external BGP-VPN to internal router VPN swap.
1601              *
1602              * VPN Instance UPDATE:
1603              * oldVpnList = 1 and newVpnList = 1 (router VPN to Ext-BGPVPN)
1604              * oldVpnList = 1 and newVpnList = 1 (Ext-BGPVPN to router VPN)
1605              *
1606              * Dual Router VPN Instance Update:
1607              * ================================
1608              * In this case single VPN interface will be part of maximum 3 VPN Instance only.
1609              *
1610              * 1st VPN Instance : router VPN or external BGP-VPN-1.
1611              * 2nd VPN Instance : router VPN or external BGP-VPN-2.
1612              * 3rd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1613              *
1614              * Dual Router --> Associated with common external BGP-VPN Instance.
1615              * 1st router and 2nd router are getting associated with single External BGP-VPN
1616              * 1) add 1st router to external bgpvpn --> oldVpnList=1, newVpnList=1;
1617              * 2) add 2nd router to the same external bgpvpn --> oldVpnList=1, newVpnList=0
1618              * In this case, we need to call removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1619              *
1620              *
1621              */
1622             isVpnInstanceUpdate = Boolean.TRUE;
1623             if (vpnUtil.isDualRouterVpnUpdate(oldVpnListCopy, newVpnListCopy)) {
1624                 if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1625                         && (oldVpnList.size() == 1 && newVpnList.size() == 0)) {
1626                     //Identify the external BGP-VPN Instance and pass that value as newVpnList
1627                     List<String> externalBgpVpnList = new ArrayList<>();
1628                     for (String newVpnName : newVpnListCopy) {
1629                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1630                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1631                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1632                                 .BgpvpnType.BGPVPNExternal) {
1633                             externalBgpVpnList.add(newVpnName);
1634                             break;
1635                         }
1636                     }
1637                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1638                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList,
1639                             externalBgpVpnList, oldVpnListCopy, futures);
1640
1641                 } else if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1642                         && (oldVpnList.size() == 0 && newVpnList.size() == 1)) {
1643                     //Identify the router VPN Instance and pass that value as oldVpnList
1644                     List<String> routerVpnList = new ArrayList<>();
1645                     for (String newVpnName : newVpnListCopy) {
1646                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1647                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1648                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1649                                 .BgpvpnType.VPN) {
1650                             routerVpnList.add(newVpnName);
1651                             break;
1652                         }
1653                     }
1654                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1655                     updateVpnInstanceChange(identifier, interfaceName, original, update, routerVpnList,
1656                             newVpnList, oldVpnListCopy, futures);
1657
1658                 } else {
1659                     //Handle remaining use cases.
1660                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1661                             oldVpnListCopy, futures);
1662                 }
1663             } else {
1664                 updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1665                         oldVpnListCopy, futures);
1666             }
1667         }
1668         return isVpnInstanceUpdate;
1669     }
1670
1671     private void updateVpnInstanceChange(InstanceIdentifier<VpnInterface> identifier, String interfaceName,
1672                                          VpnInterface original, VpnInterface update, List<String> oldVpnList,
1673                                          List<String> newVpnList, List<String> oldVpnListCopy,
1674                                          List<ListenableFuture<Void>> futures) {
1675         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1676         final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1677                 ? origAdjs.getAdjacency() : new ArrayList<>();
1678         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1679         final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1680                 ? updateAdjs.getAdjacency() : new ArrayList<>();
1681
1682         boolean isOldVpnRemoveCallExecuted = false;
1683         for (String oldVpnName : oldVpnList) {
1684             LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1685                     + "remove from vpnName {} ", interfaceName, oldVpnName);
1686             removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1687             LOG.info("updateVpnInstanceChange: Processed Remove for update on VPNInterface"
1688                             + " {} upon VPN update from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName,
1689                     newVpnList);
1690             isOldVpnRemoveCallExecuted = true;
1691         }
1692         //Wait for previous interface bindings to be removed
1693         if (isOldVpnRemoveCallExecuted && !newVpnList.isEmpty()) {
1694             try {
1695                 Thread.sleep(2000);
1696             } catch (InterruptedException e) {
1697                 //Ignore
1698             }
1699         }
1700         for (String newVpnName : newVpnList) {
1701             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1702             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1703                 LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1704                         + "onto vpnName {} ", interfaceName, newVpnName);
1705                 addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1706                 LOG.info("updateVpnInstanceChange: Processed Add for update on VPNInterface {}"
1707                                 + "from oldVpn(s) {} to newVpn {} ",
1708                         interfaceName, oldVpnListCopy, newVpnName);
1709                 /* This block will execute only if V6 subnet is associated with internet BGP-VPN.
1710                  * Use Case:
1711                  *     In Dual stack network, first V4 subnet only attached to router and router is associated
1712                  *     with internet BGP-VPN(router-gw). At this point VPN interface is having only router vpn instance.
1713                  *     Later V6 subnet is added to router, at this point existing VPN interface will get updated
1714                  *     with Internet BGP-VPN instance(Note: Internet BGP-VPN Instance update in vpn interface
1715                  *     is applicable for only on V6 subnet is added to router). newVpnList = Contains only Internet
1716                  *     BGP-VPN Instance. So we required V6 primary adjacency info needs to be populated onto
1717                  *     router VPN as well as Internet BGP-VPN.
1718                  *
1719                  *     addVpnInterfaceCall() --> It will create V6 Adj onto Internet BGP-VPN only.
1720                  *     updateVpnInstanceAdjChange() --> This method call is needed for second primary V6 Adj
1721                  *                                       update in existing router VPN instance.
1722                  */
1723                 if (vpnUtil.isBgpVpnInternet(newVpnName)) {
1724                     LOG.info("updateVpnInstanceChange: VPN Interface {} with new Adjacency {} in existing "
1725                             + "VPN instance {}", interfaceName, newAdjs, original.getVpnInstanceNames());
1726                     updateVpnInstanceAdjChange(original, update, interfaceName, futures);
1727                 }
1728             }
1729         }
1730     }
1731
1732     private List<ListenableFuture<Void>> updateVpnInstanceAdjChange(VpnInterface original, VpnInterface update,
1733                                                                     String vpnInterfaceName,
1734                                                                     List<ListenableFuture<Void>> futures) {
1735         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1736         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1737                 != null ? origAdjs.getAdjacency() : new ArrayList<>();
1738         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1739         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1740                 != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1741
1742         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1743         for (VpnInstanceNames vpnInterfaceVpnInstance : requireNonNullElse(update.getVpnInstanceNames(),
1744                 Collections.<VpnInstanceNames>emptyList())) {
1745             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1746             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1747             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1748             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1749             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1750                 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1751                 futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, confTx -> {
1752                     futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL, operTx -> {
1753                         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1754                                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1755                         LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven",
1756                                 update.getName(), newVpnName);
1757                         //handle both addition and removal of adjacencies
1758                         //currently, new adjacency may be an extra route
1759                         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1760                         if (!oldAdjs.equals(newAdjs)) {
1761                             for (Adjacency adj : copyNewAdjs) {
1762                                 if (copyOldAdjs.contains(adj)) {
1763                                     copyOldAdjs.remove(adj);
1764                                 } else {
1765                                     // add new adjacency
1766                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1767                                         addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1768                                                 dpnId, operTx, confTx, confTx);
1769                                     }
1770                                     LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to"
1771                                                     + " vpn interface {} on vpn {} dpnId {}",
1772                                             adj.getIpAddress(), adj.getNextHopIpList(),
1773                                             adj.getLabel(), adj.getSubnetId(), update.getName(),
1774                                             newVpnName, dpnId);
1775                                 }
1776                             }
1777                             for (Adjacency adj : copyOldAdjs) {
1778                                 if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1779                                     if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1780                                             && !adj.isPhysNetworkFunc()) {
1781                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1782                                                 operTx, confTx);
1783                                         //remove FIB entry
1784                                         String vpnRd = vpnUtil.getVpnRd(newVpnName);
1785                                         LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1786                                                 + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1787                                         //remove BGP entry
1788                                         fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1789                                         if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1790                                             bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1791                                         }
1792                                     } else {
1793                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1794                                                 operTx, confTx);
1795                                     }
1796                                 }
1797                                 LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1798                                                 + " vpn interface {} on vpn {}", adj.getIpAddress(), adj
1799                                                 .getNextHopIpList(),
1800                                         adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1801                             }
1802                         }
1803                     }));
1804                 }));
1805                 for (ListenableFuture<Void> future : futures) {
1806                     ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1807                             update.getName(), update.getVpnInstanceNames());
1808                 }
1809             } else {
1810                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1811                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1812             }
1813         }
1814         return futures;
1815     }
1816
1817     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1818         try {
1819             Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1820             synchronized (label.toString().intern()) {
1821                 InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1822                         .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1823                 Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1824                         LogicalDatastoreType.OPERATIONAL, lriIid);
1825                 if (opResult.isPresent()) {
1826                     LabelRouteInfo labelRouteInfo =
1827                             new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1828                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1829                             labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1830                 }
1831             }
1832             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1833                     nextHopIpList);
1834         } catch (ReadFailedException e) {
1835             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1836                     nextHopIpList);
1837         } catch (TransactionCommitFailedException e) {
1838             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1839                     nextHopIpList);
1840         }
1841     }
1842
1843     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1844         SubnetRoute route, String parentVpnRd, TypedWriteTransaction<Configuration> writeConfigTxn) {
1845
1846         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1847         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1848                 .addAugmentation(SubnetRoute.class, route).build();
1849         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1850         InstanceIdentifierBuilder<VrfTables> idBuilder =
1851             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1852         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1853         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1854         if (writeConfigTxn != null) {
1855             writeConfigTxn.merge(vrfTableId, vrfTableNew, CREATE_MISSING_PARENTS);
1856         } else {
1857             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1858         }
1859         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1860                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1861     }
1862
1863     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1864                                            Adjacency adj, BigInteger dpnId,
1865                                            TypedWriteTransaction<Operational> writeOperTxn,
1866                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1867                                            TypedReadWriteTransaction<Configuration> writeInvTxn)
1868             throws ExecutionException, InterruptedException {
1869         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1870         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1871         try {
1872             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1873                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1874             if (optVpnInterface.isPresent()) {
1875                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1876                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1877                 String vpnName = currVpnIntf.getVpnInstanceName();
1878                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1879                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1880                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1881                         LogicalDatastoreType.OPERATIONAL, adjPath);
1882                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1883                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1884                 long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1885                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1886                 List<Adjacency> adjacencies = new ArrayList<>();
1887                 if (optAdjacencies.isPresent() && optAdjacencies.get().getAdjacency() != null) {
1888                     adjacencies.addAll(optAdjacencies.get().getAdjacency());
1889                 }
1890                 long vpnId = vpnUtil.getVpnId(vpnName);
1891                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1892                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1893                 Adjacency operationalAdjacency = null;
1894                 //Handling dual stack neutron port primary adjacency
1895                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1896                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1897                             currVpnIntf.getName(), vpnName);
1898                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1899                             currVpnIntf.getName());
1900                     if (interfaceState != null) {
1901                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1902                                 currVpnIntf.getName(),
1903                                 vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
1904                     }
1905                 }
1906                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1907                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1908                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
1909                             : RouteOrigin.STATIC;
1910                     String nh = adj.getNextHopIpList().get(0);
1911                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1912                     synchronized (vpnPrefixKey.intern()) {
1913                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1914                                 vpnId, null, prefix, vpnName, nh, dpnId);
1915                         if (rdToAllocate.isPresent()) {
1916                             input.setRd(rdToAllocate.get());
1917                             operationalAdjacency = populator.createOperationalAdjacency(input);
1918                             int label = operationalAdjacency.getLabel().intValue();
1919                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1920                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
1921                                     currVpnIntf.getName(), operationalAdjacency, encapType, writeConfigTxn);
1922                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1923                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1924                                     vpnName, label, currVpnIntf.getName(), dpnId);
1925                         } else {
1926                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
1927                                     vpnName, prefix);
1928                             return;
1929                         }
1930                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1931                         // Keeping the MPLS check for now.
1932                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1933                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1934                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1935                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
1936                             vpnsToImportRoute.forEach(vpn -> {
1937                                 if (vpn.getVrfId() != null) {
1938                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(), vpnId, prefix,
1939                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
1940                                             .ifPresent(
1941                                                 rds -> vpnManager.addExtraRoute(
1942                                                         vpnUtil.getVpnName(vpn.getVpnId()),
1943                                                         adj.getIpAddress(), nh, rds,
1944                                                         currVpnIntf.getVpnInstanceName(), l3vni,
1945                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(),
1946                                                         opAdjacency, encapType, writeConfigTxn));
1947                                 }
1948                             });
1949                         }
1950                     }
1951                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1952                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1953                             currVpnIntf.getName(), vpnName);
1954
1955                     InstanceIdentifier<VpnInterface> vpnIfaceConfigidentifier = VpnUtil
1956                             .getVpnInterfaceIdentifier(currVpnIntf.getName());
1957                     Optional<VpnInterface> vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1958                             LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier);
1959                     Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1960                             Prefixes.PrefixCue.PhysNetFunc);
1961                     if (vpnIntefaceConfig.isPresent()) {
1962                         pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1963                                 vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(),
1964                                 vpnIntefaceConfig.get().getSegmentationId(), Prefixes.PrefixCue.PhysNetFunc);
1965                     }
1966
1967                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1968
1969                     writeOperTxn.merge(
1970                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
1971                                     prefix), pnfPrefix, true);
1972
1973                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1974                             adj.getIpAddress(), emptyList(), null /* EncapType */, 0 /* label */,
1975                             0 /*l3vni*/, null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1976
1977                     input.setRd(adj.getVrfId());
1978                 }
1979                 if (operationalAdjacency == null) {
1980                     operationalAdjacency = populator.createOperationalAdjacency(input);
1981                 }
1982                 adjacencies.add(operationalAdjacency);
1983                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1984                 VpnInterfaceOpDataEntry newVpnIntf =
1985                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
1986                                 aug, dpnId, currVpnIntf.getLportTag(),
1987                                 currVpnIntf.getGatewayMacAddress());
1988
1989                 writeOperTxn.merge(identifier, newVpnIntf, CREATE_MISSING_PARENTS);
1990             }
1991         } catch (ReadFailedException e) {
1992             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
1993                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
1994         }
1995     }
1996
1997     @Nullable
1998     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
1999         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
2000         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
2001     }
2002
2003     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
2004                                             BigInteger dpnId, TypedWriteTransaction<Operational> writeOperTxn,
2005                                             TypedWriteTransaction<Configuration> writeConfigTxn) {
2006         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
2007         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
2008         try {
2009             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
2010                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
2011             if (optVpnInterface.isPresent()) {
2012                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
2013                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
2014                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2015                         LogicalDatastoreType.OPERATIONAL, path);
2016                 if (optAdjacencies.isPresent()) {
2017                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
2018
2019                     if (adjacencies != null && !adjacencies.isEmpty()) {
2020                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
2021                         for (Adjacency adjacency : adjacencies) {
2022                             if (Objects.equals(adjacency.getIpAddress(), adj.getIpAddress())) {
2023                                 String rd = adjacency.getVrfId();
2024                                 if (adj.getNextHopIpList() != null) {
2025                                     for (String nh : adj.getNextHopIpList()) {
2026                                         deleteExtraRouteFromCurrentAndImportingVpns(
2027                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
2028                                                 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
2029                                     }
2030                                 } else if (adj.isPhysNetworkFunc()) {
2031                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
2032                                             adj.getIpAddress(), adj.getSubnetId());
2033                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
2034                                             writeConfigTxn);
2035                                 }
2036                                 break;
2037                             }
2038
2039                         }
2040                     }
2041                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
2042                             dpnId, adj.getVrfId());
2043                 } else {
2044                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
2045                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
2046                 }
2047             }
2048         } catch (ReadFailedException e) {
2049             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
2050                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
2051         }
2052     }
2053
2054     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
2055                                     String rd, String intfName, TypedWriteTransaction<Configuration> writeConfigTxn,
2056                                     TypedWriteTransaction<Operational> writeOperTx) {
2057         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
2058         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
2059         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
2060             String vpnRd = vpn.getVrfId();
2061             if (vpnRd != null) {
2062                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
2063                         writeOperTx);
2064             }
2065         }
2066     }
2067
2068     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
2069         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2070             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
2071             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
2072     }
2073
2074     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
2075         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2076             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
2077     }
2078
2079     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2080                                                 TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2081         if (vpnInterface == null) {
2082             return;
2083         }
2084         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
2085         if (adjs == null) {
2086             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2087                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2088             return;
2089         }
2090         for (Adjacency adj : adjs) {
2091             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2092                 String primaryInterfaceIp = adj.getIpAddress();
2093                 String macAddress = adj.getMacAddress();
2094                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2095
2096                 long label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
2097                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2098
2099                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2100                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2101                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
2102                         routerInt, label, writeConfigTxn);
2103                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2104                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2105                         macAddress);
2106                 return;
2107             }
2108         }
2109         LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2110                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}", interfaceName,
2111                 primaryRd, vpnName);
2112     }
2113
2114     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
2115             TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2116         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
2117         String rd = vpnUtil.getVpnRd(vpnName);
2118         if (adjs != null) {
2119             List<Adjacency> adjsList = requireNonNullElse(adjs.getAdjacency(), emptyList());
2120             for (Adjacency adj : adjsList) {
2121                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2122                     String primaryInterfaceIp = adj.getIpAddress();
2123                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2124                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
2125                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2126                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2127                     return;
2128                 }
2129             }
2130         } else {
2131             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2132                     vpnInterface.getName(), rd);
2133         }
2134     }
2135
2136     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2137         if (!canHandleNewVpnInterface(intefaceData.identifier, intefaceData.vpnInterface, vpnName)) {
2138             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
2139                   vpnName, intefaceData.vpnInterface.getName());
2140             return;
2141         }
2142         final VpnInterfaceKey key = intefaceData.identifier
2143                .firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
2144         final String interfaceName = key.getName();
2145         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2146                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2147         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2148                   intefaceData.identifier, vpnName);
2149     }
2150
2151     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2152                                               VpnInterface vpnInterface, String vpnName) {
2153         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2154                .get(vpnName);
2155         if (vpnInterfaces == null) {
2156             vpnInterfaces = new ConcurrentLinkedQueue<>();
2157         }
2158         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2159         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2160         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2161                 vpnInterface.getName(), vpnName);
2162     }
2163
2164     public boolean isVpnInstanceReady(String vpnInstanceName) {
2165         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2166         if (vpnRd == null) {
2167             return false;
2168         }
2169         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2170
2171         return vpnInstanceOpDataEntry != null;
2172     }
2173
2174     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2175         synchronized (vpnInstanceName.intern()) {
2176             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2177                     unprocessedVpnInterfaces.get(vpnInstanceName);
2178             if (vpnInterfaces != null) {
2179                 while (!vpnInterfaces.isEmpty()) {
2180                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2181                     if (hasVpnInstanceCreatedSuccessfully) {
2182                         processSavedInterface(savedInterface, vpnInstanceName);
2183                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2184                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2185                     } else {
2186                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2187                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2188                     }
2189                 }
2190             } else {
2191                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2192             }
2193         }
2194     }
2195
2196     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2197             VpnInterface vpnInterface) {
2198         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2199             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2200                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2201             if (vpnInterfaces != null) {
2202                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2203                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2204                             + "unprocessed list", vpnInterface.getName(),
2205                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2206                 }
2207             } else {
2208                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2209                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2210             }
2211         }
2212     }
2213
2214     public void vpnInstanceIsReady(String vpnInstanceName) {
2215         processSavedInterfaces(vpnInstanceName, true);
2216     }
2217
2218     public void vpnInstanceFailed(String vpnInstanceName) {
2219         processSavedInterfaces(vpnInstanceName, false);
2220     }
2221
2222     private static class UnprocessedVpnInterfaceData {
2223         InstanceIdentifier<VpnInterface> identifier;
2224         VpnInterface vpnInterface;
2225
2226         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2227             this.identifier = identifier;
2228             this.vpnInterface = vpnInterface;
2229         }
2230
2231         @Override
2232         public int hashCode() {
2233             final int prime = 31;
2234             int result = 1;
2235             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2236             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2237             return result;
2238         }
2239
2240         @Override
2241         public boolean equals(Object obj) {
2242             if (this == obj) {
2243                 return true;
2244             }
2245             if (obj == null) {
2246                 return false;
2247             }
2248             if (getClass() != obj.getClass()) {
2249                 return false;
2250             }
2251             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2252             if (identifier == null) {
2253                 if (other.identifier != null) {
2254                     return false;
2255                 }
2256             } else if (!identifier.equals(other.identifier)) {
2257                 return false;
2258             }
2259             if (vpnInterface == null) {
2260                 if (other.vpnInterface != null) {
2261                     return false;
2262                 }
2263             } else if (!vpnInterface.equals(other.vpnInterface)) {
2264                 return false;
2265             }
2266             return true;
2267         }
2268     }
2269
2270     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2271         String primaryRd = vpnUtil.getVpnRd(vpnName);
2272         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2273         if (vpnInstanceOpData == null) {
2274             return;
2275         }
2276         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2277         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2278             return;
2279         }
2280         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2281         vpnToDpnLists.forEach(vpnToDpnList -> {
2282             if (vpnToDpnList.getVpnInterfaces() == null) {
2283                 return;
2284             }
2285             vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2286                 try {
2287                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2288                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2289                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2290                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2291                     if (!vpnInterfaceOptional.isPresent()) {
2292                         return;
2293                     }
2294                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2295                             vpnInterface.getInterfaceName());
2296                     if (configVpnAdjacencies == null) {
2297                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2298                         return;
2299                     }
2300                     List<Adjacency> operationVpnAdjacencies = requireNonNullElse(vpnInterfaceOptional.get()
2301                             .augmentation(AdjacenciesOp.class).getAdjacency(), emptyList());
2302                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2303                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2304                     // in operational DS. These unprocessed adjacencies will be handled below.
2305                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2306                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2307                     configVpnAdjacencies.stream()
2308                         .filter(adjacency -> operationVpnAdjacencies.stream()
2309                                 .noneMatch(operationalAdjacency ->
2310                                     Objects.equals(operationalAdjacency.getIpAddress(), adjacency.getIpAddress())))
2311                         .forEach(adjacency -> {
2312                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2313                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2314                                 () -> {
2315                                     // TODO Deal with sequencing — the config tx must only submitted
2316                                     // if the oper tx goes in
2317                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2318                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2319                                         futures.add(
2320                                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx ->
2321                                                 futures.add(
2322                                                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
2323                                                         confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2324                                                             primaryRd, adjacency, vpnInterfaceOptional.get()
2325                                                                 .getDpnId(), operTx, confTx, confTx)))));
2326                                         return futures;
2327                                     } else {
2328                                         return emptyList();
2329                                     }
2330                                 });
2331                         });
2332                 } catch (ReadFailedException e) {
2333                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2334                             vpnName, primaryRd);
2335                 }
2336             });
2337         });
2338     }
2339
2340     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2341         private final String interfaceName;
2342         private final boolean add;
2343         private final String txnDestination;
2344
2345         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2346             this.interfaceName = interfaceName;
2347             this.add = add;
2348             this.txnDestination = transactionDest;
2349         }
2350
2351         @Override
2352         public void onSuccess(Void voidObj) {
2353             if (add) {
2354                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2355                         interfaceName, txnDestination);
2356             } else {
2357                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2358             }
2359         }
2360
2361         @Override
2362         public void onFailure(Throwable throwable) {
2363             if (add) {
2364                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2365                         interfaceName, txnDestination, throwable);
2366             } else {
2367                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2368                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2369             }
2370         }
2371     }
2372 }