vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14
  15 import com.google.common.base.Optional;
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.collect.Iterators;
  18 import com.google.common.util.concurrent.FutureCallback;
  19 import com.google.common.util.concurrent.Futures;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import com.google.common.util.concurrent.MoreExecutors;
  22 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  23 import java.math.BigInteger;
  24 import java.util.ArrayList;
  25 import java.util.Collections;
  26 import java.util.HashSet;
  27 import java.util.List;
  28 import java.util.Map;
  29 import java.util.Objects;
  30 import java.util.Set;
  31 import java.util.concurrent.ConcurrentHashMap;
  32 import java.util.concurrent.ConcurrentLinkedQueue;
  33 import java.util.concurrent.ExecutionException;
  34 import java.util.function.Consumer;
  35 import java.util.function.Predicate;
  36 import java.util.stream.Collectors;
  37 import javax.annotation.PostConstruct;
  38 import javax.annotation.PreDestroy;
  39 import javax.inject.Inject;
  40 import javax.inject.Singleton;
  41 import org.eclipse.jdt.annotation.Nullable;
  42 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  43 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  44 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  45 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  46 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  47 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  48 import org.opendaylight.genius.infra.Datastore.Configuration;
  49 import org.opendaylight.genius.infra.Datastore.Operational;
  50 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  51 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  52 import org.opendaylight.genius.infra.TransactionAdapter;
  53 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  54 import org.opendaylight.genius.infra.TypedWriteTransaction;
  55 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  56 import org.opendaylight.genius.mdsalutil.NWUtil;
  57 import org.opendaylight.genius.mdsalutil.NwConstants;
  58 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  59 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  60 import org.opendaylight.infrautils.caches.CacheProvider;
  61 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  62 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  63 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  64 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  65 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  66 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  67 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  68 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  69 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  70 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  71 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  72 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  73 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  74 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  75 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  76 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  77 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  78 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  79 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 119 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 120 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 121 import org.slf4j.Logger;
 122 import org.slf4j.LoggerFactory;
 123
 124 @Singleton
 125 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 126
 127     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 128     private static final short DJC_MAX_RETRIES = 3;
 129
 130     private final DataBroker dataBroker;
 131     private final ManagedNewTransactionRunner txRunner;
 132     private final IBgpManager bgpManager;
 133     private final IFibManager fibManager;
 134     private final IMdsalApiManager mdsalManager;
 135     private final IdManagerService idManager;
 136     private final OdlInterfaceRpcService ifaceMgrRpcService;
 137     private final VpnFootprintService vpnFootprintService;
 138     private final IInterfaceManager interfaceManager;
 139     private final IVpnManager vpnManager;
 140     private final ArpResponderHandler arpResponderHandler;
 141     private final JobCoordinator jobCoordinator;
 142     private final VpnUtil vpnUtil;
 143
 144     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 145
 146     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 147             new ConcurrentHashMap<>();
 148
 149     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 150
 151     @Inject
 152     public VpnInterfaceManager(final DataBroker dataBroker,
 153                                final IBgpManager bgpManager,
 154                                final IdManagerService idManager,
 155                                final IMdsalApiManager mdsalManager,
 156                                final IFibManager fibManager,
 157                                final OdlInterfaceRpcService ifaceMgrRpcService,
 158                                final VpnFootprintService vpnFootprintService,
 159                                final IInterfaceManager interfaceManager,
 160                                final IVpnManager vpnManager,
 161                                final ArpResponderHandler arpResponderHandler,
 162                                final JobCoordinator jobCoordinator,
 163                                final CacheProvider cacheProvider,
 164                                final VpnUtil vpnUtil) {
 165         super(VpnInterface.class, VpnInterfaceManager.class);
 166
 167         this.dataBroker = dataBroker;
 168         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 169         this.bgpManager = bgpManager;
 170         this.idManager = idManager;
 171         this.mdsalManager = mdsalManager;
 172         this.fibManager = fibManager;
 173         this.ifaceMgrRpcService = ifaceMgrRpcService;
 174         this.vpnFootprintService = vpnFootprintService;
 175         this.interfaceManager = interfaceManager;
 176         this.vpnManager = vpnManager;
 177         this.arpResponderHandler = arpResponderHandler;
 178         this.jobCoordinator = jobCoordinator;
 179         this.vpnUtil = vpnUtil;
 180
 181         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 182                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 183                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 184     }
 185
 186     public Runnable isNotifyTaskQueued(String intfName) {
 187         return vpnIntfMap.remove(intfName);
 188     }
 189
 190     @PostConstruct
 191     public void start() {
 192         LOG.info("{} start", getClass().getSimpleName());
 193         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 194     }
 195
 196     @Override
 197     @PreDestroy
 198     public void close() {
 199         super.close();
 200         vpnInstanceOpDataEntryCache.close();
 201     }
 202
 203     @Override
 204     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 205         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 206     }
 207
 208     @Override
 209     protected VpnInterfaceManager getDataTreeChangeListener() {
 210         return VpnInterfaceManager.this;
 211     }
 212
 213     @Override
 214     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 215         LOG.trace("Received VpnInterface add event: vpnInterface={}", vpnInterface);
 216         LOG.info("add: intfName {} onto vpnName {}", vpnInterface.getName(),
 217                 VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 218         addVpnInterface(identifier, vpnInterface, null, null);
 219     }
 220
 221     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 222                           final VpnInterface vpnInterface, String vpnName) {
 223         synchronized (vpnName.intern()) {
 224             if (isVpnInstanceReady(vpnName)) {
 225                 return true;
 226             }
 227             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 228             return false;
 229         }
 230     }
 231
 232     // TODO Clean up the exception handling
 233     @SuppressWarnings("checkstyle:IllegalCatch")
 234     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 235                              final @Nullable List<Adjacency> oldAdjs, final @Nullable List<Adjacency> newAdjs) {
 236         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
 237             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 238             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 239         }
 240     }
 241
 242     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 243                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 244         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 245         final String interfaceName = key.getName();
 246
 247         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 248             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 249                   vpnName, vpnInterface.getName());
 250             return;
 251         }
 252         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 253              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 254         List<Adjacency> copyOldAdjs = null;
 255         if (oldAdjs != null) {
 256             copyOldAdjs = new ArrayList<>();
 257             copyOldAdjs.addAll(oldAdjs);
 258         }
 259         List<Adjacency> copyNewAdjs = null;
 260         if (newAdjs != null) {
 261             copyNewAdjs = new ArrayList<>();
 262             copyNewAdjs.addAll(newAdjs);
 263         }
 264         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 265     }
 266
 267     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 268                     final VpnInterface vpnInterface, final @Nullable List<Adjacency> oldAdjs,
 269                     final @Nullable List<Adjacency> newAdjs,
 270                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 271         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 272         final String interfaceName = key.getName();
 273         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 274         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 275             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 276             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 277             if (interfaceState != null) {
 278                 try {
 279                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 280                     final int ifIndex = interfaceState.getIfIndex();
 281                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 282                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 283                         // (the inventory tx goes in last)
 284                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 285                         //set of prefix used, as entry in prefix-to-interface datastore
 286                         // is prerequisite for refresh Fib to avoid race condition leading to
 287                         // missing remote next hop in bucket actions on bgp-vpn delete
 288                         Set<String> prefixListForRefreshFib = new HashSet<>();
 289                         ListenableFuture<Void> confFuture =
 290                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 291                                 confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
 292                                     operTx -> futures.add(
 293                                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, invTx -> {
 294                                             LOG.info(
 295                                                 "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 296                                                     + " on dpn {}",
 297                                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 298                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 299                                                 confTx, operTx, invTx, interfaceState, vpnName,
 300                                                 prefixListForRefreshFib);
 301                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 302                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 303                                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 304                                                 if (newAdjs != null) {
 305                                                     for (Adjacency adj : newAdjs) {
 306                                                         if (oldAdjs.contains(adj)) {
 307                                                             oldAdjs.remove(adj);
 308                                                         } else {
 309                                                             if (!isBgpVpnInternetVpn
 310                                                                 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 311                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 312                                                                     primaryRd, adj, dpnId, operTx, confTx, invTx,
 313                                                                     prefixListForRefreshFib);
 314                                                             }
 315                                                         }
 316                                                     }
 317                                                 }
 318                                                 for (Adjacency adj : oldAdjs) {
 319                                                     if (!isBgpVpnInternetVpn
 320                                                         || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 321                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 322                                                             operTx, confTx);
 323                                                     }
 324                                                 }
 325                                             }
 326                                         })))));
 327                         Futures.addCallback(confFuture,
 328                             new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
 329                             MoreExecutors.directExecutor());
 330                         futures.add(confFuture);
 331                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 332                             MoreExecutors.directExecutor());
 333                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 334                             + " processed successfully", interfaceName, vpnName, dpnId);
 335                         return futures;
 336                     });
 337                 } catch (NumberFormatException | IllegalStateException e) {
 338                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 339                             + "interface {}. Interface addition on vpn {} failed", interfaceName,
 340                         vpnName, e);
 341                     return;
 342                 }
 343             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 344                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 345                     () -> {
 346                         ListenableFuture<Void> future =
 347                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
 348                                 createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 349                                     confTx, vpnName);
 350                                 LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 351                                     vpnName, vpnInterface.getDpnId());
 352                             });
 353                         ListenableFutures.addErrorLogging(future, LOG,
 354                             "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 355                         return Collections.singletonList(future);
 356                     });
 357             } else {
 358                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 359                     + " is not available", interfaceName, vpnName);
 360             }
 361         } else {
 362             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 363                     + " as vpn is pending delete", interfaceName, vpnName,
 364                 vpnInterface.getDpnId());
 365         }
 366     }
 367
 368     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 369     // see comments below.
 370     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 371     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 372             final int lportTag, boolean isInterfaceUp,
 373             TypedWriteTransaction<Configuration> writeConfigTxn,
 374             TypedWriteTransaction<Operational> writeOperTxn,
 375             TypedReadWriteTransaction<Configuration> writeInvTxn,
 376             Interface interfaceState, final String vpnName,
 377             Set<String> prefixListForRefreshFib) throws ExecutionException, InterruptedException {
 378         final String interfaceName = vpnInterface.getName();
 379         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 380                 vpnName);
 381         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 382         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 383         if (!isInterfaceUp) {
 384             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 385                      interfaceName, dpId, vpnName);
 386             long vpnId = vpnUtil.getVpnId(vpnName);
 387             if (vpnId == VpnConstants.INVALID_ID) {
 388                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 389                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 390                         dpId);
 391                 return;
 392             }
 393
 394             boolean waitForVpnInterfaceOpRemoval = false;
 395             if (opVpnInterface != null) {
 396                 String opVpnName = opVpnInterface.getVpnInstanceName();
 397                 String primaryInterfaceIp = null;
 398                 if (Objects.equals(opVpnName, vpnName)) {
 399                     // Please check if the primary VRF Entry does not exist for VPNInterface
 400                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 401                     // back to back
 402                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 403                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 404                     if (adjs == null) {
 405                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 406                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 407                                 vpnName);
 408                         return;
 409                     }
 410                     for (Adjacency adj : adjs) {
 411                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 412                             primaryInterfaceIp = adj.getIpAddress();
 413                             break;
 414                         }
 415                     }
 416                     if (primaryInterfaceIp == null) {
 417                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 418                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 419                                 dpId, vpnName);
 420                         return;
 421                     }
 422                     // Get the rd of the vpn instance
 423                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 424                     if (vrf != null) {
 425                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 426                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 427                         return;
 428                     }
 429                     waitForVpnInterfaceOpRemoval = true;
 430                 } else {
 431                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 432                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 433                 }
 434             }
 435             if (!waitForVpnInterfaceOpRemoval) {
 436                 // Add the VPNInterface and quit
 437                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 438                         null/*ipAddressSourceValuePair*/,
 439                         true /* add */);
 440                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 441                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 442                 if (!isBgpVpnInternetVpn) {
 443                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 444                 }
 445                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 446                         dpId, vpnName);
 447                 if (interfaceManager.isExternalInterface(interfaceName)) {
 448                     processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
 449                         NwConstants.ADD_FLOW);
 450                 }
 451                 return;
 452             }
 453
 454             // FIB didn't get a chance yet to clean up this VPNInterface
 455             // Let us give it a chance here !
 456             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 457                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 458             try {
 459                 Runnable notifyTask = new VpnNotifyTask();
 460                 synchronized (notifyTask) {
 461                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 462                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 463                     // published here while holding the lock on it so this path will hit the wait before notify can be
 464                     // invoked.
 465                     vpnIntfMap.put(interfaceName, notifyTask);
 466                     try {
 467                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 468                     } catch (InterruptedException e) {
 469                         // Ignored
 470                     }
 471                 }
 472             } finally {
 473                 vpnIntfMap.remove(interfaceName);
 474             }
 475
 476             if (opVpnInterface != null) {
 477                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 478                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 479                         dpId, vpnName);
 480                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 481                 return;
 482             }
 483             // VPNInterface got removed, proceed with Add
 484             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 485                     interfaceName, dpId, vpnName);
 486             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 487                     null/*ipAddressSourceValuePair*/,
 488                     true /* add */);
 489             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 490                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 491             if (!isBgpVpnInternetVpn) {
 492                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 493             }
 494             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 495                     + " FIB to clean up", interfaceName, dpId, vpnName);
 496             if (interfaceManager.isExternalInterface(interfaceName)) {
 497                 processExternalVpnInterface(interfaceName, vpnName, dpId,
 498                                lportTag, NwConstants.ADD_FLOW);
 499             }
 500         } else {
 501             try {
 502                 // Interface is retained in the DPN, but its Link Up.
 503                 // Advertise prefixes again for this interface to BGP
 504                 InstanceIdentifier<VpnInterface> identifier =
 505                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 506                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 507                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 508                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 509                 // Perform similar operation as interface add event for extraroutes.
 510                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 511                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 512                         LogicalDatastoreType.CONFIGURATION, path);
 513                 if (!optAdjacencies.isPresent()) {
 514                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 515                     return;
 516                 }
 517                 List<Adjacency> adjacencies = optAdjacencies.get().nonnullAdjacency();
 518                 for (Adjacency adjacency : adjacencies) {
 519                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 520                         continue;
 521                     }
 522                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 523                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 524                         continue;
 525                     }
 526                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 527                             dpId, writeOperTxn, writeConfigTxn, writeInvTxn, prefixListForRefreshFib);
 528                 }
 529             } catch (ReadFailedException e) {
 530                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 531                         interfaceName, vpnName, primaryRd, dpId);
 532             }
 533         }
 534     }
 535
 536     private void processExternalVpnInterface(String interfaceName, String vpnName, BigInteger dpId,
 537         int lportTag, int addOrRemove) {
 538         Uuid extNetworkId;
 539         try {
 540             // vpn instance of ext-net interface is the network-id
 541             extNetworkId = new Uuid(vpnName);
 542         } catch (IllegalArgumentException e) {
 543             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 544                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 545             return;
 546         }
 547
 548         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 549         if (routerIds == null || routerIds.isEmpty()) {
 550             LOG.info("processExternalVpnInterface: No router is associated with {}."
 551                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 552                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 553             return;
 554         }
 555
 556         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 557                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 558         for (Uuid routerId : routerIds) {
 559             String routerName = routerId.getValue();
 560             BigInteger primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 561             if (Objects.equals(primarySwitch, dpId)) {
 562                 Routers router = vpnUtil.getExternalRouter(routerName);
 563                 if (router != null) {
 564                     if (addOrRemove == NwConstants.ADD_FLOW) {
 565                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 566                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 567                                 dpId, interfaceName, lportTag);
 568                     } else {
 569                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 570                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 571                                 dpId, interfaceName, lportTag);
 572                     }
 573                 } else {
 574                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 575                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 576                             interfaceName, dpId, vpnName);
 577                 }
 578             }
 579         }
 580     }
 581
 582     // TODO Clean up the exception handling
 583     @SuppressWarnings("checkstyle:IllegalCatch")
 584     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 585                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 586                                                   String vpnName, String interfaceName) {
 587         if (rd == null) {
 588             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 589                   interfaceName, dpnId, vpnName);
 590             return;
 591         } else {
 592             if (rd.equals(vpnName)) {
 593                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 594                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 595                 return;
 596             }
 597         }
 598         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 599                 interfaceName, dpnId, vpnName, rd);
 600
 601         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 602         if (nextHopIp == null) {
 603             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 604                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 605                     rd, vpnName);
 606             return;
 607         }
 608
 609         try {
 610             //Read NextHops
 611             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 612             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 613                     LogicalDatastoreType.OPERATIONAL, path);
 614             if (adjacencies.isPresent()) {
 615                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 616                 if (nextHops != null && !nextHops.isEmpty()) {
 617                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 618                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 619                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 620                     long l3vni = vpnInstanceOpData.getL3vni();
 621                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 622                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 623                     for (Adjacency nextHop : nextHops) {
 624                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 625                             continue;
 626                         }
 627                         String gatewayMac = null;
 628                         long label = 0;
 629                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 630                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 631                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 632                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 633                                     .get();
 634                         } else {
 635                             label = nextHop.getLabel();
 636                         }
 637                         try {
 638                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 639                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 640                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 641                                     encapType, (int)label, l3vni, 0 /*l2vni*/,
 642                                     gatewayMac);
 643                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 644                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 645                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 646                         } catch (Exception e) {
 647                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 648                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 649                                     interfaceName, dpnId, e);
 650                         }
 651                     }
 652                 }
 653             }
 654         } catch (ReadFailedException e) {
 655             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 656                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 657         }
 658     }
 659
 660     // TODO Clean up the exception handling
 661     @SuppressWarnings("checkstyle:IllegalCatch")
 662     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 663                         String vpnName, String interfaceName, TypedWriteTransaction<Configuration> writeConfigTxn,
 664                         TypedWriteTransaction<Operational> writeOperTx) {
 665         //Read NextHops
 666         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 667         String rd = vpnUtil.getVpnRd(interfaceName);
 668         if (rd == null) {
 669             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 670                 interfaceName, vpnName);
 671             return;
 672         } else {
 673             if (rd.equals(vpnName)) {
 674                 LOG.info(
 675                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 676                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 677                 return;
 678             }
 679         }
 680         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 681                vpnName, rd);
 682         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 683         try {
 684             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 685                     path);
 686         } catch (ReadFailedException e) {
 687             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 688                     interfaceName, vpnName);
 689         }
 690         if (adjacencies.isPresent()) {
 691             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 692
 693             if (nextHops != null && !nextHops.isEmpty()) {
 694                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 695                         nextHops, interfaceName, vpnName, rd);
 696                 for (Adjacency nextHop : nextHops) {
 697                     try {
 698                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 699                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 700                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 701                                     interfaceName, vpnName);
 702                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 703                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 704                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 705                                     interfaceName, vpnName);
 706                         } else if (nextHop.getNextHopIpList() != null) {
 707                             // Perform similar operation as interface delete event for extraroutes.
 708                             String allocatedRd = nextHop.getVrfId();
 709                             for (String nh : nextHop.getNextHopIpList()) {
 710                                 deleteExtraRouteFromCurrentAndImportingVpns(
 711                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
 712                                     writeOperTx);
 713                             }
 714                         }
 715                     } catch (Exception e) {
 716                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 717                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 718                     }
 719                 }
 720             }
 721         }
 722     }
 723
 724     @SuppressWarnings("checkstyle:IllegalCatch")
 725     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 726                                                   String primaryRd, String interfaceName, final long vpnId,
 727                                                   TypedWriteTransaction<Configuration> writeConfigTxn,
 728                                                   TypedWriteTransaction<Operational> writeOperTxn,
 729                                                   TypedReadWriteTransaction<Configuration> writeInvTxn,
 730                                                   Interface interfaceState, Set<String> prefixListForRefreshFib)
 731             throws ExecutionException, InterruptedException {
 732         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 733         // Read NextHops
 734         Optional<VpnInterface> vpnInteface = Optional.absent();
 735         try {
 736             vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 737             LogicalDatastoreType.CONFIGURATION, identifier);
 738         } catch (ReadFailedException e) {
 739             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 740                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 741         }
 742         Uuid intfnetworkUuid = null;
 743         NetworkType networkType = null;
 744         Long segmentationId = Long.valueOf(-1);
 745         Adjacencies adjacencies = null;
 746         if (vpnInteface.isPresent()) {
 747             intfnetworkUuid = vpnInteface.get().getNetworkId();
 748             networkType = vpnInteface.get().getNetworkType();
 749             segmentationId = vpnInteface.get().getSegmentationId();
 750             adjacencies = vpnInteface.get().augmentation(Adjacencies.class);
 751             if (adjacencies == null) {
 752                 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 753                         null/*gwMac*/, writeOperTxn);
 754                 return;
 755             }
 756         }
 757         // Get the rd of the vpn instance
 758         String nextHopIp = null;
 759         try {
 760             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 761         } catch (Exception e) {
 762             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 763                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 764         }
 765         List<String> nhList = new ArrayList<>();
 766         if (nextHopIp != null) {
 767             nhList.add(nextHopIp);
 768             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 769                     interfaceName, dpnId, vpnName, nhList);
 770         }
 771         Optional<String> gwMac = Optional.absent();
 772         String vpnInterfaceSubnetGwMacAddress = null;
 773         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 774         Long l3vni = vpnInstanceOpData.getL3vni();
 775         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 776         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 777         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 778         List<Adjacency> nextHops = (adjacencies != null) ? adjacencies.getAdjacency() : emptyList();
 779         List<Adjacency> value = new ArrayList<>();
 780         for (Adjacency nextHop : nextHops) {
 781             String rd = primaryRd;
 782             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 783             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 784                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 785                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 786                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 787                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 788                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 789                 continue;
 790             }
 791             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 792                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 793                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 794                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 795                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 796                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 797
 798                 Prefixes prefixes = (intfnetworkUuid != null)
 799                     ? VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, intfnetworkUuid ,networkType,
 800                             segmentationId, prefixCue) :
 801                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, prefixCue);
 802                 writeOperTxn.merge(VpnUtil.getPrefixToInterfaceIdentifier(
 803                         vpnUtil.getVpnId(vpnName), prefix), prefixes, true);
 804                 final Uuid subnetId = nextHop.getSubnetId();
 805
 806                 String gatewayIp = nextHop.getSubnetGatewayIp();
 807                 if (gatewayIp == null) {
 808                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 809                     if (gatewayIpOptional.isPresent()) {
 810                         gatewayIp = gatewayIpOptional.get();
 811                     }
 812                 }
 813
 814                 if (gatewayIp != null) {
 815                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 816                     if (gwMac.isPresent()) {
 817                         // A valid mac-address is available for this subnet-gateway-ip
 818                         // Use this for programming ARP_RESPONDER table here.  And save this
 819                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 820                         // to populate L3_GW_MAC_TABLE there.
 821                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 822                                 gatewayIp, gwMac.get());
 823                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 824                     } else {
 825                         // A valid mac-address is not available for this subnet-gateway-ip
 826                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 827                         // Save this connected-mac-address as gateway-mac-address for the
 828                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 829                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 830                         if (gwMac.isPresent()) {
 831                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 832                                     NwConstants.ADD_FLOW, gwMac.get());
 833                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 834                                     gatewayIp, gwMac.get());
 835                         } else {
 836                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 837                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 838                                 + "gwIp {}",
 839                                 subnetId, interfaceName, vpnName, gatewayIp);
 840                         }
 841                     }
 842                 } else {
 843                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 844                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 845                         subnetId, interfaceName, vpnName);
 846                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 847                 }
 848                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 849                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 850             } else {
 851                 //Extra route adjacency
 852                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 853                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 854                 synchronized (vpnPrefixKey.intern()) {
 855                     java.util.Optional<String> rdToAllocate = vpnUtil
 856                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 857                                     nextHop.getNextHopIpList().get(0), dpnId);
 858                     if (rdToAllocate.isPresent()) {
 859                         rd = rdToAllocate.get();
 860                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 861                             rd, prefix);
 862                     } else {
 863                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 864                         continue;
 865                     }
 866                 }
 867                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 868                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 869                         interfaceName, dpnId);
 870             }
 871             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 872             // can be different from the gateway-mac-address within the VRFEntry as the
 873             // gateway-mac-address is a superset.
 874             RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 875             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 876                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 877                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 878             Adjacency operationalAdjacency = null;
 879             try {
 880                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 881             } catch (NullPointerException e) {
 882                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 883                     input, e.getMessage());
 884                 return;
 885             }
 886             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 887                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 888                     vpnName, l3vni, origin, interfaceName, operationalAdjacency, encapType, prefixListForRefreshFib,
 889                     writeConfigTxn);
 890             }
 891             value.add(operationalAdjacency);
 892         }
 893
 894         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 895         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 896                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 897
 898         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 899                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 900                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 901
 902         for (Adjacency nextHop : aug.getAdjacency()) {
 903             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 904             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 905                 RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 906                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 907                 registeredPopulator.populateFib(input, writeConfigTxn);
 908             }
 909         }
 910     }
 911
 912     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 913                                               long lportTag, String gwMac,
 914                                               TypedWriteTransaction<Operational> writeOperTxn) {
 915         VpnInterfaceOpDataEntry opInterface =
 916               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac);
 917         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 918             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 919         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
 920         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 921                 interfaceName, dpnId, vpnName);
 922     }
 923
 924     // TODO Clean up the exception handling
 925     @SuppressWarnings("checkstyle:IllegalCatch")
 926     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 927                                            StateTunnelList stateTunnelList,
 928                                            TypedWriteTransaction<Configuration> writeConfigTxn,
 929                                            TypedWriteTransaction<Operational> writeOperTxn) {
 930
 931         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
 932         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 933         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 934         List<Adjacency> adjList =
 935             adjacencies != null && adjacencies.getAdjacency() != null ? adjacencies.getAdjacency() : emptyList();
 936         if (adjList.isEmpty()) {
 937             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 938                     vpnInterface, srcDpnId);
 939             return;
 940         }
 941         String prefix = null;
 942         long label = 0;
 943         List<Adjacency> value = new ArrayList<>();
 944         boolean isNextHopAddReqd = false;
 945         String vpnName = vpnInterface.getVpnInstanceName();
 946         long vpnId = vpnUtil.getVpnId(vpnName);
 947         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 948         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 949                 vpnInterface.getName(), vpnInterface.getDpnId(),
 950                 vpnInterface.getVpnInstanceName(), adjList);
 951         for (Adjacency adj : adjList) {
 952             String rd = adj.getVrfId();
 953             rd = rd != null ? rd : vpnName;
 954             prefix = adj.getIpAddress();
 955             label = adj.getLabel();
 956             List<String> nhList = Collections.singletonList(srcTepIp);
 957             List<String> nextHopList = adj.getNextHopIpList();
 958             // If TEP is added , update the nexthop of primary adjacency.
 959             // Secondary adj nexthop is already pointing to primary adj IP address.
 960             if (nextHopList == null || nextHopList.isEmpty()) {
 961                 isNextHopAddReqd = true;
 962             }
 963
 964             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 965                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 966             } else {
 967                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 968                 if (!vrfEntryOptional.isPresent()) {
 969                     continue;
 970                 }
 971                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 972                 if (!nhList.contains(srcTepIp)) {
 973                     nhList.add(srcTepIp);
 974                     isNextHopAddReqd = true;
 975                 }
 976                 value.add(adj);
 977             }
 978
 979             if (isNextHopAddReqd) {
 980                 updateLabelMapper(label, nhList);
 981                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 982                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 983                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 984                 // Update the VRF entry with nextHop
 985                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 986                         label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
 987
 988                 //Get the list of VPN's importing this route(prefix) .
 989                 // Then update the VRF entry with nhList
 990                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
 991                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 992                     String vpnRd = vpn.getVrfId();
 993                     if (vpnRd != null) {
 994                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
 995                             srcTepIp, label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
 996                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 997                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
 998                             vpnInterface.getName(), srcDpnId, vpnName,
 999                             vpn.getVpnInstanceName(), vpnRd);
1000                     }
1001                 }
1002                 // Advertise the prefix to BGP only for external vpn
1003                 // since there is a nexthop change.
1004                 try {
1005                     if (!rd.equalsIgnoreCase(vpnName)) {
1006                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
1007                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
1008                                 null /*gatewayMacAddress*/);
1009                     }
1010                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
1011                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
1012                             srcDpnId, vpnName);
1013                 } catch (Exception ex) {
1014                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
1015                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1016                             vpnInterface.getName(), srcDpnId, vpnName, ex);
1017                 }
1018             }
1019         }
1020         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1021         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1022                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1023                 .addAugmentation(AdjacenciesOp.class, aug).build();
1024         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1025                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1026         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1027         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
1028                 vpnInterface.getName(), srcDpnId, vpnName);
1029
1030     }
1031
1032     // TODO Clean up the exception handling
1033     @SuppressWarnings("checkstyle:IllegalCatch")
1034     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1035                                               StateTunnelList stateTunnelList,
1036                                               TypedWriteTransaction<Configuration> writeConfigTxn,
1037                                               TypedWriteTransaction<Operational> writeOperTxn) {
1038
1039         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1040         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1041         String prefix = null;
1042         long label = 0;
1043         boolean isNextHopRemoveReqd = false;
1044         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
1045         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
1046         String vpnName = vpnInterface.getVpnInstanceName();
1047         long vpnId = vpnUtil.getVpnId(vpnName);
1048         String primaryRd = vpnUtil.getVpnRd(vpnName);
1049         if (adjList != null) {
1050             List<Adjacency> value = new ArrayList<>();
1051             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1052                     vpnInterface.getName(), vpnInterface.getDpnId(),
1053                     vpnInterface.getVpnInstanceName(), adjList);
1054             for (Adjacency adj : adjList) {
1055                 List<String> nhList = new ArrayList<>();
1056                 String rd = adj.getVrfId();
1057                 rd = rd != null ? rd : vpnName;
1058                 prefix = adj.getIpAddress();
1059                 List<String> nextHopList = adj.getNextHopIpList();
1060                 label = adj.getLabel();
1061                 if (nextHopList != null && !nextHopList.isEmpty()) {
1062                     isNextHopRemoveReqd = true;
1063                 }
1064                 // If TEP is deleted , remove the nexthop from primary adjacency.
1065                 // Secondary adj nexthop will continue to point to primary adj IP address.
1066                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1067                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1068                 } else {
1069                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1070                     if (!vrfEntryOptional.isPresent()) {
1071                         continue;
1072                     }
1073                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1074                     if (nhList.contains(srcTepIp)) {
1075                         nhList.remove(srcTepIp);
1076                         isNextHopRemoveReqd = true;
1077                     }
1078                     value.add(adj);
1079                 }
1080
1081                 if (isNextHopRemoveReqd) {
1082                     updateLabelMapper(label, nhList);
1083                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1084                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1085                             prefix, nhList, vpnName,
1086                             vpnId, rd, vpnInterface.getName());
1087                     // Update the VRF entry with removed nextHop
1088                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1089                             label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1090
1091                     //Get the list of VPN's importing this route(prefix) .
1092                     // Then update the VRF entry with nhList
1093                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1094                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1095                         String vpnRd = vpn.getVrfId();
1096                         if (vpnRd != null) {
1097                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1098                                 srcTepIp, label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1099                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1100                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1101                                     nhList, label, vpnInterface.getName(), srcDpnId,
1102                                     vpnName,
1103                                     vpn.getVpnInstanceName(), vpnRd);
1104                         }
1105                     }
1106
1107                     // Withdraw prefix from BGP only for external vpn.
1108                     try {
1109                         if (!rd.equalsIgnoreCase(vpnName)) {
1110                             bgpManager.withdrawPrefix(rd, prefix);
1111                         }
1112                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1113                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1114                                 vpnInterface.getName(), srcDpnId,
1115                                 vpnName);
1116                     } catch (Exception ex) {
1117                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1118                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1119                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1120                     }
1121                 }
1122             }
1123             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1124             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1125                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1126                     .addAugmentation(AdjacenciesOp.class, aug).build();
1127             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1128                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1129             writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1130             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1131                          vpnInterface.getName(), srcDpnId, vpnName);
1132         }
1133     }
1134
1135     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1136         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1137
1138         String vpnRd = vpnUtil.getVpnRd(vpnName);
1139         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1140         if (vpnInstanceOpDataEntry == null) {
1141             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1142                     + " to check for vpns exporting the routes", vpnName);
1143             return vpnsToExportRoute;
1144         }
1145
1146         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1147             if (input.getVpnInstanceName() == null) {
1148                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1149                         input.getVrfId());
1150                 return false;
1151             }
1152             return !input.getVpnInstanceName().equals(vpnName);
1153         };
1154
1155         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1156             Iterable<String> commonRTs =
1157                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1158                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1159             return Iterators.size(commonRTs.iterator()) > 0;
1160         };
1161
1162         vpnsToExportRoute =
1163                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1164                         Collectors.toList());
1165         return vpnsToExportRoute;
1166     }
1167
1168     // TODO Clean up the exception handling
1169     @SuppressWarnings("checkstyle:IllegalCatch")
1170     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1171         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1172         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1173             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1174             if (vrfEntries != null) {
1175                 ListenableFutures.addErrorLogging(
1176                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1177                         for (VrfEntry vrfEntry : vrfEntries) {
1178                             try {
1179                                 if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1180                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1181                                     LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1182                                             + " is not a controller managed non intervpn link route. Ignoring.",
1183                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1184                                     continue;
1185                                 }
1186                                 String prefix = vrfEntry.getDestPrefix();
1187                                 String gwMac = vrfEntry.getGatewayMacAddress();
1188                                 vrfEntry.nonnullRoutePaths().forEach(routePath -> {
1189                                     String nh = routePath.getNexthopAddress();
1190                                     int label = routePath.getLabel().intValue();
1191                                     if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1192                                         vrfEntry.getOrigin()))) {
1193                                         LOG.info(
1194                                             "handleVpnsExportingRoutesImporting: Importing fib entry rd {}"
1195                                                 + " prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1196                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1197                                         fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1198                                             Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1199                                             0 /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1200                                             confTx);
1201                                     } else {
1202                                         LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry"
1203                                                 + " rd {} prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1204                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1205                                         SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1206                                         importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1207                                             confTx);
1208                                     }
1209                                 });
1210                             } catch (RuntimeException e) {
1211                                 LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1212                                         + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1213                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1214                             }
1215                         }
1216                     }), LOG, "Error handing VPN exporting routes");
1217             } else {
1218                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1219                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1220             }
1221         }
1222     }
1223
1224     @Override
1225     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1226         LOG.trace("Received VpnInterface remove event: vpnInterface={}", vpnInterface);
1227         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1228         final String interfaceName = key.getName();
1229         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
1230             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1231             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1232         }
1233     }
1234
1235     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1236                                 final VpnInterface vpnInterface, final String vpnName,
1237                                 final String interfaceName) {
1238         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1239             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1240                 ListenableFuture<Void> future =
1241                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1242                         deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1243                         LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1244                     });
1245                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1246                         vpnInterface.getName(), vpnName);
1247                 return Collections.singletonList(future);
1248             }, DJC_MAX_RETRIES);
1249         } else {
1250             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1251             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1252         }
1253     }
1254
1255     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1256     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1257                                            final VpnInterface vpnInterface, final String vpnName,
1258                                            final String interfaceName, final Interface interfaceState) {
1259         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1260                 vpnName, vpnInterface.getDpnId());
1261         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1262         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1263             () -> {
1264                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1265                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1266                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1267                         writeOperTxn -> futures.add(
1268                             txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeInvTxn -> {
1269                                 LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1270                                         interfaceName, vpnName);
1271                                 BigInteger dpId;
1272                                 int ifIndex;
1273                                 String gwMacAddress;
1274                                 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1275                                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1276                                 Optional<VpnInterfaceOpDataEntry> optVpnInterface;
1277                                 try {
1278                                     optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1279                                             LogicalDatastoreType.OPERATIONAL, interfaceId);
1280                                 } catch (ReadFailedException e) {
1281                                     LOG.error("remove: Failed to read data store for interface {} vpn {}",
1282                                             interfaceName, vpnName);
1283                                     return;
1284                                 }
1285                                 if (interfaceState != null) {
1286                                     try {
1287                                         dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1288                                     } catch (NumberFormatException | IllegalStateException e) {
1289                                         LOG.error("remove: Unable to retrieve dpnId from interface operational"
1290                                                         + " data store for interface {} on dpn {} for vpn {} Fetching"
1291                                                         + " from vpn interface op data store. ", interfaceName,
1292                                                 vpnInterface.getDpnId(), vpnName, e);
1293                                         dpId = BigInteger.ZERO;
1294                                     }
1295                                     ifIndex = interfaceState.getIfIndex();
1296                                     gwMacAddress = interfaceState.getPhysAddress().getValue();
1297                                 } else {
1298                                     LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1299                                             + " from vpn interface op.", interfaceName);
1300                                     if (optVpnInterface.isPresent()) {
1301                                         VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1302                                         dpId = vpnOpInterface.getDpnId();
1303                                         ifIndex = vpnOpInterface.getLportTag().intValue();
1304                                         gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1305                                     } else {
1306                                         LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1307                                                 + " as interfaceState and vpn interface op is not"
1308                                                 + " available", interfaceName, vpnName);
1309                                         return;
1310                                     }
1311                                 }
1312                                 processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1313                                         optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1314                                         writeConfigTxn, writeOperTxn, writeInvTxn);
1315                                 LOG.info(
1316                                         "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1317                                                 + "successfully",
1318                                         interfaceName, vpnInterface.getDpnId(), vpnName);
1319                             })))));
1320                 futures.add(configFuture);
1321                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1322                 return futures;
1323             }, DJC_MAX_RETRIES);
1324     }
1325
1326     protected void processVpnInterfaceDown(BigInteger dpId,
1327                                            String interfaceName,
1328                                            int lportTag,
1329                                            String gwMac,
1330                                            VpnInterfaceOpDataEntry vpnOpInterface,
1331                                            boolean isInterfaceStateDown,
1332                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1333                                            TypedWriteTransaction<Operational> writeOperTxn,
1334                                            TypedReadWriteTransaction<Configuration> writeInvTxn)
1335             throws ExecutionException, InterruptedException {
1336         if (vpnOpInterface == null) {
1337             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1338                     + " as it is not available in operational data store", interfaceName, dpId);
1339             return;
1340         }
1341         final String vpnName = vpnOpInterface.getVpnInstanceName();
1342         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1343                                                     interfaceName, vpnName);
1344         if (!isInterfaceStateDown) {
1345             final long vpnId = vpnUtil.getVpnId(vpnName);
1346             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName,  null);
1347             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1348             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1349                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1350             if (interfaceManager.isExternalInterface(interfaceName)) {
1351                 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1352                     NwConstants.DEL_FLOW);
1353             }
1354             if (!isBgpVpnInternetVpn) {
1355                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1356             }
1357             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1358                     + " successful", interfaceName, dpId, vpnName);
1359         } else {
1360             // Interface is retained in the DPN, but its Link Down.
1361             // Only withdraw the prefixes for this interface from BGP
1362             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1363         }
1364     }
1365
1366     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1367                                           final String vpnName, final long vpnId, String gwMac,
1368                                           TypedWriteTransaction<Configuration> writeConfigTxn,
1369                                           TypedWriteTransaction<Operational> writeOperTxn,
1370                                           TypedReadWriteTransaction<Configuration> writeInvTxn)
1371             throws ExecutionException, InterruptedException {
1372         //Read NextHops
1373         try {
1374             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1375                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1376             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1377             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1378                     LogicalDatastoreType.OPERATIONAL, path);
1379             boolean isNonPrimaryAdjIp = Boolean.FALSE;
1380             String primaryRd = vpnUtil.getVpnRd(vpnName);
1381             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1382                     interfaceName, dpnId, vpnName, primaryRd);
1383             if (adjacencies.isPresent() && adjacencies.get().getAdjacency() != null
1384                     && !adjacencies.get().getAdjacency().isEmpty()) {
1385                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1386                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1387                         interfaceName, dpnId, vpnName, nextHops);
1388                 for (Adjacency nextHop : nextHops) {
1389                     if (nextHop.isPhysNetworkFunc()) {
1390                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1391                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1392                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1393                                 null/*writeCfgTxn*/);
1394                     } else {
1395                         String rd = nextHop.getVrfId();
1396                         List<String> nhList;
1397                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1398                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1399                             isNonPrimaryAdjIp = Boolean.TRUE;
1400                         } else {
1401                             // This is a primary adjacency
1402                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1403                                     : emptyList();
1404                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1405                                     interfaceName, writeInvTxn);
1406                         }
1407                         if (!nhList.isEmpty()) {
1408                             if (Objects.equals(primaryRd, vpnName)) {
1409                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1410                                 //remove from FIB directly
1411                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1412                                         interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1413                             } else {
1414                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1415                                         interfaceName, writeConfigTxn, writeOperTxn);
1416                             }
1417                         } else {
1418                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1419                                             + " interface {}", nextHop.getIpAddress(), rd,
1420                                     nextHop.getAdjacencyType().toString(), interfaceName);
1421                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1422                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1423                         }
1424                     }
1425                     String ip = nextHop.getIpAddress().split("/")[0];
1426                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1427                     if (vpnVipToPort != null && vpnVipToPort.getPortName().equals(interfaceName)) {
1428                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1429                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1430                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1431                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1432                     }
1433                     // Remove the MIP-IP from VpnPortIpToPort.
1434                     if (isNonPrimaryAdjIp) {
1435                         VpnPortipToPort persistedIp = vpnUtil.getVpnPortipToPort(vpnName, ip);
1436                         if (persistedIp != null && persistedIp.isLearntIp()
1437                                 && persistedIp.getPortName().equals(interfaceName)) {
1438                             VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1439                             LOG.info(
1440                                     "removeAdjacenciesFromVpn: Learnt-IP: {} interface {} of vpn {} removed "
1441                                             + "from VpnPortipToPort",
1442                                     persistedIp.getPortFixedip(), persistedIp.getPortName(), vpnName);
1443                         }
1444                     }
1445                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1446                     if (vpnPortipToPort != null) {
1447                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1448                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1449                                  + "Interface {} ip {} on dpn {} for vpn {}",
1450                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1451                     }
1452                 }
1453             } else {
1454                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1455                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1456                         + " Removing it.", interfaceName, vpnName, dpnId);
1457                 writeOperTxn.delete(identifier);
1458             }
1459         } catch (ReadFailedException e) {
1460             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1461                     interfaceName, dpnId, vpnName);
1462         }
1463     }
1464
1465     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1466                                                             String interfaceName, BigInteger dpnId,
1467                                                             TypedWriteTransaction<Configuration> writeConfigTxn,
1468                                                             TypedWriteTransaction<Operational> writeOperTx) {
1469         return (nh) -> {
1470             String primaryRd = vpnUtil.getVpnRd(vpnName);
1471             String prefix = nextHop.getIpAddress();
1472             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1473             LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1474                     nextHop, vpnName, interfaceName, dpnId);
1475             synchronized (vpnNamePrefixKey.intern()) {
1476                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1477                         prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1478                     //If extra-route is present behind at least one VM, then do not remove or update
1479                     //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1480                     //prefixtointerface DS
1481                     return;
1482                 }
1483                 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1484                         writeConfigTxn);
1485             }
1486             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1487                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1488                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1489         };
1490     }
1491
1492     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1493                                            BigInteger dpnId, String rd, String interfaceName,
1494                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1495                                            TypedWriteTransaction<Operational> writeOperTx) {
1496         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1497                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1498         nhList.forEach((nh) -> {
1499             //IRT: remove routes from other vpns importing it
1500             vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1501                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1502             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1503                 String vpnRd = vpn.getVrfId();
1504                 if (vpnRd != null) {
1505                     fibManager.removeOrUpdateFibEntry(vpnRd,
1506                             nextHop.getIpAddress(), nh, writeConfigTxn);
1507                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1508                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1509                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1510                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1511                 }
1512             }
1513         });
1514     }
1515
1516     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1517                                                  int lportTag, String gwMac, String interfaceName,
1518                                                  TypedReadWriteTransaction<Configuration> writeInvTxn)
1519             throws ExecutionException, InterruptedException {
1520         final Uuid subnetId = nextHop.getSubnetId();
1521         if (nextHop.getSubnetGatewayMacAddress() == null) {
1522             // A valid mac-address was not available for this subnet-gateway-ip
1523             // So a connected-mac-address was used for this subnet and we need
1524             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1525             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1526         }
1527         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1528                 subnetId);
1529     }
1530
1531     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1532                                                   String interfaceName) {
1533         // This is either an extra-route (or) a learned IP via subnet-route
1534         List<String> nhList = null;
1535         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1536         if (nextHopIp == null || nextHopIp.isEmpty()) {
1537             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1538                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1539                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1540                     vpnName);
1541             nhList = emptyList();
1542         } else {
1543             nhList = Collections.singletonList(nextHopIp);
1544         }
1545         return nhList;
1546     }
1547
1548     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1549         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1550         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1551         // else use connected interface
1552         if (gwPort != null && gwPort.isSubnetIp()) {
1553             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1554                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1555             return Optional.of(gwPort.getMacAddress());
1556         }
1557         return Optional.absent();
1558     }
1559
1560     @Override
1561     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1562         final VpnInterface update) {
1563         LOG.trace("Received VpnInterface update event: original={}, update={}", original, update);
1564         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}", update.getName(),
1565                 update.getDpnId(), original.getVpnInstanceNames(), update.getVpnInstanceNames());
1566         final String vpnInterfaceName = update.getName();
1567         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1568         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1569         //handles switching between <internal VPN - external VPN>
1570         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1571             List<ListenableFuture<Void>> futures = new ArrayList<>();
1572             if (handleVpnInstanceUpdateForVpnInterface(identifier, original, update, futures)) {
1573                 LOG.info("update: handled Instance update for VPNInterface {} on dpn {} from oldVpn(s) {} "
1574                                 + "to newVpn(s) {}",
1575                         original.getName(), dpnId,
1576                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1577                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1578                 return emptyList();
1579             }
1580             updateVpnInstanceAdjChange(original, update, vpnInterfaceName, futures);
1581             return futures;
1582         });
1583     }
1584
1585     private boolean handleVpnInstanceUpdateForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1586                                                            VpnInterface original, VpnInterface update,
1587                                                            List<ListenableFuture<Void>> futures) {
1588         boolean isVpnInstanceUpdate = false;
1589         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1590         final String interfaceName = key.getName();
1591         List<String> oldVpnList = VpnUtil.getVpnListForVpnInterface(original);
1592         List<String> oldVpnListCopy = new ArrayList<>();
1593         oldVpnListCopy.addAll(oldVpnList);
1594         List<String> newVpnList = VpnUtil.getVpnListForVpnInterface(update);
1595         List<String> newVpnListCopy = new ArrayList<>();
1596         newVpnListCopy.addAll(newVpnList);
1597
1598         oldVpnList.removeAll(newVpnList);
1599         newVpnList.removeAll(oldVpnListCopy);
1600         //This block will execute only on if there is a change in the VPN Instance.
1601         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1602             /*
1603              * Internet BGP-VPN Instance update with single router:
1604              * ====================================================
1605              * In this case single VPN Interface will be part of maximum 2 VPN Instance only.
1606              *     1st VPN Instance : router VPN or external BGP-VPN.
1607              *     2nd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1608              *
1609              * VPN Instance UPDATE:
1610              * oldVpnList = 0 and newVpnList = 1 (Internet BGP-VPN)
1611              * oldVpnList = 1 and newVpnList = 0 (Internet BGP-VPN)
1612              *
1613              * External BGP-VPN Instance update with single router:
1614              * ====================================================
1615              * In this case single VPN interface will be part of maximum 1 VPN Instance only.
1616              *
1617              * Updated VPN Instance will be always either internal router VPN to
1618              * external BGP-VPN or external BGP-VPN to internal router VPN swap.
1619              *
1620              * VPN Instance UPDATE:
1621              * oldVpnList = 1 and newVpnList = 1 (router VPN to Ext-BGPVPN)
1622              * oldVpnList = 1 and newVpnList = 1 (Ext-BGPVPN to router VPN)
1623              *
1624              * Dual Router VPN Instance Update:
1625              * ================================
1626              * In this case single VPN interface will be part of maximum 3 VPN Instance only.
1627              *
1628              * 1st VPN Instance : router VPN or external BGP-VPN-1.
1629              * 2nd VPN Instance : router VPN or external BGP-VPN-2.
1630              * 3rd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1631              *
1632              * Dual Router --> Associated with common external BGP-VPN Instance.
1633              * 1st router and 2nd router are getting associated with single External BGP-VPN
1634              * 1) add 1st router to external bgpvpn --> oldVpnList=1, newVpnList=1;
1635              * 2) add 2nd router to the same external bgpvpn --> oldVpnList=1, newVpnList=0
1636              * In this case, we need to call removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1637              *
1638              *
1639              */
1640             isVpnInstanceUpdate = true;
1641             if (VpnUtil.isDualRouterVpnUpdate(oldVpnListCopy, newVpnListCopy)) {
1642                 if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1643                         && (oldVpnList.size() == 1 && newVpnList.size() == 0)) {
1644                     //Identify the external BGP-VPN Instance and pass that value as newVpnList
1645                     List<String> externalBgpVpnList = new ArrayList<>();
1646                     for (String newVpnName : newVpnListCopy) {
1647                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1648                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1649                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1650                                 .BgpvpnType.BGPVPNExternal) {
1651                             externalBgpVpnList.add(newVpnName);
1652                             break;
1653                         }
1654                     }
1655                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1656                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList,
1657                             externalBgpVpnList, oldVpnListCopy, futures);
1658
1659                 } else if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1660                         && (oldVpnList.size() == 0 && newVpnList.size() == 1)) {
1661                     //Identify the router VPN Instance and pass that value as oldVpnList
1662                     List<String> routerVpnList = new ArrayList<>();
1663                     for (String newVpnName : newVpnListCopy) {
1664                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1665                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1666                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1667                                 .BgpvpnType.VPN) {
1668                             routerVpnList.add(newVpnName);
1669                             break;
1670                         }
1671                     }
1672                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1673                     updateVpnInstanceChange(identifier, interfaceName, original, update, routerVpnList,
1674                             newVpnList, oldVpnListCopy, futures);
1675
1676                 } else {
1677                     //Handle remaining use cases.
1678                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1679                             oldVpnListCopy, futures);
1680                 }
1681             } else {
1682                 updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1683                         oldVpnListCopy, futures);
1684             }
1685         }
1686         return isVpnInstanceUpdate;
1687     }
1688
1689     private void updateVpnInstanceChange(InstanceIdentifier<VpnInterface> identifier, String interfaceName,
1690                                          VpnInterface original, VpnInterface update, List<String> oldVpnList,
1691                                          List<String> newVpnList, List<String> oldVpnListCopy,
1692                                          List<ListenableFuture<Void>> futures) {
1693         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1694         final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1695                 ? origAdjs.getAdjacency() : new ArrayList<>();
1696         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1697         final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1698                 ? updateAdjs.getAdjacency() : new ArrayList<>();
1699
1700         boolean isOldVpnRemoveCallExecuted = false;
1701         for (String oldVpnName : oldVpnList) {
1702             LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1703                     + "remove from vpnName {} ", interfaceName, oldVpnName);
1704             removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1705             LOG.info("updateVpnInstanceChange: Processed Remove for update on VPNInterface"
1706                             + " {} upon VPN update from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName,
1707                     newVpnList);
1708             isOldVpnRemoveCallExecuted = true;
1709         }
1710         //Wait for previous interface bindings to be removed
1711         if (isOldVpnRemoveCallExecuted && !newVpnList.isEmpty()) {
1712             try {
1713                 Thread.sleep(2000);
1714             } catch (InterruptedException e) {
1715                 //Ignore
1716             }
1717         }
1718         for (String newVpnName : newVpnList) {
1719             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1720             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1721                 LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1722                         + "onto vpnName {} ", interfaceName, newVpnName);
1723                 addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1724                 LOG.info("updateVpnInstanceChange: Processed Add for update on VPNInterface {}"
1725                                 + "from oldVpn(s) {} to newVpn {} ",
1726                         interfaceName, oldVpnListCopy, newVpnName);
1727                 /* This block will execute only if V6 subnet is associated with internet BGP-VPN.
1728                  * Use Case:
1729                  *     In Dual stack network, first V4 subnet only attached to router and router is associated
1730                  *     with internet BGP-VPN(router-gw). At this point VPN interface is having only router vpn instance.
1731                  *     Later V6 subnet is added to router, at this point existing VPN interface will get updated
1732                  *     with Internet BGP-VPN instance(Note: Internet BGP-VPN Instance update in vpn interface
1733                  *     is applicable for only on V6 subnet is added to router). newVpnList = Contains only Internet
1734                  *     BGP-VPN Instance. So we required V6 primary adjacency info needs to be populated onto
1735                  *     router VPN as well as Internet BGP-VPN.
1736                  *
1737                  *     addVpnInterfaceCall() --> It will create V6 Adj onto Internet BGP-VPN only.
1738                  *     updateVpnInstanceAdjChange() --> This method call is needed for second primary V6 Adj
1739                  *                                       update in existing router VPN instance.
1740                  */
1741                 if (vpnUtil.isBgpVpnInternet(newVpnName)) {
1742                     LOG.info("updateVpnInstanceChange: VPN Interface {} with new Adjacency {} in existing "
1743                             + "VPN instance {}", interfaceName, newAdjs, original.getVpnInstanceNames());
1744                     updateVpnInstanceAdjChange(original, update, interfaceName, futures);
1745                 }
1746             }
1747         }
1748     }
1749
1750     private List<ListenableFuture<Void>> updateVpnInstanceAdjChange(VpnInterface original, VpnInterface update,
1751                                                                     String vpnInterfaceName,
1752                                                                     List<ListenableFuture<Void>> futures) {
1753         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1754         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1755                 != null ? origAdjs.getAdjacency() : new ArrayList<>();
1756         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1757         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1758                 != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1759
1760         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1761         for (VpnInstanceNames vpnInterfaceVpnInstance : update.nonnullVpnInstanceNames()) {
1762             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1763             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1764             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1765             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1766             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1767                 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1768                 //set of prefix used as entry in prefix-to-interface datastore
1769                 // is prerequisite for refresh Fib to avoid race condition leading to missing remote next hop
1770                 // in bucket actions on bgp-vpn delete
1771                 Set<String> prefixListForRefreshFib = new HashSet<>();
1772                 ListenableFuture<Void> configTxFuture = txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
1773                     confTx -> futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL,
1774                         operTx -> {
1775                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1776                                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1777                             LOG.info("VPN Interface update event-intfName {} onto vpnName {} running config-driven",
1778                                     update.getName(), newVpnName);
1779                             //handle both addition and removal of adjacencies
1780                             // currently, new adjacency may be an extra route
1781                             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1782                             if (!oldAdjs.equals(newAdjs)) {
1783                                 for (Adjacency adj : copyNewAdjs) {
1784                                     if (copyOldAdjs.contains(adj)) {
1785                                         copyOldAdjs.remove(adj);
1786                                     } else {
1787                                         // add new adjacency
1788                                         if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1789                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1790                                                     dpnId, operTx, confTx, confTx, prefixListForRefreshFib);
1791                                         }
1792                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} "
1793                                             + " added to vpn interface {} on vpn {} dpnId {}",
1794                                             adj.getIpAddress(), adj.getNextHopIpList(), adj.getLabel(),
1795                                             adj.getSubnetId(), update.getName(), newVpnName, dpnId);
1796                                     }
1797                                 }
1798                                 for (Adjacency adj : copyOldAdjs) {
1799                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1800                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1801                                             && !adj.isPhysNetworkFunc()) {
1802                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, operTx,
1803                                                 confTx);
1804                                             //remove FIB entry
1805                                             String vpnRd = vpnUtil.getVpnRd(newVpnName);
1806                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1807                                                 + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1808                                             //remove BGP entry
1809                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1810                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1811                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1812                                             }
1813                                         } else {
1814                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1815                                                 operTx, confTx);
1816                                         }
1817                                     }
1818                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1819                                         + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1820                                         adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1821                                 }
1822                             }
1823                         })));
1824                 Futures.addCallback(configTxFuture, new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
1825                     MoreExecutors.directExecutor());
1826                 futures.add(configTxFuture);
1827                 for (ListenableFuture<Void> future : futures) {
1828                     ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1829                             update.getName(), update.getVpnInstanceNames());
1830                 }
1831             } else {
1832                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1833                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1834             }
1835         }
1836         return futures;
1837     }
1838
1839     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1840         try {
1841             Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1842             synchronized (label.toString().intern()) {
1843                 InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1844                         .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1845                 Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1846                         LogicalDatastoreType.OPERATIONAL, lriIid);
1847                 if (opResult.isPresent()) {
1848                     LabelRouteInfo labelRouteInfo =
1849                             new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1850                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1851                             labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1852                 }
1853             }
1854             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1855                     nextHopIpList);
1856         } catch (ReadFailedException e) {
1857             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1858                     nextHopIpList);
1859         } catch (TransactionCommitFailedException e) {
1860             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1861                     nextHopIpList);
1862         }
1863     }
1864
1865     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1866         SubnetRoute route, String parentVpnRd, TypedWriteTransaction<Configuration> writeConfigTxn) {
1867
1868         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1869         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1870                 .addAugmentation(SubnetRoute.class, route).build();
1871         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1872         InstanceIdentifierBuilder<VrfTables> idBuilder =
1873             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1874         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1875         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1876         if (writeConfigTxn != null) {
1877             writeConfigTxn.merge(vrfTableId, vrfTableNew, CREATE_MISSING_PARENTS);
1878         } else {
1879             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1880         }
1881         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1882                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1883     }
1884
1885     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1886                                            Adjacency adj, BigInteger dpnId,
1887                                            TypedWriteTransaction<Operational> writeOperTxn,
1888                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1889                                            TypedReadWriteTransaction<Configuration> writeInvTxn,
1890                                            Set<String> prefixListForRefreshFib)
1891             throws ExecutionException, InterruptedException {
1892         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1893         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1894         try {
1895             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1896                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1897             if (optVpnInterface.isPresent()) {
1898                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1899                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1900                 String vpnName = currVpnIntf.getVpnInstanceName();
1901                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1902                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1903                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1904                         LogicalDatastoreType.OPERATIONAL, adjPath);
1905                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1906                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1907                 long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1908                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1909                 List<Adjacency> adjacencies = new ArrayList<>();
1910                 if (optAdjacencies.isPresent() && optAdjacencies.get().getAdjacency() != null) {
1911                     adjacencies.addAll(optAdjacencies.get().getAdjacency());
1912                 }
1913                 long vpnId = vpnUtil.getVpnId(vpnName);
1914                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1915                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1916                 Adjacency operationalAdjacency = null;
1917                 //Handling dual stack neutron port primary adjacency
1918                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1919                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1920                             currVpnIntf.getName(), vpnName);
1921                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1922                             currVpnIntf.getName());
1923                     if (interfaceState != null) {
1924                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1925                             currVpnIntf.getName(), vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState,
1926                             prefixListForRefreshFib);
1927                     }
1928                 }
1929                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1930                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1931                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.LearntIp ? RouteOrigin.DYNAMIC
1932                             : RouteOrigin.STATIC;
1933                     String nh = adj.getNextHopIpList().get(0);
1934                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1935                     synchronized (vpnPrefixKey.intern()) {
1936                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1937                                 vpnId, null, prefix, vpnName, nh, dpnId);
1938                         if (rdToAllocate.isPresent()) {
1939                             input.setRd(rdToAllocate.get());
1940                             operationalAdjacency = populator.createOperationalAdjacency(input);
1941                             int label = operationalAdjacency.getLabel().intValue();
1942                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1943                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
1944                                     currVpnIntf.getName(), operationalAdjacency, encapType,
1945                                     prefixListForRefreshFib, writeConfigTxn);
1946                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1947                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1948                                     vpnName, label, currVpnIntf.getName(), dpnId);
1949                         } else {
1950                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
1951                                     vpnName, prefix);
1952                             return;
1953                         }
1954                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1955                         // Keeping the MPLS check for now.
1956                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1957                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1958                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1959                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
1960                             vpnsToImportRoute.forEach(vpn -> {
1961                                 if (vpn.getVrfId() != null) {
1962                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(), vpnId, prefix,
1963                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
1964                                             .ifPresent(
1965                                                 rds -> vpnManager.addExtraRoute(
1966                                                         vpnUtil.getVpnName(vpn.getVpnId()), adj.getIpAddress(),
1967                                                         nh, rds, currVpnIntf.getVpnInstanceName(), l3vni,
1968                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(), opAdjacency,
1969                                                         encapType, prefixListForRefreshFib, writeConfigTxn));
1970                                 }
1971                             });
1972                         }
1973                     }
1974                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1975                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1976                             currVpnIntf.getName(), vpnName);
1977
1978                     InstanceIdentifier<VpnInterface> vpnIfaceConfigidentifier = VpnUtil
1979                             .getVpnInterfaceIdentifier(currVpnIntf.getName());
1980                     Optional<VpnInterface> vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1981                             LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier);
1982                     Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1983                             Prefixes.PrefixCue.PhysNetFunc);
1984                     if (vpnIntefaceConfig.isPresent()) {
1985                         pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1986                                 vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(),
1987                                 vpnIntefaceConfig.get().getSegmentationId(), Prefixes.PrefixCue.PhysNetFunc);
1988                     }
1989
1990                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1991
1992                     writeOperTxn.merge(
1993                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
1994                                     prefix), pnfPrefix, true);
1995
1996                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1997                             adj.getIpAddress(), emptyList(), null /* EncapType */, 0 /* label */,
1998                             0 /*l3vni*/, null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1999
2000                     input.setRd(adj.getVrfId());
2001                 }
2002                 if (operationalAdjacency == null) {
2003                     operationalAdjacency = populator.createOperationalAdjacency(input);
2004                 }
2005                 adjacencies.add(operationalAdjacency);
2006                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
2007                 VpnInterfaceOpDataEntry newVpnIntf =
2008                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
2009                                 aug, dpnId, currVpnIntf.getLportTag(),
2010                                 currVpnIntf.getGatewayMacAddress());
2011
2012                 writeOperTxn.merge(identifier, newVpnIntf, CREATE_MISSING_PARENTS);
2013             }
2014         } catch (ReadFailedException e) {
2015             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
2016                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
2017         }
2018     }
2019
2020     @Nullable
2021     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
2022         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
2023         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
2024     }
2025
2026     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
2027                                             BigInteger dpnId, TypedWriteTransaction<Operational> writeOperTxn,
2028                                             TypedWriteTransaction<Configuration> writeConfigTxn) {
2029         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
2030         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
2031         try {
2032             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
2033                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
2034             if (optVpnInterface.isPresent()) {
2035                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
2036                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
2037                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2038                         LogicalDatastoreType.OPERATIONAL, path);
2039                 if (optAdjacencies.isPresent()) {
2040                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
2041
2042                     if (adjacencies != null && !adjacencies.isEmpty()) {
2043                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
2044                         for (Adjacency adjacency : adjacencies) {
2045                             if (Objects.equals(adjacency.getIpAddress(), adj.getIpAddress())) {
2046                                 String rd = adjacency.getVrfId();
2047                                 if (adj.getNextHopIpList() != null) {
2048                                     for (String nh : adj.getNextHopIpList()) {
2049                                         deleteExtraRouteFromCurrentAndImportingVpns(
2050                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
2051                                                 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
2052                                     }
2053                                 } else if (adj.isPhysNetworkFunc()) {
2054                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
2055                                             adj.getIpAddress(), adj.getSubnetId());
2056                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
2057                                             writeConfigTxn);
2058                                 }
2059                                 break;
2060                             }
2061
2062                         }
2063                     }
2064                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
2065                             dpnId, adj.getVrfId());
2066                 } else {
2067                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
2068                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
2069                 }
2070             }
2071         } catch (ReadFailedException e) {
2072             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
2073                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
2074         }
2075     }
2076
2077     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
2078                                     String rd, String intfName, TypedWriteTransaction<Configuration> writeConfigTxn,
2079                                     TypedWriteTransaction<Operational> writeOperTx) {
2080         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
2081         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
2082         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
2083             String vpnRd = vpn.getVrfId();
2084             if (vpnRd != null) {
2085                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
2086                         writeOperTx);
2087             }
2088         }
2089     }
2090
2091     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
2092         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2093             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
2094             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
2095     }
2096
2097     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
2098         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2099             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
2100     }
2101
2102     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2103                                                 TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2104         if (vpnInterface == null) {
2105             return;
2106         }
2107         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
2108         if (adjs == null) {
2109             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2110                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2111             return;
2112         }
2113         for (Adjacency adj : adjs) {
2114             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2115                 String primaryInterfaceIp = adj.getIpAddress();
2116                 String macAddress = adj.getMacAddress();
2117                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2118
2119                 long label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
2120                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2121
2122                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2123                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2124                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
2125                         routerInt, label, writeConfigTxn);
2126                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2127                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2128                         macAddress);
2129             } else {
2130                 LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2131                                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}",
2132                         interfaceName, primaryRd, vpnName);
2133             }
2134         }
2135     }
2136
2137     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
2138             TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2139         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
2140         String rd = vpnUtil.getVpnRd(vpnName);
2141         if (adjs != null) {
2142             List<Adjacency> adjsList = adjs.nonnullAdjacency();
2143             for (Adjacency adj : adjsList) {
2144                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2145                     String primaryInterfaceIp = adj.getIpAddress();
2146                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2147                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
2148                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2149                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2150                 }
2151             }
2152         } else {
2153             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2154                     vpnInterface.getName(), rd);
2155         }
2156     }
2157
2158     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2159         final VpnInterfaceKey key = intefaceData.identifier.firstKeyOf(VpnInterface.class);
2160         final String interfaceName = key.getName();
2161         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2162                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2163         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2164                   intefaceData.identifier, vpnName);
2165     }
2166
2167     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2168                                               VpnInterface vpnInterface, String vpnName) {
2169         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2170                .get(vpnName);
2171         if (vpnInterfaces == null) {
2172             vpnInterfaces = new ConcurrentLinkedQueue<>();
2173         }
2174         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2175         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2176         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2177                 vpnInterface.getName(), vpnName);
2178     }
2179
2180     public boolean isVpnInstanceReady(String vpnInstanceName) {
2181         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2182         if (vpnRd == null) {
2183             return false;
2184         }
2185         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2186
2187         return vpnInstanceOpDataEntry != null;
2188     }
2189
2190     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2191         synchronized (vpnInstanceName.intern()) {
2192             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2193                     unprocessedVpnInterfaces.get(vpnInstanceName);
2194             if (vpnInterfaces != null) {
2195                 while (!vpnInterfaces.isEmpty()) {
2196                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2197                     if (hasVpnInstanceCreatedSuccessfully) {
2198                         processSavedInterface(savedInterface, vpnInstanceName);
2199                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2200                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2201                     } else {
2202                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2203                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2204                     }
2205                 }
2206             } else {
2207                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2208             }
2209         }
2210     }
2211
2212     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2213             VpnInterface vpnInterface) {
2214         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2215             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2216                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2217             if (vpnInterfaces != null) {
2218                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2219                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2220                             + "unprocessed list", vpnInterface.getName(),
2221                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2222                 }
2223             } else {
2224                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2225                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2226             }
2227         }
2228     }
2229
2230     public void vpnInstanceIsReady(String vpnInstanceName) {
2231         processSavedInterfaces(vpnInstanceName, true);
2232     }
2233
2234     public void vpnInstanceFailed(String vpnInstanceName) {
2235         processSavedInterfaces(vpnInstanceName, false);
2236     }
2237
2238     private static class UnprocessedVpnInterfaceData {
2239         InstanceIdentifier<VpnInterface> identifier;
2240         VpnInterface vpnInterface;
2241
2242         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2243             this.identifier = identifier;
2244             this.vpnInterface = vpnInterface;
2245         }
2246
2247         @Override
2248         public int hashCode() {
2249             final int prime = 31;
2250             int result = 1;
2251             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2252             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2253             return result;
2254         }
2255
2256         @Override
2257         public boolean equals(Object obj) {
2258             if (this == obj) {
2259                 return true;
2260             }
2261             if (obj == null) {
2262                 return false;
2263             }
2264             if (getClass() != obj.getClass()) {
2265                 return false;
2266             }
2267             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2268             if (identifier == null) {
2269                 if (other.identifier != null) {
2270                     return false;
2271                 }
2272             } else if (!identifier.equals(other.identifier)) {
2273                 return false;
2274             }
2275             if (vpnInterface == null) {
2276                 if (other.vpnInterface != null) {
2277                     return false;
2278                 }
2279             } else if (!vpnInterface.equals(other.vpnInterface)) {
2280                 return false;
2281             }
2282             return true;
2283         }
2284     }
2285
2286     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2287         String primaryRd = vpnUtil.getVpnRd(vpnName);
2288         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2289         if (vpnInstanceOpData == null) {
2290             return;
2291         }
2292         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2293         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2294             return;
2295         }
2296         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2297         vpnToDpnLists.forEach(vpnToDpnList -> {
2298             if (vpnToDpnList.getVpnInterfaces() == null) {
2299                 return;
2300             }
2301             vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2302                 try {
2303                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2304                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2305                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2306                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2307                     if (!vpnInterfaceOptional.isPresent()) {
2308                         return;
2309                     }
2310                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2311                             vpnInterface.getInterfaceName());
2312                     if (configVpnAdjacencies == null) {
2313                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2314                         return;
2315                     }
2316                     List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2317                             .augmentation(AdjacenciesOp.class).nonnullAdjacency();
2318                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2319                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2320                     // in operational DS. These unprocessed adjacencies will be handled below.
2321                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2322                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2323                     configVpnAdjacencies.stream()
2324                         .filter(adjacency -> operationVpnAdjacencies.stream()
2325                                 .noneMatch(operationalAdjacency ->
2326                                     Objects.equals(operationalAdjacency.getIpAddress(), adjacency.getIpAddress())))
2327                         .forEach(adjacency -> {
2328                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2329                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2330                                 () -> {
2331                                     // TODO Deal with sequencing — the config tx must only submitted
2332                                     // if the oper tx goes in
2333                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2334                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2335                                         futures.add(
2336                                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
2337                                                 //set of prefix used, as entry in prefix-to-interface datastore
2338                                                 // is prerequisite for refresh Fib to avoid race condition leading
2339                                                 // to missing remote next hop in bucket actions on bgp-vpn delete
2340                                                 Set<String> prefixListForRefreshFib = new HashSet<>();
2341                                                 ListenableFuture<Void> configTxFuture =
2342                                                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
2343                                                         confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2344                                                             primaryRd, adjacency, vpnInterfaceOptional.get().getDpnId(),
2345                                                                 operTx, confTx, confTx, prefixListForRefreshFib));
2346                                                 Futures.addCallback(configTxFuture,
2347                                                     new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
2348                                                     MoreExecutors.directExecutor());
2349                                                 futures.add(configTxFuture);
2350                                             }));
2351                                         return futures;
2352                                     } else {
2353                                         return emptyList();
2354                                     }
2355                                 });
2356                         });
2357                 } catch (ReadFailedException e) {
2358                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2359                             vpnName, primaryRd);
2360                 }
2361             });
2362         });
2363     }
2364
2365     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2366         private final String interfaceName;
2367         private final boolean add;
2368         private final String txnDestination;
2369
2370         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2371             this.interfaceName = interfaceName;
2372             this.add = add;
2373             this.txnDestination = transactionDest;
2374         }
2375
2376         @Override
2377         public void onSuccess(Void voidObj) {
2378             if (add) {
2379                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2380                         interfaceName, txnDestination);
2381             } else {
2382                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2383             }
2384         }
2385
2386         @Override
2387         public void onFailure(Throwable throwable) {
2388             if (add) {
2389                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2390                         interfaceName, txnDestination, throwable);
2391             } else {
2392                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2393                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2394             }
2395         }
2396     }
2397
2398     private class VpnInterfaceCallBackHandler implements FutureCallback<Void> {
2399         private final String primaryRd;
2400         private final Set<String> prefixListForRefreshFib;
2401
2402         VpnInterfaceCallBackHandler(String primaryRd, Set<String> prefixListForRefreshFib) {
2403             this.primaryRd = primaryRd;
2404             this.prefixListForRefreshFib = prefixListForRefreshFib;
2405         }
2406
2407         @Override
2408         public void onSuccess(Void voidObj) {
2409             prefixListForRefreshFib.forEach(prefix -> {
2410                 fibManager.refreshVrfEntry(primaryRd, prefix);
2411             });
2412         }
2413
2414         @Override
2415         public void onFailure(Throwable throwable) {
2416             LOG.debug("write Tx config operation failed {}", throwable);
2417         }
2418     }
2419 }