vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14
  15 import com.google.common.base.Optional;
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.collect.Iterators;
  18 import com.google.common.util.concurrent.FutureCallback;
  19 import com.google.common.util.concurrent.Futures;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import com.google.common.util.concurrent.MoreExecutors;
  22 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  23 import java.math.BigInteger;
  24 import java.util.ArrayList;
  25 import java.util.Collections;
  26 import java.util.HashSet;
  27 import java.util.List;
  28 import java.util.Map;
  29 import java.util.Objects;
  30 import java.util.Set;
  31 import java.util.concurrent.ConcurrentHashMap;
  32 import java.util.concurrent.ConcurrentLinkedQueue;
  33 import java.util.concurrent.ExecutionException;
  34 import java.util.function.Consumer;
  35 import java.util.function.Predicate;
  36 import java.util.stream.Collectors;
  37 import javax.annotation.PostConstruct;
  38 import javax.annotation.PreDestroy;
  39 import javax.inject.Inject;
  40 import javax.inject.Singleton;
  41 import org.eclipse.jdt.annotation.Nullable;
  42 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  43 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  44 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  45 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  46 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  47 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  48 import org.opendaylight.genius.infra.Datastore.Configuration;
  49 import org.opendaylight.genius.infra.Datastore.Operational;
  50 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  51 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  52 import org.opendaylight.genius.infra.TransactionAdapter;
  53 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  54 import org.opendaylight.genius.infra.TypedWriteTransaction;
  55 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  56 import org.opendaylight.genius.mdsalutil.NWUtil;
  57 import org.opendaylight.genius.mdsalutil.NwConstants;
  58 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  59 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  60 import org.opendaylight.infrautils.caches.CacheProvider;
  61 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  62 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  63 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  64 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  65 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  66 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  67 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  68 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  69 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  70 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  71 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  72 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  73 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  74 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  75 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  76 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  77 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  78 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  79 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 119 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 120 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 121 import org.slf4j.Logger;
 122 import org.slf4j.LoggerFactory;
 123
 124 @Singleton
 125 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 126
 127     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 128     private static final short DJC_MAX_RETRIES = 3;
 129
 130     private final DataBroker dataBroker;
 131     private final ManagedNewTransactionRunner txRunner;
 132     private final IBgpManager bgpManager;
 133     private final IFibManager fibManager;
 134     private final IMdsalApiManager mdsalManager;
 135     private final IdManagerService idManager;
 136     private final OdlInterfaceRpcService ifaceMgrRpcService;
 137     private final VpnFootprintService vpnFootprintService;
 138     private final IInterfaceManager interfaceManager;
 139     private final IVpnManager vpnManager;
 140     private final ArpResponderHandler arpResponderHandler;
 141     private final JobCoordinator jobCoordinator;
 142     private final VpnUtil vpnUtil;
 143
 144     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 145
 146     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 147             new ConcurrentHashMap<>();
 148
 149     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 150
 151     @Inject
 152     public VpnInterfaceManager(final DataBroker dataBroker,
 153                                final IBgpManager bgpManager,
 154                                final IdManagerService idManager,
 155                                final IMdsalApiManager mdsalManager,
 156                                final IFibManager fibManager,
 157                                final OdlInterfaceRpcService ifaceMgrRpcService,
 158                                final VpnFootprintService vpnFootprintService,
 159                                final IInterfaceManager interfaceManager,
 160                                final IVpnManager vpnManager,
 161                                final ArpResponderHandler arpResponderHandler,
 162                                final JobCoordinator jobCoordinator,
 163                                final CacheProvider cacheProvider,
 164                                final VpnUtil vpnUtil) {
 165         super(VpnInterface.class, VpnInterfaceManager.class);
 166
 167         this.dataBroker = dataBroker;
 168         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 169         this.bgpManager = bgpManager;
 170         this.idManager = idManager;
 171         this.mdsalManager = mdsalManager;
 172         this.fibManager = fibManager;
 173         this.ifaceMgrRpcService = ifaceMgrRpcService;
 174         this.vpnFootprintService = vpnFootprintService;
 175         this.interfaceManager = interfaceManager;
 176         this.vpnManager = vpnManager;
 177         this.arpResponderHandler = arpResponderHandler;
 178         this.jobCoordinator = jobCoordinator;
 179         this.vpnUtil = vpnUtil;
 180
 181         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 182                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 183                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 184     }
 185
 186     public Runnable isNotifyTaskQueued(String intfName) {
 187         return vpnIntfMap.remove(intfName);
 188     }
 189
 190     @PostConstruct
 191     public void start() {
 192         LOG.info("{} start", getClass().getSimpleName());
 193         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 194     }
 195
 196     @Override
 197     @PreDestroy
 198     public void close() {
 199         super.close();
 200         vpnInstanceOpDataEntryCache.close();
 201     }
 202
 203     @Override
 204     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 205         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 206     }
 207
 208     @Override
 209     protected VpnInterfaceManager getDataTreeChangeListener() {
 210         return VpnInterfaceManager.this;
 211     }
 212
 213     @Override
 214     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 215         LOG.trace("Received VpnInterface add event: vpnInterface={}", vpnInterface);
 216         LOG.info("add: intfName {} onto vpnName {}", vpnInterface.getName(),
 217                 VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 218         addVpnInterface(identifier, vpnInterface, null, null);
 219     }
 220
 221     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 222                           final VpnInterface vpnInterface, String vpnName) {
 223         synchronized (vpnName.intern()) {
 224             if (isVpnInstanceReady(vpnName)) {
 225                 return true;
 226             }
 227             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 228             return false;
 229         }
 230     }
 231
 232     // TODO Clean up the exception handling
 233     @SuppressWarnings("checkstyle:IllegalCatch")
 234     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 235                              final @Nullable List<Adjacency> oldAdjs, final @Nullable List<Adjacency> newAdjs) {
 236         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
 237             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 238             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 239         }
 240     }
 241
 242     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 243                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 244         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 245         final String interfaceName = key.getName();
 246
 247         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 248             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 249                   vpnName, vpnInterface.getName());
 250             return;
 251         }
 252         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 253              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 254         List<Adjacency> copyOldAdjs = null;
 255         if (oldAdjs != null) {
 256             copyOldAdjs = new ArrayList<>();
 257             copyOldAdjs.addAll(oldAdjs);
 258         }
 259         List<Adjacency> copyNewAdjs = null;
 260         if (newAdjs != null) {
 261             copyNewAdjs = new ArrayList<>();
 262             copyNewAdjs.addAll(newAdjs);
 263         }
 264         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 265     }
 266
 267     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 268                     final VpnInterface vpnInterface, final @Nullable List<Adjacency> oldAdjs,
 269                     final @Nullable List<Adjacency> newAdjs,
 270                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 271         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 272         final String interfaceName = key.getName();
 273         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 274         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 275             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 276             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 277             if (interfaceState != null) {
 278                 try {
 279                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 280                     final int ifIndex = interfaceState.getIfIndex();
 281                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 282                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 283                         // (the inventory tx goes in last)
 284                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 285                         //set of prefix used, as entry in prefix-to-interface datastore
 286                         // is prerequisite for refresh Fib to avoid race condition leading to
 287                         // missing remote next hop in bucket actions on bgp-vpn delete
 288                         Set<String> prefixListForRefreshFib = new HashSet<>();
 289                         ListenableFuture<Void> confFuture =
 290                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 291                                 confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
 292                                     operTx -> futures.add(
 293                                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, invTx -> {
 294                                             LOG.info(
 295                                                 "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 296                                                     + " on dpn {}",
 297                                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 298                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 299                                                 confTx, operTx, invTx, interfaceState, vpnName,
 300                                                 prefixListForRefreshFib);
 301                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 302                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 303                                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 304                                                 if (newAdjs != null) {
 305                                                     for (Adjacency adj : newAdjs) {
 306                                                         if (oldAdjs.contains(adj)) {
 307                                                             oldAdjs.remove(adj);
 308                                                         } else {
 309                                                             if (!isBgpVpnInternetVpn
 310                                                                 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 311                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 312                                                                     primaryRd, adj, dpnId, operTx, confTx, invTx,
 313                                                                     prefixListForRefreshFib);
 314                                                             }
 315                                                         }
 316                                                     }
 317                                                 }
 318                                                 for (Adjacency adj : oldAdjs) {
 319                                                     if (!isBgpVpnInternetVpn
 320                                                         || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 321                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 322                                                             operTx, confTx);
 323                                                     }
 324                                                 }
 325                                             }
 326                                         })))));
 327                         Futures.addCallback(confFuture,
 328                             new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
 329                             MoreExecutors.directExecutor());
 330                         futures.add(confFuture);
 331                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 332                             MoreExecutors.directExecutor());
 333                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 334                             + " processed successfully", interfaceName, vpnName, dpnId);
 335                         return futures;
 336                     });
 337                 } catch (NumberFormatException | IllegalStateException e) {
 338                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 339                             + "interface {}. Interface addition on vpn {} failed", interfaceName,
 340                         vpnName, e);
 341                     return;
 342                 }
 343             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 344                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 345                     () -> {
 346                         ListenableFuture<Void> future =
 347                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
 348                                 createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 349                                     confTx, vpnName);
 350                                 LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 351                                     vpnName, vpnInterface.getDpnId());
 352                             });
 353                         ListenableFutures.addErrorLogging(future, LOG,
 354                             "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 355                         return Collections.singletonList(future);
 356                     });
 357             } else {
 358                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 359                     + " is not available", interfaceName, vpnName);
 360             }
 361         } else {
 362             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 363                     + " as vpn is pending delete", interfaceName, vpnName,
 364                 vpnInterface.getDpnId());
 365         }
 366     }
 367
 368     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 369     // see comments below.
 370     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 371     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 372             final int lportTag, boolean isInterfaceUp,
 373             TypedWriteTransaction<Configuration> writeConfigTxn,
 374             TypedWriteTransaction<Operational> writeOperTxn,
 375             TypedReadWriteTransaction<Configuration> writeInvTxn,
 376             Interface interfaceState, final String vpnName,
 377             Set<String> prefixListForRefreshFib) throws ExecutionException, InterruptedException {
 378         final String interfaceName = vpnInterface.getName();
 379         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 380                 vpnName);
 381         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 382         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 383         if (!isInterfaceUp) {
 384             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 385                      interfaceName, dpId, vpnName);
 386             long vpnId = vpnUtil.getVpnId(vpnName);
 387             if (vpnId == VpnConstants.INVALID_ID) {
 388                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 389                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 390                         dpId);
 391                 return;
 392             }
 393
 394             boolean waitForVpnInterfaceOpRemoval = false;
 395             if (opVpnInterface != null) {
 396                 String opVpnName = opVpnInterface.getVpnInstanceName();
 397                 String primaryInterfaceIp = null;
 398                 if (Objects.equals(opVpnName, vpnName)) {
 399                     // Please check if the primary VRF Entry does not exist for VPNInterface
 400                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 401                     // back to back
 402                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 403                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 404                     if (adjs == null) {
 405                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 406                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 407                                 vpnName);
 408                         return;
 409                     }
 410                     for (Adjacency adj : adjs) {
 411                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 412                             primaryInterfaceIp = adj.getIpAddress();
 413                             break;
 414                         }
 415                     }
 416                     if (primaryInterfaceIp == null) {
 417                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 418                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 419                                 dpId, vpnName);
 420                         return;
 421                     }
 422                     // Get the rd of the vpn instance
 423                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 424                     if (vrf != null) {
 425                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 426                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 427                         return;
 428                     }
 429                     waitForVpnInterfaceOpRemoval = true;
 430                 } else {
 431                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 432                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 433                 }
 434             }
 435             if (!waitForVpnInterfaceOpRemoval) {
 436                 // Add the VPNInterface and quit
 437                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 438                         null/*ipAddressSourceValuePair*/,
 439                         true /* add */);
 440                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 441                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 442                 if (!isBgpVpnInternetVpn) {
 443                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 444                 }
 445                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 446                         dpId, vpnName);
 447                 if (interfaceManager.isExternalInterface(interfaceName)) {
 448                     processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
 449                         NwConstants.ADD_FLOW);
 450                 }
 451                 return;
 452             }
 453
 454             // FIB didn't get a chance yet to clean up this VPNInterface
 455             // Let us give it a chance here !
 456             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 457                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 458             try {
 459                 Runnable notifyTask = new VpnNotifyTask();
 460                 synchronized (notifyTask) {
 461                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 462                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 463                     // published here while holding the lock on it so this path will hit the wait before notify can be
 464                     // invoked.
 465                     vpnIntfMap.put(interfaceName, notifyTask);
 466                     try {
 467                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 468                     } catch (InterruptedException e) {
 469                         // Ignored
 470                     }
 471                 }
 472             } finally {
 473                 vpnIntfMap.remove(interfaceName);
 474             }
 475
 476             if (opVpnInterface != null) {
 477                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 478                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 479                         dpId, vpnName);
 480                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 481                 return;
 482             }
 483             // VPNInterface got removed, proceed with Add
 484             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 485                     interfaceName, dpId, vpnName);
 486             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 487                     null/*ipAddressSourceValuePair*/,
 488                     true /* add */);
 489             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 490                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 491             if (!isBgpVpnInternetVpn) {
 492                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 493             }
 494             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 495                     + " FIB to clean up", interfaceName, dpId, vpnName);
 496             if (interfaceManager.isExternalInterface(interfaceName)) {
 497                 processExternalVpnInterface(interfaceName, vpnName, dpId,
 498                                lportTag, NwConstants.ADD_FLOW);
 499             }
 500         } else {
 501             try {
 502                 // Interface is retained in the DPN, but its Link Up.
 503                 // Advertise prefixes again for this interface to BGP
 504                 InstanceIdentifier<VpnInterface> identifier =
 505                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 506                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 507                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 508                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 509                 // Perform similar operation as interface add event for extraroutes.
 510                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 511                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 512                         LogicalDatastoreType.CONFIGURATION, path);
 513                 if (!optAdjacencies.isPresent()) {
 514                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 515                     return;
 516                 }
 517                 List<Adjacency> adjacencies = optAdjacencies.get().nonnullAdjacency();
 518                 for (Adjacency adjacency : adjacencies) {
 519                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 520                         continue;
 521                     }
 522                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 523                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 524                         continue;
 525                     }
 526                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 527                             dpId, writeOperTxn, writeConfigTxn, writeInvTxn, prefixListForRefreshFib);
 528                 }
 529             } catch (ReadFailedException e) {
 530                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 531                         interfaceName, vpnName, primaryRd, dpId);
 532             }
 533         }
 534     }
 535
 536     private void processExternalVpnInterface(String interfaceName, String vpnName, BigInteger dpId,
 537         int lportTag, int addOrRemove) {
 538         Uuid extNetworkId;
 539         try {
 540             // vpn instance of ext-net interface is the network-id
 541             extNetworkId = new Uuid(vpnName);
 542         } catch (IllegalArgumentException e) {
 543             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 544                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 545             return;
 546         }
 547
 548         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 549         if (routerIds == null || routerIds.isEmpty()) {
 550             LOG.info("processExternalVpnInterface: No router is associated with {}."
 551                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 552                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 553             return;
 554         }
 555
 556         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 557                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 558         for (Uuid routerId : routerIds) {
 559             String routerName = routerId.getValue();
 560             BigInteger primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 561             if (Objects.equals(primarySwitch, dpId)) {
 562                 Routers router = vpnUtil.getExternalRouter(routerName);
 563                 if (router != null) {
 564                     if (addOrRemove == NwConstants.ADD_FLOW) {
 565                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 566                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 567                                 dpId, interfaceName, lportTag);
 568                     } else {
 569                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 570                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 571                                 dpId, interfaceName, lportTag);
 572                     }
 573                 } else {
 574                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 575                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 576                             interfaceName, dpId, vpnName);
 577                 }
 578             }
 579         }
 580     }
 581
 582     // TODO Clean up the exception handling
 583     @SuppressWarnings("checkstyle:IllegalCatch")
 584     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 585                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 586                                                   String vpnName, String interfaceName) {
 587         if (rd == null) {
 588             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 589                   interfaceName, dpnId, vpnName);
 590             return;
 591         } else {
 592             if (rd.equals(vpnName)) {
 593                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 594                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 595                 return;
 596             }
 597         }
 598         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 599                 interfaceName, dpnId, vpnName, rd);
 600
 601         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 602         if (nextHopIp == null) {
 603             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 604                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 605                     rd, vpnName);
 606             return;
 607         }
 608
 609         try {
 610             //Read NextHops
 611             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 612             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 613                     LogicalDatastoreType.OPERATIONAL, path);
 614             if (adjacencies.isPresent()) {
 615                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 616                 if (nextHops != null && !nextHops.isEmpty()) {
 617                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 618                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 619                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 620                     long l3vni = vpnInstanceOpData.getL3vni();
 621                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 622                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 623                     for (Adjacency nextHop : nextHops) {
 624                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 625                             continue;
 626                         }
 627                         String gatewayMac = null;
 628                         long label = 0;
 629                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 630                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 631                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 632                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 633                                     .get();
 634                         } else {
 635                             label = nextHop.getLabel();
 636                         }
 637                         try {
 638                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 639                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 640                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 641                                     encapType, (int)label, l3vni, 0 /*l2vni*/,
 642                                     gatewayMac);
 643                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 644                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 645                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 646                         } catch (Exception e) {
 647                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 648                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 649                                     interfaceName, dpnId, e);
 650                         }
 651                     }
 652                 }
 653             }
 654         } catch (ReadFailedException e) {
 655             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 656                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 657         }
 658     }
 659
 660     // TODO Clean up the exception handling
 661     @SuppressWarnings("checkstyle:IllegalCatch")
 662     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 663                         String vpnName, String interfaceName, TypedWriteTransaction<Configuration> writeConfigTxn,
 664                         TypedWriteTransaction<Operational> writeOperTx) {
 665         //Read NextHops
 666         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 667         String rd = vpnUtil.getVpnRd(interfaceName);
 668         if (rd == null) {
 669             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 670                 interfaceName, vpnName);
 671             return;
 672         } else {
 673             if (rd.equals(vpnName)) {
 674                 LOG.info(
 675                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 676                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 677                 return;
 678             }
 679         }
 680         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 681                vpnName, rd);
 682         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 683         try {
 684             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 685                     path);
 686         } catch (ReadFailedException e) {
 687             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 688                     interfaceName, vpnName);
 689         }
 690         if (adjacencies.isPresent()) {
 691             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 692
 693             if (nextHops != null && !nextHops.isEmpty()) {
 694                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 695                         nextHops, interfaceName, vpnName, rd);
 696                 for (Adjacency nextHop : nextHops) {
 697                     try {
 698                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 699                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 700                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 701                                     interfaceName, vpnName);
 702                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 703                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 704                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 705                                     interfaceName, vpnName);
 706                         } else if (nextHop.getNextHopIpList() != null) {
 707                             // Perform similar operation as interface delete event for extraroutes.
 708                             String allocatedRd = nextHop.getVrfId();
 709                             for (String nh : nextHop.getNextHopIpList()) {
 710                                 deleteExtraRouteFromCurrentAndImportingVpns(
 711                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
 712                                     writeOperTx);
 713                             }
 714                         }
 715                     } catch (Exception e) {
 716                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 717                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 718                     }
 719                 }
 720             }
 721         }
 722     }
 723
 724     @SuppressWarnings("checkstyle:IllegalCatch")
 725     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 726                                                   String primaryRd, String interfaceName, final long vpnId,
 727                                                   TypedWriteTransaction<Configuration> writeConfigTxn,
 728                                                   TypedWriteTransaction<Operational> writeOperTxn,
 729                                                   TypedReadWriteTransaction<Configuration> writeInvTxn,
 730                                                   Interface interfaceState, Set<String> prefixListForRefreshFib)
 731             throws ExecutionException, InterruptedException {
 732         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 733         // Read NextHops
 734         Optional<VpnInterface> vpnInteface = Optional.absent();
 735         try {
 736             vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 737             LogicalDatastoreType.CONFIGURATION, identifier);
 738         } catch (ReadFailedException e) {
 739             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 740                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 741         }
 742         Uuid intfnetworkUuid = null;
 743         NetworkType networkType = null;
 744         Long segmentationId = Long.valueOf(-1);
 745         Adjacencies adjacencies = null;
 746         if (vpnInteface.isPresent()) {
 747             intfnetworkUuid = vpnInteface.get().getNetworkId();
 748             networkType = vpnInteface.get().getNetworkType();
 749             segmentationId = vpnInteface.get().getSegmentationId();
 750             adjacencies = vpnInteface.get().augmentation(Adjacencies.class);
 751             if (adjacencies == null) {
 752                 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 753                         null/*gwMac*/, writeOperTxn);
 754                 return;
 755             }
 756         }
 757         // Get the rd of the vpn instance
 758         String nextHopIp = null;
 759         try {
 760             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 761         } catch (Exception e) {
 762             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 763                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 764         }
 765         List<String> nhList = new ArrayList<>();
 766         if (nextHopIp != null) {
 767             nhList.add(nextHopIp);
 768             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 769                     interfaceName, dpnId, vpnName, nhList);
 770         }
 771         Optional<String> gwMac = Optional.absent();
 772         String vpnInterfaceSubnetGwMacAddress = null;
 773         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 774         Long l3vni = vpnInstanceOpData.getL3vni();
 775         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 776         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 777         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 778         List<Adjacency> nextHops = (adjacencies != null) ? adjacencies.getAdjacency() : emptyList();
 779         List<Adjacency> value = new ArrayList<>();
 780         for (Adjacency nextHop : nextHops) {
 781             String rd = primaryRd;
 782             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 783             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 784                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 785                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 786                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 787                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 788                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 789                 continue;
 790             }
 791             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 792                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 793                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 794                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 795                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 796                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 797
 798                 Prefixes prefixes = (intfnetworkUuid != null)
 799                     ? VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, intfnetworkUuid ,networkType,
 800                             segmentationId, prefixCue) :
 801                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, prefixCue);
 802                 writeOperTxn.merge(VpnUtil.getPrefixToInterfaceIdentifier(
 803                         vpnUtil.getVpnId(vpnName), prefix), prefixes, true);
 804                 final Uuid subnetId = nextHop.getSubnetId();
 805
 806                 String gatewayIp = nextHop.getSubnetGatewayIp();
 807                 if (gatewayIp == null) {
 808                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 809                     if (gatewayIpOptional.isPresent()) {
 810                         gatewayIp = gatewayIpOptional.get();
 811                     }
 812                 }
 813
 814                 if (gatewayIp != null) {
 815                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 816                     if (gwMac.isPresent()) {
 817                         // A valid mac-address is available for this subnet-gateway-ip
 818                         // Use this for programming ARP_RESPONDER table here.  And save this
 819                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 820                         // to populate L3_GW_MAC_TABLE there.
 821                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 822                                 gatewayIp, gwMac.get());
 823                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 824                     } else {
 825                         // A valid mac-address is not available for this subnet-gateway-ip
 826                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 827                         // Save this connected-mac-address as gateway-mac-address for the
 828                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 829                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 830                         if (gwMac.isPresent()) {
 831                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 832                                     NwConstants.ADD_FLOW, gwMac.get());
 833                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 834                                     gatewayIp, gwMac.get());
 835                         } else {
 836                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 837                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 838                                 + "gwIp {}",
 839                                 subnetId, interfaceName, vpnName, gatewayIp);
 840                         }
 841                     }
 842                 } else {
 843                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 844                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 845                         subnetId, interfaceName, vpnName);
 846                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 847                 }
 848                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 849                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 850             } else {
 851                 //Extra route adjacency
 852                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 853                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 854                 synchronized (vpnPrefixKey.intern()) {
 855                     java.util.Optional<String> rdToAllocate = vpnUtil
 856                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 857                                     nextHop.getNextHopIpList().get(0), dpnId);
 858                     if (rdToAllocate.isPresent()) {
 859                         rd = rdToAllocate.get();
 860                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 861                             rd, prefix);
 862                     } else {
 863                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 864                         continue;
 865                     }
 866                 }
 867                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 868                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 869                         interfaceName, dpnId);
 870             }
 871             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 872             // can be different from the gateway-mac-address within the VRFEntry as the
 873             // gateway-mac-address is a superset.
 874             RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 875             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 876                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 877                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 878             Adjacency operationalAdjacency = null;
 879             try {
 880                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 881             } catch (NullPointerException e) {
 882                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 883                     input, e.getMessage());
 884                 return;
 885             }
 886             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 887                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 888                     vpnName, l3vni, origin, interfaceName, operationalAdjacency, encapType, prefixListForRefreshFib,
 889                     writeConfigTxn);
 890             }
 891             value.add(operationalAdjacency);
 892         }
 893
 894         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 895         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 896                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 897
 898         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 899                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 900                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 901
 902         for (Adjacency nextHop : aug.getAdjacency()) {
 903             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 904             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 905                 RouteOrigin origin = VpnUtil.getRouteOrigin(nextHop.getAdjacencyType());
 906                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 907                 registeredPopulator.populateFib(input, writeConfigTxn);
 908             }
 909         }
 910     }
 911
 912     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 913                                               long lportTag, String gwMac,
 914                                               TypedWriteTransaction<Operational> writeOperTxn) {
 915         VpnInterfaceOpDataEntry opInterface =
 916               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac);
 917         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 918             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 919         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
 920         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 921                 interfaceName, dpnId, vpnName);
 922     }
 923
 924     // TODO Clean up the exception handling
 925     @SuppressWarnings("checkstyle:IllegalCatch")
 926     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 927                                            StateTunnelList stateTunnelList,
 928                                            TypedWriteTransaction<Configuration> writeConfigTxn,
 929                                            TypedWriteTransaction<Operational> writeOperTxn) {
 930
 931         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
 932         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 933         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 934         List<Adjacency> adjList =
 935             adjacencies != null && adjacencies.getAdjacency() != null ? adjacencies.getAdjacency() : emptyList();
 936         if (adjList.isEmpty()) {
 937             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 938                     vpnInterface, srcDpnId);
 939             return;
 940         }
 941         String prefix = null;
 942         long label = 0;
 943         List<Adjacency> value = new ArrayList<>();
 944         boolean isNextHopAddReqd = false;
 945         String vpnName = vpnInterface.getVpnInstanceName();
 946         long vpnId = vpnUtil.getVpnId(vpnName);
 947         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 948         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 949                 vpnInterface.getName(), vpnInterface.getDpnId(),
 950                 vpnInterface.getVpnInstanceName(), adjList);
 951         for (Adjacency adj : adjList) {
 952             String rd = adj.getVrfId();
 953             rd = rd != null ? rd : vpnName;
 954             prefix = adj.getIpAddress();
 955             label = adj.getLabel();
 956             List<String> nhList = Collections.singletonList(srcTepIp);
 957             List<String> nextHopList = adj.getNextHopIpList();
 958             // If TEP is added , update the nexthop of primary adjacency.
 959             // Secondary adj nexthop is already pointing to primary adj IP address.
 960             if (nextHopList == null || nextHopList.isEmpty()) {
 961                 isNextHopAddReqd = true;
 962             }
 963
 964             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 965                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 966             } else {
 967                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 968                 if (!vrfEntryOptional.isPresent()) {
 969                     continue;
 970                 }
 971                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 972                 if (!nhList.contains(srcTepIp)) {
 973                     nhList.add(srcTepIp);
 974                     isNextHopAddReqd = true;
 975                 }
 976                 value.add(adj);
 977             }
 978
 979             if (isNextHopAddReqd) {
 980                 updateLabelMapper(label, nhList);
 981                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 982                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 983                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 984                 // Update the VRF entry with nextHop
 985                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 986                         label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
 987
 988                 //Get the list of VPN's importing this route(prefix) .
 989                 // Then update the VRF entry with nhList
 990                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
 991                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 992                     String vpnRd = vpn.getVrfId();
 993                     if (vpnRd != null) {
 994                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
 995                             srcTepIp, label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
 996                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 997                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
 998                             vpnInterface.getName(), srcDpnId, vpnName,
 999                             vpn.getVpnInstanceName(), vpnRd);
1000                     }
1001                 }
1002                 // Advertise the prefix to BGP only for external vpn
1003                 // since there is a nexthop change.
1004                 try {
1005                     if (!rd.equalsIgnoreCase(vpnName)) {
1006                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
1007                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
1008                                 null /*gatewayMacAddress*/);
1009                     }
1010                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
1011                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
1012                             srcDpnId, vpnName);
1013                 } catch (Exception ex) {
1014                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
1015                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1016                             vpnInterface.getName(), srcDpnId, vpnName, ex);
1017                 }
1018             }
1019         }
1020         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1021         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1022                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1023                 .addAugmentation(AdjacenciesOp.class, aug).build();
1024         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1025                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1026         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1027         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
1028                 vpnInterface.getName(), srcDpnId, vpnName);
1029
1030     }
1031
1032     // TODO Clean up the exception handling
1033     @SuppressWarnings("checkstyle:IllegalCatch")
1034     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1035                                               StateTunnelList stateTunnelList,
1036                                               TypedWriteTransaction<Configuration> writeConfigTxn,
1037                                               TypedWriteTransaction<Operational> writeOperTxn) {
1038
1039         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1040         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1041         String prefix = null;
1042         long label = 0;
1043         boolean isNextHopRemoveReqd = false;
1044         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
1045         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
1046         String vpnName = vpnInterface.getVpnInstanceName();
1047         long vpnId = vpnUtil.getVpnId(vpnName);
1048         String primaryRd = vpnUtil.getVpnRd(vpnName);
1049         if (adjList != null) {
1050             List<Adjacency> value = new ArrayList<>();
1051             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1052                     vpnInterface.getName(), vpnInterface.getDpnId(),
1053                     vpnInterface.getVpnInstanceName(), adjList);
1054             for (Adjacency adj : adjList) {
1055                 List<String> nhList = new ArrayList<>();
1056                 String rd = adj.getVrfId();
1057                 rd = rd != null ? rd : vpnName;
1058                 prefix = adj.getIpAddress();
1059                 List<String> nextHopList = adj.getNextHopIpList();
1060                 label = adj.getLabel();
1061                 if (nextHopList != null && !nextHopList.isEmpty()) {
1062                     isNextHopRemoveReqd = true;
1063                 }
1064                 // If TEP is deleted , remove the nexthop from primary adjacency.
1065                 // Secondary adj nexthop will continue to point to primary adj IP address.
1066                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1067                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1068                 } else {
1069                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1070                     if (!vrfEntryOptional.isPresent()) {
1071                         continue;
1072                     }
1073                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1074                     if (nhList.contains(srcTepIp)) {
1075                         nhList.remove(srcTepIp);
1076                         isNextHopRemoveReqd = true;
1077                     }
1078                     value.add(adj);
1079                 }
1080
1081                 if (isNextHopRemoveReqd) {
1082                     updateLabelMapper(label, nhList);
1083                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1084                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1085                             prefix, nhList, vpnName,
1086                             vpnId, rd, vpnInterface.getName());
1087                     // Update the VRF entry with removed nextHop
1088                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1089                             label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1090
1091                     //Get the list of VPN's importing this route(prefix) .
1092                     // Then update the VRF entry with nhList
1093                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1094                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1095                         String vpnRd = vpn.getVrfId();
1096                         if (vpnRd != null) {
1097                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1098                                 srcTepIp, label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1099                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1100                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1101                                     nhList, label, vpnInterface.getName(), srcDpnId,
1102                                     vpnName,
1103                                     vpn.getVpnInstanceName(), vpnRd);
1104                         }
1105                     }
1106
1107                     // Withdraw prefix from BGP only for external vpn.
1108                     try {
1109                         if (!rd.equalsIgnoreCase(vpnName)) {
1110                             bgpManager.withdrawPrefix(rd, prefix);
1111                         }
1112                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1113                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1114                                 vpnInterface.getName(), srcDpnId,
1115                                 vpnName);
1116                     } catch (Exception ex) {
1117                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1118                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1119                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1120                     }
1121                 }
1122             }
1123             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1124             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1125                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1126                     .addAugmentation(AdjacenciesOp.class, aug).build();
1127             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1128                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1129             writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1130             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1131                          vpnInterface.getName(), srcDpnId, vpnName);
1132         }
1133     }
1134
1135     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1136         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1137
1138         String vpnRd = vpnUtil.getVpnRd(vpnName);
1139         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1140         if (vpnInstanceOpDataEntry == null) {
1141             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1142                     + " to check for vpns exporting the routes", vpnName);
1143             return vpnsToExportRoute;
1144         }
1145
1146         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1147             if (input.getVpnInstanceName() == null) {
1148                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1149                         input.getVrfId());
1150                 return false;
1151             }
1152             return !input.getVpnInstanceName().equals(vpnName);
1153         };
1154
1155         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1156             Iterable<String> commonRTs =
1157                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1158                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1159             return Iterators.size(commonRTs.iterator()) > 0;
1160         };
1161
1162         vpnsToExportRoute =
1163                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1164                         Collectors.toList());
1165         return vpnsToExportRoute;
1166     }
1167
1168     // TODO Clean up the exception handling
1169     @SuppressWarnings("checkstyle:IllegalCatch")
1170     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1171         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1172         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1173             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1174             if (vrfEntries != null) {
1175                 ListenableFutures.addErrorLogging(
1176                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1177                         for (VrfEntry vrfEntry : vrfEntries) {
1178                             try {
1179                                 if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1180                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1181                                     LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1182                                             + " is not a controller managed non intervpn link route. Ignoring.",
1183                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1184                                     continue;
1185                                 }
1186                                 String prefix = vrfEntry.getDestPrefix();
1187                                 String gwMac = vrfEntry.getGatewayMacAddress();
1188                                 vrfEntry.nonnullRoutePaths().forEach(routePath -> {
1189                                     String nh = routePath.getNexthopAddress();
1190                                     int label = routePath.getLabel().intValue();
1191                                     if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1192                                         vrfEntry.getOrigin()))) {
1193                                         LOG.info(
1194                                             "handleVpnsExportingRoutesImporting: Importing fib entry rd {}"
1195                                                 + " prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1196                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1197                                         fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1198                                             Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1199                                             0 /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1200                                             confTx);
1201                                     } else {
1202                                         LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry"
1203                                                 + " rd {} prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1204                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1205                                         SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1206                                         importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1207                                             confTx);
1208                                     }
1209                                 });
1210                             } catch (RuntimeException e) {
1211                                 LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1212                                         + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1213                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1214                             }
1215                         }
1216                     }), LOG, "Error handing VPN exporting routes");
1217             } else {
1218                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1219                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1220             }
1221         }
1222     }
1223
1224     @Override
1225     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1226         LOG.trace("Received VpnInterface remove event: vpnInterface={}", vpnInterface);
1227         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1228         final String interfaceName = key.getName();
1229         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
1230             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1231             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1232         }
1233     }
1234
1235     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1236                                 final VpnInterface vpnInterface, final String vpnName,
1237                                 final String interfaceName) {
1238         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1239             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1240                 ListenableFuture<Void> future =
1241                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1242                         deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1243                         LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1244                     });
1245                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1246                         vpnInterface.getName(), vpnName);
1247                 return Collections.singletonList(future);
1248             }, DJC_MAX_RETRIES);
1249         } else {
1250             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1251             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1252         }
1253     }
1254
1255     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1256     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1257                                            final VpnInterface vpnInterface, final String vpnName,
1258                                            final String interfaceName, final Interface interfaceState) {
1259         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1260                 vpnName, vpnInterface.getDpnId());
1261         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1262         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1263             () -> {
1264                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1265                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1266                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1267                         writeOperTxn -> futures.add(
1268                             txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeInvTxn -> {
1269                                 LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1270                                         interfaceName, vpnName);
1271                                 BigInteger dpId;
1272                                 int ifIndex;
1273                                 String gwMacAddress;
1274                                 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1275                                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1276                                 Optional<VpnInterfaceOpDataEntry> optVpnInterface;
1277                                 try {
1278                                     optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1279                                             LogicalDatastoreType.OPERATIONAL, interfaceId);
1280                                 } catch (ReadFailedException e) {
1281                                     LOG.error("remove: Failed to read data store for interface {} vpn {}",
1282                                             interfaceName, vpnName);
1283                                     return;
1284                                 }
1285                                 if (interfaceState != null) {
1286                                     try {
1287                                         dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1288                                     } catch (NumberFormatException | IllegalStateException e) {
1289                                         LOG.error("remove: Unable to retrieve dpnId from interface operational"
1290                                                         + " data store for interface {} on dpn {} for vpn {} Fetching"
1291                                                         + " from vpn interface op data store. ", interfaceName,
1292                                                 vpnInterface.getDpnId(), vpnName, e);
1293                                         dpId = BigInteger.ZERO;
1294                                     }
1295                                     ifIndex = interfaceState.getIfIndex();
1296                                     gwMacAddress = interfaceState.getPhysAddress().getValue();
1297                                 } else {
1298                                     LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1299                                             + " from vpn interface op.", interfaceName);
1300                                     if (optVpnInterface.isPresent()) {
1301                                         VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1302                                         dpId = vpnOpInterface.getDpnId();
1303                                         ifIndex = vpnOpInterface.getLportTag().intValue();
1304                                         gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1305                                     } else {
1306                                         LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1307                                                 + " as interfaceState and vpn interface op is not"
1308                                                 + " available", interfaceName, vpnName);
1309                                         return;
1310                                     }
1311                                 }
1312                                 processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1313                                         optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1314                                         writeConfigTxn, writeOperTxn, writeInvTxn);
1315                                 LOG.info(
1316                                         "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1317                                                 + "successfully",
1318                                         interfaceName, vpnInterface.getDpnId(), vpnName);
1319                             })))));
1320                 futures.add(configFuture);
1321                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(
1322                         interfaceName, false, "Config"), MoreExecutors.directExecutor());
1323                 return futures;
1324             }, DJC_MAX_RETRIES);
1325     }
1326
1327     protected void processVpnInterfaceDown(BigInteger dpId,
1328                                            String interfaceName,
1329                                            int lportTag,
1330                                            String gwMac,
1331                                            VpnInterfaceOpDataEntry vpnOpInterface,
1332                                            boolean isInterfaceStateDown,
1333                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1334                                            TypedWriteTransaction<Operational> writeOperTxn,
1335                                            TypedReadWriteTransaction<Configuration> writeInvTxn)
1336             throws ExecutionException, InterruptedException {
1337         if (vpnOpInterface == null) {
1338             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1339                     + " as it is not available in operational data store", interfaceName, dpId);
1340             return;
1341         }
1342         final String vpnName = vpnOpInterface.getVpnInstanceName();
1343         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1344                                                     interfaceName, vpnName);
1345         if (!isInterfaceStateDown) {
1346             final long vpnId = vpnUtil.getVpnId(vpnName);
1347             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName,  null);
1348             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1349             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1350                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1351             if (interfaceManager.isExternalInterface(interfaceName)) {
1352                 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1353                     NwConstants.DEL_FLOW);
1354             }
1355             if (!isBgpVpnInternetVpn) {
1356                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1357             }
1358             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1359                     + " successful", interfaceName, dpId, vpnName);
1360         } else {
1361             // Interface is retained in the DPN, but its Link Down.
1362             // Only withdraw the prefixes for this interface from BGP
1363             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1364         }
1365     }
1366
1367     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1368                                           final String vpnName, final long vpnId, String gwMac,
1369                                           TypedWriteTransaction<Configuration> writeConfigTxn,
1370                                           TypedWriteTransaction<Operational> writeOperTxn,
1371                                           TypedReadWriteTransaction<Configuration> writeInvTxn)
1372             throws ExecutionException, InterruptedException {
1373         //Read NextHops
1374         try {
1375             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1376                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1377             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1378             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1379                     LogicalDatastoreType.OPERATIONAL, path);
1380             boolean isNonPrimaryAdjIp = Boolean.FALSE;
1381             String primaryRd = vpnUtil.getVpnRd(vpnName);
1382             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1383                     interfaceName, dpnId, vpnName, primaryRd);
1384             if (adjacencies.isPresent() && adjacencies.get().getAdjacency() != null
1385                     && !adjacencies.get().getAdjacency().isEmpty()) {
1386                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1387                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1388                         interfaceName, dpnId, vpnName, nextHops);
1389                 for (Adjacency nextHop : nextHops) {
1390                     if (nextHop.isPhysNetworkFunc()) {
1391                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1392                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1393                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1394                                 null/*writeCfgTxn*/);
1395                     } else {
1396                         String rd = nextHop.getVrfId();
1397                         List<String> nhList;
1398                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1399                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1400                             isNonPrimaryAdjIp = Boolean.TRUE;
1401                         } else {
1402                             // This is a primary adjacency
1403                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1404                                     : emptyList();
1405                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1406                                     interfaceName, writeInvTxn);
1407                         }
1408                         if (!nhList.isEmpty()) {
1409                             if (Objects.equals(primaryRd, vpnName)) {
1410                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1411                                 //remove from FIB directly
1412                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1413                                         interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1414                             } else {
1415                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1416                                         interfaceName, writeConfigTxn, writeOperTxn);
1417                             }
1418                         } else {
1419                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1420                                             + " interface {}", nextHop.getIpAddress(), rd,
1421                                     nextHop.getAdjacencyType().toString(), interfaceName);
1422                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1423                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1424                         }
1425                     }
1426                     String ip = nextHop.getIpAddress().split("/")[0];
1427                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1428                     if (vpnVipToPort != null && vpnVipToPort.getPortName().equals(interfaceName)) {
1429                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1430                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1431                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1432                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1433                     }
1434                     // Remove the MIP-IP from VpnPortIpToPort.
1435                     if (isNonPrimaryAdjIp) {
1436                         VpnPortipToPort persistedIp = vpnUtil.getVpnPortipToPort(vpnName, ip);
1437                         if (persistedIp != null && persistedIp.isLearntIp()
1438                                 && persistedIp.getPortName().equals(interfaceName)) {
1439                             VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1440                             LOG.info(
1441                                     "removeAdjacenciesFromVpn: Learnt-IP: {} interface {} of vpn {} removed "
1442                                             + "from VpnPortipToPort",
1443                                     persistedIp.getPortFixedip(), persistedIp.getPortName(), vpnName);
1444                         }
1445                     }
1446                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1447                     if (vpnPortipToPort != null) {
1448                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1449                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1450                                  + "Interface {} ip {} on dpn {} for vpn {}",
1451                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1452                     }
1453                 }
1454             } else {
1455                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1456                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1457                         + " Removing it.", interfaceName, vpnName, dpnId);
1458                 writeOperTxn.delete(identifier);
1459             }
1460         } catch (ReadFailedException e) {
1461             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1462                     interfaceName, dpnId, vpnName);
1463         }
1464     }
1465
1466     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1467                                                             String interfaceName, BigInteger dpnId,
1468                                                             TypedWriteTransaction<Configuration> writeConfigTxn,
1469                                                             TypedWriteTransaction<Operational> writeOperTx) {
1470         return (nh) -> {
1471             String primaryRd = vpnUtil.getVpnRd(vpnName);
1472             String prefix = nextHop.getIpAddress();
1473             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1474             LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1475                     nextHop, vpnName, interfaceName, dpnId);
1476             synchronized (vpnNamePrefixKey.intern()) {
1477                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1478                         prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1479                     //If extra-route is present behind at least one VM, then do not remove or update
1480                     //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1481                     //prefixtointerface DS
1482                     return;
1483                 }
1484                 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1485                         writeConfigTxn);
1486             }
1487             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1488                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1489                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1490         };
1491     }
1492
1493     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1494                                            BigInteger dpnId, String rd, String interfaceName,
1495                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1496                                            TypedWriteTransaction<Operational> writeOperTx) {
1497         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1498                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1499         nhList.forEach((nh) -> {
1500             //IRT: remove routes from other vpns importing it
1501             vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1502                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1503             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1504                 String vpnRd = vpn.getVrfId();
1505                 if (vpnRd != null) {
1506                     fibManager.removeOrUpdateFibEntry(vpnRd,
1507                             nextHop.getIpAddress(), nh, writeConfigTxn);
1508                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1509                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1510                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1511                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1512                 }
1513             }
1514         });
1515     }
1516
1517     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1518                                                  int lportTag, String gwMac, String interfaceName,
1519                                                  TypedReadWriteTransaction<Configuration> writeInvTxn)
1520             throws ExecutionException, InterruptedException {
1521         final Uuid subnetId = nextHop.getSubnetId();
1522         if (nextHop.getSubnetGatewayMacAddress() == null) {
1523             // A valid mac-address was not available for this subnet-gateway-ip
1524             // So a connected-mac-address was used for this subnet and we need
1525             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1526             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1527         }
1528         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1529                 subnetId);
1530     }
1531
1532     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1533                                                   String interfaceName) {
1534         // This is either an extra-route (or) a learned IP via subnet-route
1535         List<String> nhList = null;
1536         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1537         if (nextHopIp == null || nextHopIp.isEmpty()) {
1538             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1539                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1540                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1541                     vpnName);
1542             nhList = emptyList();
1543         } else {
1544             nhList = Collections.singletonList(nextHopIp);
1545         }
1546         return nhList;
1547     }
1548
1549     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1550         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1551         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1552         // else use connected interface
1553         if (gwPort != null && gwPort.isSubnetIp()) {
1554             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1555                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1556             return Optional.of(gwPort.getMacAddress());
1557         }
1558         return Optional.absent();
1559     }
1560
1561     @Override
1562     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1563         final VpnInterface update) {
1564         LOG.trace("Received VpnInterface update event: original={}, update={}", original, update);
1565         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}", update.getName(),
1566                 update.getDpnId(), original.getVpnInstanceNames(), update.getVpnInstanceNames());
1567         final String vpnInterfaceName = update.getName();
1568         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1569         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1570         //handles switching between <internal VPN - external VPN>
1571         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1572             List<ListenableFuture<Void>> futures = new ArrayList<>();
1573             if (handleVpnInstanceUpdateForVpnInterface(identifier, original, update, futures)) {
1574                 LOG.info("update: handled Instance update for VPNInterface {} on dpn {} from oldVpn(s) {} "
1575                                 + "to newVpn(s) {}",
1576                         original.getName(), dpnId,
1577                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1578                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1579                 return emptyList();
1580             }
1581             updateVpnInstanceAdjChange(original, update, vpnInterfaceName, futures);
1582             return futures;
1583         });
1584     }
1585
1586     private boolean handleVpnInstanceUpdateForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1587                                                            VpnInterface original, VpnInterface update,
1588                                                            List<ListenableFuture<Void>> futures) {
1589         boolean isVpnInstanceUpdate = false;
1590         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1591         final String interfaceName = key.getName();
1592         List<String> oldVpnList = VpnUtil.getVpnListForVpnInterface(original);
1593         List<String> oldVpnListCopy = new ArrayList<>();
1594         oldVpnListCopy.addAll(oldVpnList);
1595         List<String> newVpnList = VpnUtil.getVpnListForVpnInterface(update);
1596         List<String> newVpnListCopy = new ArrayList<>();
1597         newVpnListCopy.addAll(newVpnList);
1598
1599         oldVpnList.removeAll(newVpnList);
1600         newVpnList.removeAll(oldVpnListCopy);
1601         //This block will execute only on if there is a change in the VPN Instance.
1602         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1603             /*
1604              * Internet BGP-VPN Instance update with single router:
1605              * ====================================================
1606              * In this case single VPN Interface will be part of maximum 2 VPN Instance only.
1607              *     1st VPN Instance : router VPN or external BGP-VPN.
1608              *     2nd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1609              *
1610              * VPN Instance UPDATE:
1611              * oldVpnList = 0 and newVpnList = 1 (Internet BGP-VPN)
1612              * oldVpnList = 1 and newVpnList = 0 (Internet BGP-VPN)
1613              *
1614              * External BGP-VPN Instance update with single router:
1615              * ====================================================
1616              * In this case single VPN interface will be part of maximum 1 VPN Instance only.
1617              *
1618              * Updated VPN Instance will be always either internal router VPN to
1619              * external BGP-VPN or external BGP-VPN to internal router VPN swap.
1620              *
1621              * VPN Instance UPDATE:
1622              * oldVpnList = 1 and newVpnList = 1 (router VPN to Ext-BGPVPN)
1623              * oldVpnList = 1 and newVpnList = 1 (Ext-BGPVPN to router VPN)
1624              *
1625              * Dual Router VPN Instance Update:
1626              * ================================
1627              * In this case single VPN interface will be part of maximum 3 VPN Instance only.
1628              *
1629              * 1st VPN Instance : router VPN or external BGP-VPN-1.
1630              * 2nd VPN Instance : router VPN or external BGP-VPN-2.
1631              * 3rd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1632              *
1633              * Dual Router --> Associated with common external BGP-VPN Instance.
1634              * 1st router and 2nd router are getting associated with single External BGP-VPN
1635              * 1) add 1st router to external bgpvpn --> oldVpnList=1, newVpnList=1;
1636              * 2) add 2nd router to the same external bgpvpn --> oldVpnList=1, newVpnList=0
1637              * In this case, we need to call removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1638              *
1639              *
1640              */
1641             isVpnInstanceUpdate = true;
1642             if (VpnUtil.isDualRouterVpnUpdate(oldVpnListCopy, newVpnListCopy)) {
1643                 if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1644                         && (oldVpnList.size() == 1 && newVpnList.size() == 0)) {
1645                     //Identify the external BGP-VPN Instance and pass that value as newVpnList
1646                     List<String> externalBgpVpnList = new ArrayList<>();
1647                     for (String newVpnName : newVpnListCopy) {
1648                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1649                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1650                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1651                                 .BgpvpnType.BGPVPNExternal) {
1652                             externalBgpVpnList.add(newVpnName);
1653                             break;
1654                         }
1655                     }
1656                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1657                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList,
1658                             externalBgpVpnList, oldVpnListCopy, futures);
1659
1660                 } else if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1661                         && (oldVpnList.size() == 0 && newVpnList.size() == 1)) {
1662                     //Identify the router VPN Instance and pass that value as oldVpnList
1663                     List<String> routerVpnList = new ArrayList<>();
1664                     for (String newVpnName : newVpnListCopy) {
1665                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1666                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1667                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1668                                 .BgpvpnType.VPN) {
1669                             routerVpnList.add(newVpnName);
1670                             break;
1671                         }
1672                     }
1673                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1674                     updateVpnInstanceChange(identifier, interfaceName, original, update, routerVpnList,
1675                             newVpnList, oldVpnListCopy, futures);
1676
1677                 } else {
1678                     //Handle remaining use cases.
1679                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1680                             oldVpnListCopy, futures);
1681                 }
1682             } else {
1683                 updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1684                         oldVpnListCopy, futures);
1685             }
1686         }
1687         return isVpnInstanceUpdate;
1688     }
1689
1690     private void updateVpnInstanceChange(InstanceIdentifier<VpnInterface> identifier, String interfaceName,
1691                                          VpnInterface original, VpnInterface update, List<String> oldVpnList,
1692                                          List<String> newVpnList, List<String> oldVpnListCopy,
1693                                          List<ListenableFuture<Void>> futures) {
1694         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1695         final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1696                 ? origAdjs.getAdjacency() : new ArrayList<>();
1697         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1698         final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1699                 ? updateAdjs.getAdjacency() : new ArrayList<>();
1700
1701         boolean isOldVpnRemoveCallExecuted = false;
1702         for (String oldVpnName : oldVpnList) {
1703             LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1704                     + "remove from vpnName {} ", interfaceName, oldVpnName);
1705             removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1706             LOG.info("updateVpnInstanceChange: Processed Remove for update on VPNInterface"
1707                             + " {} upon VPN update from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName,
1708                     newVpnList);
1709             isOldVpnRemoveCallExecuted = true;
1710         }
1711         //Wait for previous interface bindings to be removed
1712         if (isOldVpnRemoveCallExecuted && !newVpnList.isEmpty()) {
1713             try {
1714                 Thread.sleep(2000);
1715             } catch (InterruptedException e) {
1716                 //Ignore
1717             }
1718         }
1719         for (String newVpnName : newVpnList) {
1720             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1721             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1722                 LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1723                         + "onto vpnName {} ", interfaceName, newVpnName);
1724                 addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1725                 LOG.info("updateVpnInstanceChange: Processed Add for update on VPNInterface {}"
1726                                 + "from oldVpn(s) {} to newVpn {} ",
1727                         interfaceName, oldVpnListCopy, newVpnName);
1728                 /* This block will execute only if V6 subnet is associated with internet BGP-VPN.
1729                  * Use Case:
1730                  *     In Dual stack network, first V4 subnet only attached to router and router is associated
1731                  *     with internet BGP-VPN(router-gw). At this point VPN interface is having only router vpn instance.
1732                  *     Later V6 subnet is added to router, at this point existing VPN interface will get updated
1733                  *     with Internet BGP-VPN instance(Note: Internet BGP-VPN Instance update in vpn interface
1734                  *     is applicable for only on V6 subnet is added to router). newVpnList = Contains only Internet
1735                  *     BGP-VPN Instance. So we required V6 primary adjacency info needs to be populated onto
1736                  *     router VPN as well as Internet BGP-VPN.
1737                  *
1738                  *     addVpnInterfaceCall() --> It will create V6 Adj onto Internet BGP-VPN only.
1739                  *     updateVpnInstanceAdjChange() --> This method call is needed for second primary V6 Adj
1740                  *                                       update in existing router VPN instance.
1741                  */
1742                 if (vpnUtil.isBgpVpnInternet(newVpnName)) {
1743                     LOG.info("updateVpnInstanceChange: VPN Interface {} with new Adjacency {} in existing "
1744                             + "VPN instance {}", interfaceName, newAdjs, original.getVpnInstanceNames());
1745                     updateVpnInstanceAdjChange(original, update, interfaceName, futures);
1746                 }
1747             }
1748         }
1749     }
1750
1751     private List<ListenableFuture<Void>> updateVpnInstanceAdjChange(VpnInterface original, VpnInterface update,
1752                                                                     String vpnInterfaceName,
1753                                                                     List<ListenableFuture<Void>> futures) {
1754         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1755         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1756                 != null ? origAdjs.getAdjacency() : new ArrayList<>();
1757         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1758         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1759                 != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1760
1761         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1762         for (VpnInstanceNames vpnInterfaceVpnInstance : update.nonnullVpnInstanceNames()) {
1763             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1764             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1765             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1766             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1767             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1768                 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1769                 //set of prefix used as entry in prefix-to-interface datastore
1770                 // is prerequisite for refresh Fib to avoid race condition leading to missing remote next hop
1771                 // in bucket actions on bgp-vpn delete
1772                 Set<String> prefixListForRefreshFib = new HashSet<>();
1773                 ListenableFuture<Void> configTxFuture = txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
1774                     confTx -> futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL,
1775                         operTx -> {
1776                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1777                                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1778                             LOG.info("VPN Interface update event-intfName {} onto vpnName {} running config-driven",
1779                                     update.getName(), newVpnName);
1780                             //handle both addition and removal of adjacencies
1781                             // currently, new adjacency may be an extra route
1782                             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1783                             if (!oldAdjs.equals(newAdjs)) {
1784                                 for (Adjacency adj : copyNewAdjs) {
1785                                     if (copyOldAdjs.contains(adj)) {
1786                                         copyOldAdjs.remove(adj);
1787                                     } else {
1788                                         // add new adjacency
1789                                         if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1790                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1791                                                     dpnId, operTx, confTx, confTx, prefixListForRefreshFib);
1792                                         }
1793                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} "
1794                                             + " added to vpn interface {} on vpn {} dpnId {}",
1795                                             adj.getIpAddress(), adj.getNextHopIpList(), adj.getLabel(),
1796                                             adj.getSubnetId(), update.getName(), newVpnName, dpnId);
1797                                     }
1798                                 }
1799                                 for (Adjacency adj : copyOldAdjs) {
1800                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1801                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1802                                             && !adj.isPhysNetworkFunc()) {
1803                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, operTx,
1804                                                 confTx);
1805                                             //remove FIB entry
1806                                             String vpnRd = vpnUtil.getVpnRd(newVpnName);
1807                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1808                                                 + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1809                                             //remove BGP entry
1810                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1811                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1812                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1813                                             }
1814                                         } else {
1815                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1816                                                 operTx, confTx);
1817                                         }
1818                                     }
1819                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1820                                         + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1821                                         adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1822                                 }
1823                             }
1824                         })));
1825                 Futures.addCallback(configTxFuture, new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
1826                     MoreExecutors.directExecutor());
1827                 futures.add(configTxFuture);
1828                 for (ListenableFuture<Void> future : futures) {
1829                     ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1830                             update.getName(), update.getVpnInstanceNames());
1831                 }
1832             } else {
1833                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1834                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1835             }
1836         }
1837         return futures;
1838     }
1839
1840     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1841         try {
1842             Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1843             synchronized (label.toString().intern()) {
1844                 InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1845                         .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1846                 Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1847                         LogicalDatastoreType.OPERATIONAL, lriIid);
1848                 if (opResult.isPresent()) {
1849                     LabelRouteInfo labelRouteInfo =
1850                             new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1851                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1852                             labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1853                 }
1854             }
1855             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1856                     nextHopIpList);
1857         } catch (ReadFailedException e) {
1858             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1859                     nextHopIpList);
1860         } catch (TransactionCommitFailedException e) {
1861             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1862                     nextHopIpList);
1863         }
1864     }
1865
1866     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1867         SubnetRoute route, String parentVpnRd, TypedWriteTransaction<Configuration> writeConfigTxn) {
1868
1869         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1870         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1871                 .addAugmentation(SubnetRoute.class, route).build();
1872         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1873         InstanceIdentifierBuilder<VrfTables> idBuilder =
1874             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1875         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1876         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1877         if (writeConfigTxn != null) {
1878             writeConfigTxn.merge(vrfTableId, vrfTableNew, CREATE_MISSING_PARENTS);
1879         } else {
1880             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1881         }
1882         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1883                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1884     }
1885
1886     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1887                                            Adjacency adj, BigInteger dpnId,
1888                                            TypedWriteTransaction<Operational> writeOperTxn,
1889                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1890                                            TypedReadWriteTransaction<Configuration> writeInvTxn,
1891                                            Set<String> prefixListForRefreshFib)
1892             throws ExecutionException, InterruptedException {
1893         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1894         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1895         try {
1896             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1897                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1898             if (optVpnInterface.isPresent()) {
1899                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1900                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1901                 String vpnName = currVpnIntf.getVpnInstanceName();
1902                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1903                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1904                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1905                         LogicalDatastoreType.OPERATIONAL, adjPath);
1906                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1907                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1908                 long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1909                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1910                 List<Adjacency> adjacencies = new ArrayList<>();
1911                 if (optAdjacencies.isPresent() && optAdjacencies.get().getAdjacency() != null) {
1912                     adjacencies.addAll(optAdjacencies.get().getAdjacency());
1913                 }
1914                 long vpnId = vpnUtil.getVpnId(vpnName);
1915                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1916                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1917                 Adjacency operationalAdjacency = null;
1918                 //Handling dual stack neutron port primary adjacency
1919                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1920                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1921                             currVpnIntf.getName(), vpnName);
1922                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1923                             currVpnIntf.getName());
1924                     if (interfaceState != null) {
1925                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1926                             currVpnIntf.getName(), vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState,
1927                             prefixListForRefreshFib);
1928                     }
1929                 }
1930                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1931                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1932                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.LearntIp ? RouteOrigin.DYNAMIC
1933                             : RouteOrigin.STATIC;
1934                     String nh = adj.getNextHopIpList().get(0);
1935                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1936                     synchronized (vpnPrefixKey.intern()) {
1937                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1938                                 vpnId, null, prefix, vpnName, nh, dpnId);
1939                         if (rdToAllocate.isPresent()) {
1940                             input.setRd(rdToAllocate.get());
1941                             operationalAdjacency = populator.createOperationalAdjacency(input);
1942                             int label = operationalAdjacency.getLabel().intValue();
1943                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1944                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
1945                                     currVpnIntf.getName(), operationalAdjacency, encapType,
1946                                     prefixListForRefreshFib, writeConfigTxn);
1947                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1948                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1949                                     vpnName, label, currVpnIntf.getName(), dpnId);
1950                         } else {
1951                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
1952                                     vpnName, prefix);
1953                             return;
1954                         }
1955                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1956                         // Keeping the MPLS check for now.
1957                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1958                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1959                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1960                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
1961                             vpnsToImportRoute.forEach(vpn -> {
1962                                 if (vpn.getVrfId() != null) {
1963                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(), vpnId, prefix,
1964                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
1965                                             .ifPresent(
1966                                                 rds -> vpnManager.addExtraRoute(
1967                                                         vpnUtil.getVpnName(vpn.getVpnId()), adj.getIpAddress(),
1968                                                         nh, rds, currVpnIntf.getVpnInstanceName(), l3vni,
1969                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(), opAdjacency,
1970                                                         encapType, prefixListForRefreshFib, writeConfigTxn));
1971                                 }
1972                             });
1973                         }
1974                     }
1975                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1976                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1977                             currVpnIntf.getName(), vpnName);
1978
1979                     InstanceIdentifier<VpnInterface> vpnIfaceConfigidentifier = VpnUtil
1980                             .getVpnInterfaceIdentifier(currVpnIntf.getName());
1981                     Optional<VpnInterface> vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1982                             LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier);
1983                     Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1984                             Prefixes.PrefixCue.PhysNetFunc);
1985                     if (vpnIntefaceConfig.isPresent()) {
1986                         pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1987                                 vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(),
1988                                 vpnIntefaceConfig.get().getSegmentationId(), Prefixes.PrefixCue.PhysNetFunc);
1989                     }
1990
1991                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1992
1993                     writeOperTxn.merge(
1994                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
1995                                     prefix), pnfPrefix, true);
1996
1997                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1998                             adj.getIpAddress(), emptyList(), null /* EncapType */, 0 /* label */,
1999                             0 /*l3vni*/, null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
2000
2001                     input.setRd(adj.getVrfId());
2002                 }
2003                 if (operationalAdjacency == null) {
2004                     operationalAdjacency = populator.createOperationalAdjacency(input);
2005                 }
2006                 adjacencies.add(operationalAdjacency);
2007                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
2008                 VpnInterfaceOpDataEntry newVpnIntf =
2009                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
2010                                 aug, dpnId, currVpnIntf.getLportTag(),
2011                                 currVpnIntf.getGatewayMacAddress());
2012
2013                 writeOperTxn.merge(identifier, newVpnIntf, CREATE_MISSING_PARENTS);
2014             }
2015         } catch (ReadFailedException e) {
2016             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
2017                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
2018         }
2019     }
2020
2021     @Nullable
2022     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
2023         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
2024         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
2025     }
2026
2027     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
2028                                             BigInteger dpnId, TypedWriteTransaction<Operational> writeOperTxn,
2029                                             TypedWriteTransaction<Configuration> writeConfigTxn) {
2030         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
2031         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
2032         try {
2033             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
2034                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
2035             if (optVpnInterface.isPresent()) {
2036                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
2037                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
2038                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2039                         LogicalDatastoreType.OPERATIONAL, path);
2040                 if (optAdjacencies.isPresent()) {
2041                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
2042
2043                     if (adjacencies != null && !adjacencies.isEmpty()) {
2044                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
2045                         for (Adjacency adjacency : adjacencies) {
2046                             if (Objects.equals(adjacency.getIpAddress(), adj.getIpAddress())) {
2047                                 String rd = adjacency.getVrfId();
2048                                 if (adj.getNextHopIpList() != null) {
2049                                     for (String nh : adj.getNextHopIpList()) {
2050                                         deleteExtraRouteFromCurrentAndImportingVpns(
2051                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
2052                                                 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
2053                                     }
2054                                 } else if (adj.isPhysNetworkFunc()) {
2055                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
2056                                             adj.getIpAddress(), adj.getSubnetId());
2057                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
2058                                             writeConfigTxn);
2059                                 }
2060                                 break;
2061                             }
2062
2063                         }
2064                     }
2065                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
2066                             dpnId, adj.getVrfId());
2067                 } else {
2068                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
2069                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
2070                 }
2071             }
2072         } catch (ReadFailedException e) {
2073             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
2074                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
2075         }
2076     }
2077
2078     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
2079                                     String rd, String intfName, TypedWriteTransaction<Configuration> writeConfigTxn,
2080                                     TypedWriteTransaction<Operational> writeOperTx) {
2081         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
2082         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
2083         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
2084             String vpnRd = vpn.getVrfId();
2085             if (vpnRd != null) {
2086                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
2087                         writeOperTx);
2088             }
2089         }
2090     }
2091
2092     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
2093         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2094             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
2095             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
2096     }
2097
2098     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
2099         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2100             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
2101     }
2102
2103     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2104                                                 TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2105         if (vpnInterface == null) {
2106             return;
2107         }
2108         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
2109         if (adjs == null) {
2110             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2111                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2112             return;
2113         }
2114         for (Adjacency adj : adjs) {
2115             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2116                 String primaryInterfaceIp = adj.getIpAddress();
2117                 String macAddress = adj.getMacAddress();
2118                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2119
2120                 long label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
2121                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2122
2123                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2124                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2125                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
2126                         routerInt, label, writeConfigTxn);
2127                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2128                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2129                         macAddress);
2130             } else {
2131                 LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2132                                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}",
2133                         interfaceName, primaryRd, vpnName);
2134             }
2135         }
2136     }
2137
2138     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
2139             TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2140         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
2141         String rd = vpnUtil.getVpnRd(vpnName);
2142         if (adjs != null) {
2143             List<Adjacency> adjsList = adjs.nonnullAdjacency();
2144             for (Adjacency adj : adjsList) {
2145                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2146                     String primaryInterfaceIp = adj.getIpAddress();
2147                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2148                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
2149                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2150                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2151                 }
2152             }
2153         } else {
2154             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2155                     vpnInterface.getName(), rd);
2156         }
2157     }
2158
2159     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2160         final VpnInterfaceKey key = intefaceData.identifier.firstKeyOf(VpnInterface.class);
2161         final String interfaceName = key.getName();
2162         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2163                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2164         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2165                   intefaceData.identifier, vpnName);
2166     }
2167
2168     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2169                                               VpnInterface vpnInterface, String vpnName) {
2170         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2171                .get(vpnName);
2172         if (vpnInterfaces == null) {
2173             vpnInterfaces = new ConcurrentLinkedQueue<>();
2174         }
2175         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2176         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2177         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2178                 vpnInterface.getName(), vpnName);
2179     }
2180
2181     public boolean isVpnInstanceReady(String vpnInstanceName) {
2182         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2183         if (vpnRd == null) {
2184             return false;
2185         }
2186         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2187
2188         return vpnInstanceOpDataEntry != null;
2189     }
2190
2191     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2192         synchronized (vpnInstanceName.intern()) {
2193             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2194                     unprocessedVpnInterfaces.get(vpnInstanceName);
2195             if (vpnInterfaces != null) {
2196                 while (!vpnInterfaces.isEmpty()) {
2197                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2198                     if (hasVpnInstanceCreatedSuccessfully) {
2199                         processSavedInterface(savedInterface, vpnInstanceName);
2200                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2201                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2202                     } else {
2203                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2204                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2205                     }
2206                 }
2207             } else {
2208                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2209             }
2210         }
2211     }
2212
2213     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2214             VpnInterface vpnInterface) {
2215         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2216             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2217                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2218             if (vpnInterfaces != null) {
2219                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2220                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2221                             + "unprocessed list", vpnInterface.getName(),
2222                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2223                 }
2224             } else {
2225                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2226                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2227             }
2228         }
2229     }
2230
2231     public void vpnInstanceIsReady(String vpnInstanceName) {
2232         processSavedInterfaces(vpnInstanceName, true);
2233     }
2234
2235     public void vpnInstanceFailed(String vpnInstanceName) {
2236         processSavedInterfaces(vpnInstanceName, false);
2237     }
2238
2239     private static class UnprocessedVpnInterfaceData {
2240         InstanceIdentifier<VpnInterface> identifier;
2241         VpnInterface vpnInterface;
2242
2243         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2244             this.identifier = identifier;
2245             this.vpnInterface = vpnInterface;
2246         }
2247
2248         @Override
2249         public int hashCode() {
2250             final int prime = 31;
2251             int result = 1;
2252             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2253             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2254             return result;
2255         }
2256
2257         @Override
2258         public boolean equals(Object obj) {
2259             if (this == obj) {
2260                 return true;
2261             }
2262             if (obj == null) {
2263                 return false;
2264             }
2265             if (getClass() != obj.getClass()) {
2266                 return false;
2267             }
2268             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2269             if (identifier == null) {
2270                 if (other.identifier != null) {
2271                     return false;
2272                 }
2273             } else if (!identifier.equals(other.identifier)) {
2274                 return false;
2275             }
2276             if (vpnInterface == null) {
2277                 if (other.vpnInterface != null) {
2278                     return false;
2279                 }
2280             } else if (!vpnInterface.equals(other.vpnInterface)) {
2281                 return false;
2282             }
2283             return true;
2284         }
2285     }
2286
2287     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2288         String primaryRd = vpnUtil.getVpnRd(vpnName);
2289         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2290         if (vpnInstanceOpData == null) {
2291             return;
2292         }
2293         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2294         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2295             return;
2296         }
2297         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2298         vpnToDpnLists.forEach(vpnToDpnList -> {
2299             if (vpnToDpnList.getVpnInterfaces() == null) {
2300                 return;
2301             }
2302             vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2303                 try {
2304                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2305                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2306                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2307                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2308                     if (!vpnInterfaceOptional.isPresent()) {
2309                         return;
2310                     }
2311                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2312                             vpnInterface.getInterfaceName());
2313                     if (configVpnAdjacencies == null) {
2314                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2315                         return;
2316                     }
2317                     List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2318                             .augmentation(AdjacenciesOp.class).nonnullAdjacency();
2319                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2320                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2321                     // in operational DS. These unprocessed adjacencies will be handled below.
2322                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2323                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2324                     configVpnAdjacencies.stream()
2325                         .filter(adjacency -> operationVpnAdjacencies.stream()
2326                                 .noneMatch(operationalAdjacency ->
2327                                     Objects.equals(operationalAdjacency.getIpAddress(), adjacency.getIpAddress())))
2328                         .forEach(adjacency -> {
2329                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2330                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2331                                 () -> {
2332                                     // TODO Deal with sequencing — the config tx must only submitted
2333                                     // if the oper tx goes in
2334                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2335                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2336                                         futures.add(
2337                                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
2338                                                 //set of prefix used, as entry in prefix-to-interface datastore
2339                                                 // is prerequisite for refresh Fib to avoid race condition leading
2340                                                 // to missing remote next hop in bucket actions on bgp-vpn delete
2341                                                 Set<String> prefixListForRefreshFib = new HashSet<>();
2342                                                 ListenableFuture<Void> configTxFuture =
2343                                                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
2344                                                         confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2345                                                             primaryRd, adjacency, vpnInterfaceOptional.get().getDpnId(),
2346                                                                 operTx, confTx, confTx, prefixListForRefreshFib));
2347                                                 Futures.addCallback(configTxFuture,
2348                                                     new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
2349                                                     MoreExecutors.directExecutor());
2350                                                 futures.add(configTxFuture);
2351                                             }));
2352                                         return futures;
2353                                     } else {
2354                                         return emptyList();
2355                                     }
2356                                 });
2357                         });
2358                 } catch (ReadFailedException e) {
2359                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2360                             vpnName, primaryRd);
2361                 }
2362             });
2363         });
2364     }
2365
2366     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2367         private final String interfaceName;
2368         private final boolean add;
2369         private final String txnDestination;
2370
2371         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2372             this.interfaceName = interfaceName;
2373             this.add = add;
2374             this.txnDestination = transactionDest;
2375         }
2376
2377         @Override
2378         public void onSuccess(Void voidObj) {
2379             if (add) {
2380                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2381                         interfaceName, txnDestination);
2382             } else {
2383                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2384             }
2385         }
2386
2387         @Override
2388         public void onFailure(Throwable throwable) {
2389             if (add) {
2390                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2391                         interfaceName, txnDestination, throwable);
2392             } else {
2393                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2394                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2395             }
2396         }
2397     }
2398
2399     private class VpnInterfaceCallBackHandler implements FutureCallback<Void> {
2400         private final String primaryRd;
2401         private final Set<String> prefixListForRefreshFib;
2402
2403         VpnInterfaceCallBackHandler(String primaryRd, Set<String> prefixListForRefreshFib) {
2404             this.primaryRd = primaryRd;
2405             this.prefixListForRefreshFib = prefixListForRefreshFib;
2406         }
2407
2408         @Override
2409         public void onSuccess(Void voidObj) {
2410             prefixListForRefreshFib.forEach(prefix -> {
2411                 fibManager.refreshVrfEntry(primaryRd, prefix);
2412             });
2413         }
2414
2415         @Override
2416         public void onFailure(Throwable throwable) {
2417             LOG.debug("write Tx config operation failed {}", throwable);
2418         }
2419     }
2420 }