vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import static java.util.Collections.emptyList;
  11 import static org.opendaylight.controller.md.sal.binding.api.WriteTransaction.CREATE_MISSING_PARENTS;
  12 import static org.opendaylight.genius.infra.Datastore.CONFIGURATION;
  13 import static org.opendaylight.genius.infra.Datastore.OPERATIONAL;
  14
  15 import com.google.common.base.Optional;
  16 import com.google.common.base.Preconditions;
  17 import com.google.common.collect.Iterators;
  18 import com.google.common.util.concurrent.FutureCallback;
  19 import com.google.common.util.concurrent.Futures;
  20 import com.google.common.util.concurrent.ListenableFuture;
  21 import com.google.common.util.concurrent.MoreExecutors;
  22 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  23 import java.math.BigInteger;
  24 import java.util.ArrayList;
  25 import java.util.Collections;
  26 import java.util.HashSet;
  27 import java.util.List;
  28 import java.util.Map;
  29 import java.util.Objects;
  30 import java.util.Set;
  31 import java.util.concurrent.ConcurrentHashMap;
  32 import java.util.concurrent.ConcurrentLinkedQueue;
  33 import java.util.concurrent.ExecutionException;
  34 import java.util.function.Consumer;
  35 import java.util.function.Predicate;
  36 import java.util.stream.Collectors;
  37 import javax.annotation.Nullable;
  38 import javax.annotation.PostConstruct;
  39 import javax.annotation.PreDestroy;
  40 import javax.inject.Inject;
  41 import javax.inject.Singleton;
  42 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  43 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  44 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  45 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  46 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  47 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  48 import org.opendaylight.genius.infra.Datastore.Configuration;
  49 import org.opendaylight.genius.infra.Datastore.Operational;
  50 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  51 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  52 import org.opendaylight.genius.infra.TransactionAdapter;
  53 import org.opendaylight.genius.infra.TypedReadWriteTransaction;
  54 import org.opendaylight.genius.infra.TypedWriteTransaction;
  55 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  56 import org.opendaylight.genius.mdsalutil.NWUtil;
  57 import org.opendaylight.genius.mdsalutil.NwConstants;
  58 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  59 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  60 import org.opendaylight.infrautils.caches.CacheProvider;
  61 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  62 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  63 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  64 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  65 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  66 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  67 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  68 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  69 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  70 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  71 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  72 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  73 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  74 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  75 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  76 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  77 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  78 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  79 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 116 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 117 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType;
 118 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 119 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 120 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 121 import org.slf4j.Logger;
 122 import org.slf4j.LoggerFactory;
 123
 124 @Singleton
 125 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 126
 127     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 128     private static final short DJC_MAX_RETRIES = 3;
 129
 130     private final DataBroker dataBroker;
 131     private final ManagedNewTransactionRunner txRunner;
 132     private final IBgpManager bgpManager;
 133     private final IFibManager fibManager;
 134     private final IMdsalApiManager mdsalManager;
 135     private final IdManagerService idManager;
 136     private final OdlInterfaceRpcService ifaceMgrRpcService;
 137     private final VpnFootprintService vpnFootprintService;
 138     private final IInterfaceManager interfaceManager;
 139     private final IVpnManager vpnManager;
 140     private final ArpResponderHandler arpResponderHandler;
 141     private final JobCoordinator jobCoordinator;
 142     private final VpnUtil vpnUtil;
 143
 144     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 145
 146     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 147             new ConcurrentHashMap<>();
 148
 149     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 150
 151     @Inject
 152     public VpnInterfaceManager(final DataBroker dataBroker,
 153                                final IBgpManager bgpManager,
 154                                final IdManagerService idManager,
 155                                final IMdsalApiManager mdsalManager,
 156                                final IFibManager fibManager,
 157                                final OdlInterfaceRpcService ifaceMgrRpcService,
 158                                final VpnFootprintService vpnFootprintService,
 159                                final IInterfaceManager interfaceManager,
 160                                final IVpnManager vpnManager,
 161                                final ArpResponderHandler arpResponderHandler,
 162                                final JobCoordinator jobCoordinator,
 163                                final CacheProvider cacheProvider,
 164                                final VpnUtil vpnUtil) {
 165         super(VpnInterface.class, VpnInterfaceManager.class);
 166
 167         this.dataBroker = dataBroker;
 168         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 169         this.bgpManager = bgpManager;
 170         this.idManager = idManager;
 171         this.mdsalManager = mdsalManager;
 172         this.fibManager = fibManager;
 173         this.ifaceMgrRpcService = ifaceMgrRpcService;
 174         this.vpnFootprintService = vpnFootprintService;
 175         this.interfaceManager = interfaceManager;
 176         this.vpnManager = vpnManager;
 177         this.arpResponderHandler = arpResponderHandler;
 178         this.jobCoordinator = jobCoordinator;
 179         this.vpnUtil = vpnUtil;
 180
 181         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 182                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 183                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 184     }
 185
 186     public Runnable isNotifyTaskQueued(String intfName) {
 187         return vpnIntfMap.remove(intfName);
 188     }
 189
 190     @PostConstruct
 191     public void start() {
 192         LOG.info("{} start", getClass().getSimpleName());
 193         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 194     }
 195
 196     @Override
 197     @PreDestroy
 198     public void close() {
 199         super.close();
 200         vpnInstanceOpDataEntryCache.close();
 201     }
 202
 203     @Override
 204     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 205         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 206     }
 207
 208     @Override
 209     protected VpnInterfaceManager getDataTreeChangeListener() {
 210         return VpnInterfaceManager.this;
 211     }
 212
 213     @Override
 214     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 215         LOG.trace("Received VpnInterface add event: vpnInterface={}", vpnInterface);
 216         LOG.info("add: intfName {} onto vpnName {}", vpnInterface.getName(),
 217                 VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 218         addVpnInterface(identifier, vpnInterface, null, null);
 219     }
 220
 221     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 222                           final VpnInterface vpnInterface, String vpnName) {
 223         synchronized (vpnName.intern()) {
 224             if (isVpnInstanceReady(vpnName)) {
 225                 return true;
 226             }
 227             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 228             return false;
 229         }
 230     }
 231
 232     // TODO Clean up the exception handling
 233     @SuppressWarnings("checkstyle:IllegalCatch")
 234     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 235                              final @Nullable List<Adjacency> oldAdjs, final @Nullable List<Adjacency> newAdjs) {
 236         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
 237             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 238             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 239         }
 240     }
 241
 242     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 243                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 244         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 245         final String interfaceName = key.getName();
 246
 247         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 248             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 249                   vpnName, vpnInterface.getName());
 250             return;
 251         }
 252         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 253              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 254         List<Adjacency> copyOldAdjs = null;
 255         if (oldAdjs != null) {
 256             copyOldAdjs = new ArrayList<>();
 257             copyOldAdjs.addAll(oldAdjs);
 258         }
 259         List<Adjacency> copyNewAdjs = null;
 260         if (newAdjs != null) {
 261             copyNewAdjs = new ArrayList<>();
 262             copyNewAdjs.addAll(newAdjs);
 263         }
 264         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 265     }
 266
 267     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 268                     final VpnInterface vpnInterface, final @Nullable List<Adjacency> oldAdjs,
 269                     final @Nullable List<Adjacency> newAdjs,
 270                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 271         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
 272         final String interfaceName = key.getName();
 273         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 274         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 275             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 276             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 277             if (interfaceState != null) {
 278                 try {
 279                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 280                     final int ifIndex = interfaceState.getIfIndex();
 281                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 282                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 283                         // (the inventory tx goes in last)
 284                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 285                         //set of prefix used, as entry in prefix-to-interface datastore
 286                         // is prerequisite for refresh Fib to avoid race condition leading to
 287                         // missing remote next hop in bucket actions on bgp-vpn delete
 288                         Set<String> prefixListForRefreshFib = new HashSet<>();
 289                         ListenableFuture<Void> confFuture =
 290                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
 291                                 confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
 292                                     operTx -> futures.add(
 293                                         txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, invTx -> {
 294                                             LOG.info(
 295                                                 "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 296                                                     + " on dpn {}",
 297                                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 298                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 299                                                 confTx, operTx, invTx, interfaceState, vpnName,
 300                                                 prefixListForRefreshFib);
 301                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 302                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 303                                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 304                                                 if (newAdjs != null) {
 305                                                     for (Adjacency adj : newAdjs) {
 306                                                         if (oldAdjs.contains(adj)) {
 307                                                             oldAdjs.remove(adj);
 308                                                         } else {
 309                                                             if (!isBgpVpnInternetVpn
 310                                                                 || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 311                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 312                                                                     primaryRd, adj, dpnId, operTx, confTx, invTx,
 313                                                                     prefixListForRefreshFib);
 314                                                             }
 315                                                         }
 316                                                     }
 317                                                 }
 318                                                 for (Adjacency adj : oldAdjs) {
 319                                                     if (!isBgpVpnInternetVpn
 320                                                         || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 321                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 322                                                             operTx, confTx);
 323                                                     }
 324                                                 }
 325                                             }
 326                                         })))));
 327                         Futures.addCallback(confFuture,
 328                             new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
 329                             MoreExecutors.directExecutor());
 330                         futures.add(confFuture);
 331                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 332                             MoreExecutors.directExecutor());
 333                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 334                             + " processed successfully", interfaceName, vpnName, dpnId);
 335                         return futures;
 336                     });
 337                 } catch (NumberFormatException | IllegalStateException e) {
 338                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 339                             + "interface {}. Interface addition on vpn {} failed", interfaceName,
 340                         vpnName, e);
 341                     return;
 342                 }
 343             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 344                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 345                     () -> {
 346                         ListenableFuture<Void> future =
 347                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
 348                                 createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 349                                     confTx, vpnName);
 350                                 LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 351                                     vpnName, vpnInterface.getDpnId());
 352                             });
 353                         ListenableFutures.addErrorLogging(future, LOG,
 354                             "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 355                         return Collections.singletonList(future);
 356                     });
 357             } else {
 358                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 359                     + " is not available", interfaceName, vpnName);
 360             }
 361         } else {
 362             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 363                     + " as vpn is pending delete", interfaceName, vpnName,
 364                 vpnInterface.getDpnId());
 365         }
 366     }
 367
 368     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 369     // see comments below.
 370     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 371     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 372             final int lportTag, boolean isInterfaceUp,
 373             TypedWriteTransaction<Configuration> writeConfigTxn,
 374             TypedWriteTransaction<Operational> writeOperTxn,
 375             TypedReadWriteTransaction<Configuration> writeInvTxn,
 376             Interface interfaceState, final String vpnName,
 377             Set<String> prefixListForRefreshFib) throws ExecutionException, InterruptedException {
 378         final String interfaceName = vpnInterface.getName();
 379         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 380                 vpnName);
 381         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 382         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 383         if (!isInterfaceUp) {
 384             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 385                      interfaceName, dpId, vpnName);
 386             long vpnId = vpnUtil.getVpnId(vpnName);
 387             if (vpnId == VpnConstants.INVALID_ID) {
 388                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 389                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 390                         dpId);
 391                 return;
 392             }
 393
 394             boolean waitForVpnInterfaceOpRemoval = false;
 395             if (opVpnInterface != null) {
 396                 String opVpnName = opVpnInterface.getVpnInstanceName();
 397                 String primaryInterfaceIp = null;
 398                 if (Objects.equals(opVpnName, vpnName)) {
 399                     // Please check if the primary VRF Entry does not exist for VPNInterface
 400                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 401                     // back to back
 402                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 403                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 404                     if (adjs == null) {
 405                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 406                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 407                                 vpnName);
 408                         return;
 409                     }
 410                     for (Adjacency adj : adjs) {
 411                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 412                             primaryInterfaceIp = adj.getIpAddress();
 413                             break;
 414                         }
 415                     }
 416                     if (primaryInterfaceIp == null) {
 417                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 418                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 419                                 dpId, vpnName);
 420                         return;
 421                     }
 422                     // Get the rd of the vpn instance
 423                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 424                     if (vrf != null) {
 425                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 426                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 427                         return;
 428                     }
 429                     waitForVpnInterfaceOpRemoval = true;
 430                 } else {
 431                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 432                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 433                 }
 434             }
 435             if (!waitForVpnInterfaceOpRemoval) {
 436                 // Add the VPNInterface and quit
 437                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 438                         null/*ipAddressSourceValuePair*/,
 439                         true /* add */);
 440                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 441                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 442                 if (!isBgpVpnInternetVpn) {
 443                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 444                 }
 445                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 446                         dpId, vpnName);
 447                 if (interfaceManager.isExternalInterface(interfaceName)) {
 448                     processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
 449                         NwConstants.ADD_FLOW);
 450                 }
 451                 return;
 452             }
 453
 454             // FIB didn't get a chance yet to clean up this VPNInterface
 455             // Let us give it a chance here !
 456             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 457                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 458             try {
 459                 Runnable notifyTask = new VpnNotifyTask();
 460                 synchronized (notifyTask) {
 461                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 462                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 463                     // published here while holding the lock on it so this path will hit the wait before notify can be
 464                     // invoked.
 465                     vpnIntfMap.put(interfaceName, notifyTask);
 466                     try {
 467                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 468                     } catch (InterruptedException e) {
 469                         // Ignored
 470                     }
 471                 }
 472             } finally {
 473                 vpnIntfMap.remove(interfaceName);
 474             }
 475
 476             if (opVpnInterface != null) {
 477                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 478                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 479                         dpId, vpnName);
 480                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 481                 return;
 482             }
 483             // VPNInterface got removed, proceed with Add
 484             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 485                     interfaceName, dpId, vpnName);
 486             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 487                     null/*ipAddressSourceValuePair*/,
 488                     true /* add */);
 489             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 490                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState, prefixListForRefreshFib);
 491             if (!isBgpVpnInternetVpn) {
 492                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 493             }
 494             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 495                     + " FIB to clean up", interfaceName, dpId, vpnName);
 496             if (interfaceManager.isExternalInterface(interfaceName)) {
 497                 processExternalVpnInterface(interfaceName, vpnName, dpId,
 498                                lportTag, NwConstants.ADD_FLOW);
 499             }
 500         } else {
 501             try {
 502                 // Interface is retained in the DPN, but its Link Up.
 503                 // Advertise prefixes again for this interface to BGP
 504                 InstanceIdentifier<VpnInterface> identifier =
 505                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 506                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 507                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 508                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 509                 // Perform similar operation as interface add event for extraroutes.
 510                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 511                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 512                         LogicalDatastoreType.CONFIGURATION, path);
 513                 if (!optAdjacencies.isPresent()) {
 514                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 515                     return;
 516                 }
 517                 List<Adjacency> adjacencies = optAdjacencies.get().nonnullAdjacency();
 518                 for (Adjacency adjacency : adjacencies) {
 519                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 520                         continue;
 521                     }
 522                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 523                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 524                         continue;
 525                     }
 526                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 527                             dpId, writeOperTxn, writeConfigTxn, writeInvTxn, prefixListForRefreshFib);
 528                 }
 529             } catch (ReadFailedException e) {
 530                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 531                         interfaceName, vpnName, primaryRd, dpId);
 532             }
 533         }
 534     }
 535
 536     private void processExternalVpnInterface(String interfaceName, String vpnName, BigInteger dpId,
 537         int lportTag, int addOrRemove) {
 538         Uuid extNetworkId;
 539         try {
 540             // vpn instance of ext-net interface is the network-id
 541             extNetworkId = new Uuid(vpnName);
 542         } catch (IllegalArgumentException e) {
 543             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 544                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 545             return;
 546         }
 547
 548         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 549         if (routerIds == null || routerIds.isEmpty()) {
 550             LOG.info("processExternalVpnInterface: No router is associated with {}."
 551                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 552                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 553             return;
 554         }
 555
 556         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 557                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 558         for (Uuid routerId : routerIds) {
 559             String routerName = routerId.getValue();
 560             BigInteger primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 561             if (Objects.equals(primarySwitch, dpId)) {
 562                 Routers router = vpnUtil.getExternalRouter(routerName);
 563                 if (router != null) {
 564                     if (addOrRemove == NwConstants.ADD_FLOW) {
 565                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 566                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 567                                 dpId, interfaceName, lportTag);
 568                     } else {
 569                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 570                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 571                                 dpId, interfaceName, lportTag);
 572                     }
 573                 } else {
 574                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 575                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 576                             interfaceName, dpId, vpnName);
 577                 }
 578             }
 579         }
 580     }
 581
 582     // TODO Clean up the exception handling
 583     @SuppressWarnings("checkstyle:IllegalCatch")
 584     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 585                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 586                                                   String vpnName, String interfaceName) {
 587         if (rd == null) {
 588             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 589                   interfaceName, dpnId, vpnName);
 590             return;
 591         } else {
 592             if (rd.equals(vpnName)) {
 593                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 594                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 595                 return;
 596             }
 597         }
 598         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 599                 interfaceName, dpnId, vpnName, rd);
 600
 601         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 602         if (nextHopIp == null) {
 603             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 604                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 605                     rd, vpnName);
 606             return;
 607         }
 608
 609         try {
 610             //Read NextHops
 611             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 612             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 613                     LogicalDatastoreType.OPERATIONAL, path);
 614             if (adjacencies.isPresent()) {
 615                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 616                 if (nextHops != null && !nextHops.isEmpty()) {
 617                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 618                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 619                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 620                     long l3vni = vpnInstanceOpData.getL3vni();
 621                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 622                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 623                     for (Adjacency nextHop : nextHops) {
 624                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 625                             continue;
 626                         }
 627                         String gatewayMac = null;
 628                         long label = 0;
 629                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 630                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 631                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 632                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 633                                     .get();
 634                         } else {
 635                             label = nextHop.getLabel();
 636                         }
 637                         try {
 638                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 639                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 640                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 641                                     encapType, (int)label, l3vni, 0 /*l2vni*/,
 642                                     gatewayMac);
 643                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 644                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 645                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 646                         } catch (Exception e) {
 647                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 648                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 649                                     interfaceName, dpnId, e);
 650                         }
 651                     }
 652                 }
 653             }
 654         } catch (ReadFailedException e) {
 655             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 656                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 657         }
 658     }
 659
 660     // TODO Clean up the exception handling
 661     @SuppressWarnings("checkstyle:IllegalCatch")
 662     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 663                         String vpnName, String interfaceName, TypedWriteTransaction<Configuration> writeConfigTxn,
 664                         TypedWriteTransaction<Operational> writeOperTx) {
 665         //Read NextHops
 666         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 667         String rd = vpnUtil.getVpnRd(interfaceName);
 668         if (rd == null) {
 669             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 670                 interfaceName, vpnName);
 671             return;
 672         } else {
 673             if (rd.equals(vpnName)) {
 674                 LOG.info(
 675                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 676                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 677                 return;
 678             }
 679         }
 680         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 681                vpnName, rd);
 682         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 683         try {
 684             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 685                     path);
 686         } catch (ReadFailedException e) {
 687             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 688                     interfaceName, vpnName);
 689         }
 690         if (adjacencies.isPresent()) {
 691             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 692
 693             if (nextHops != null && !nextHops.isEmpty()) {
 694                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 695                         nextHops, interfaceName, vpnName, rd);
 696                 for (Adjacency nextHop : nextHops) {
 697                     try {
 698                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 699                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 700                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 701                                     interfaceName, vpnName);
 702                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 703                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 704                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 705                                     interfaceName, vpnName);
 706                         } else if (nextHop.getNextHopIpList() != null) {
 707                             // Perform similar operation as interface delete event for extraroutes.
 708                             String allocatedRd = nextHop.getVrfId();
 709                             for (String nh : nextHop.getNextHopIpList()) {
 710                                 deleteExtraRouteFromCurrentAndImportingVpns(
 711                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
 712                                     writeOperTx);
 713                             }
 714                         }
 715                     } catch (Exception e) {
 716                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 717                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 718                     }
 719                 }
 720             }
 721         }
 722     }
 723
 724     @SuppressWarnings("checkstyle:IllegalCatch")
 725     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 726                                                   String primaryRd, String interfaceName, final long vpnId,
 727                                                   TypedWriteTransaction<Configuration> writeConfigTxn,
 728                                                   TypedWriteTransaction<Operational> writeOperTxn,
 729                                                   TypedReadWriteTransaction<Configuration> writeInvTxn,
 730                                                   Interface interfaceState, Set<String> prefixListForRefreshFib)
 731             throws ExecutionException, InterruptedException {
 732         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 733         // Read NextHops
 734         Optional<VpnInterface> vpnInteface = Optional.absent();
 735         try {
 736             vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 737             LogicalDatastoreType.CONFIGURATION, identifier);
 738         } catch (ReadFailedException e) {
 739             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 740                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 741         }
 742         Uuid intfnetworkUuid = null;
 743         NetworkType networkType = null;
 744         Long segmentationId = Long.valueOf(-1);
 745         Adjacencies adjacencies = null;
 746         if (vpnInteface.isPresent()) {
 747             intfnetworkUuid = vpnInteface.get().getNetworkId();
 748             networkType = vpnInteface.get().getNetworkType();
 749             segmentationId = vpnInteface.get().getSegmentationId();
 750             adjacencies = vpnInteface.get().augmentation(Adjacencies.class);
 751             if (adjacencies == null) {
 752                 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 753                         null/*gwMac*/, writeOperTxn);
 754                 return;
 755             }
 756         }
 757         // Get the rd of the vpn instance
 758         String nextHopIp = null;
 759         try {
 760             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 761         } catch (Exception e) {
 762             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 763                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 764         }
 765         List<String> nhList = new ArrayList<>();
 766         if (nextHopIp != null) {
 767             nhList.add(nextHopIp);
 768             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 769                     interfaceName, dpnId, vpnName, nhList);
 770         }
 771         Optional<String> gwMac = Optional.absent();
 772         String vpnInterfaceSubnetGwMacAddress = null;
 773         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 774         Long l3vni = vpnInstanceOpData.getL3vni();
 775         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 776         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 777         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 778         List<Adjacency> nextHops = (adjacencies != null) ? adjacencies.getAdjacency() : emptyList();
 779         List<Adjacency> value = new ArrayList<>();
 780         for (Adjacency nextHop : nextHops) {
 781             String rd = primaryRd;
 782             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 783             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 784                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 785                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 786                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 787                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 788                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 789                 continue;
 790             }
 791             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 792                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 793                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 794                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 795                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 796                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 797
 798                 Prefixes prefixes = (intfnetworkUuid != null)
 799                     ? VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, intfnetworkUuid ,networkType,
 800                             segmentationId, prefixCue) :
 801                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, prefixCue);
 802                 writeOperTxn.merge(VpnUtil.getPrefixToInterfaceIdentifier(
 803                         vpnUtil.getVpnId(vpnName), prefix), prefixes, true);
 804                 final Uuid subnetId = nextHop.getSubnetId();
 805
 806                 String gatewayIp = nextHop.getSubnetGatewayIp();
 807                 if (gatewayIp == null) {
 808                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 809                     if (gatewayIpOptional.isPresent()) {
 810                         gatewayIp = gatewayIpOptional.get();
 811                     }
 812                 }
 813
 814                 if (gatewayIp != null) {
 815                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 816                     if (gwMac.isPresent()) {
 817                         // A valid mac-address is available for this subnet-gateway-ip
 818                         // Use this for programming ARP_RESPONDER table here.  And save this
 819                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 820                         // to populate L3_GW_MAC_TABLE there.
 821                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 822                                 gatewayIp, gwMac.get());
 823                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 824                     } else {
 825                         // A valid mac-address is not available for this subnet-gateway-ip
 826                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 827                         // Save this connected-mac-address as gateway-mac-address for the
 828                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 829                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 830                         if (gwMac.isPresent()) {
 831                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 832                                     NwConstants.ADD_FLOW, gwMac.get());
 833                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 834                                     gatewayIp, gwMac.get());
 835                         } else {
 836                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 837                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 838                                 + "gwIp {}",
 839                                 subnetId, interfaceName, vpnName, gatewayIp);
 840                         }
 841                     }
 842                 } else {
 843                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 844                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 845                         subnetId, interfaceName, vpnName);
 846                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 847                 }
 848                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 849                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 850             } else {
 851                 //Extra route adjacency
 852                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 853                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 854                 synchronized (vpnPrefixKey.intern()) {
 855                     java.util.Optional<String> rdToAllocate = vpnUtil
 856                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 857                                     nextHop.getNextHopIpList().get(0), dpnId);
 858                     if (rdToAllocate.isPresent()) {
 859                         rd = rdToAllocate.get();
 860                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 861                             rd, prefix);
 862                     } else {
 863                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 864                         continue;
 865                     }
 866                 }
 867                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 868                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 869                         interfaceName, dpnId);
 870             }
 871             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 872             // can be different from the gateway-mac-address within the VRFEntry as the
 873             // gateway-mac-address is a superset.
 874             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 875                     : RouteOrigin.STATIC;
 876             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 877                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 878                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 879             Adjacency operationalAdjacency = null;
 880             try {
 881                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 882             } catch (NullPointerException e) {
 883                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 884                     input, e.getMessage());
 885                 return;
 886             }
 887             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 888                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 889                     vpnName, l3vni, origin, interfaceName, operationalAdjacency, encapType, prefixListForRefreshFib,
 890                     writeConfigTxn);
 891             }
 892             value.add(operationalAdjacency);
 893         }
 894
 895         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 896         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 897                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 898
 899         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 900                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 901                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 902
 903         for (Adjacency nextHop : aug.getAdjacency()) {
 904             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 905             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 906                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 907                         : RouteOrigin.STATIC;
 908                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 909                 registeredPopulator.populateFib(input, writeConfigTxn);
 910             }
 911         }
 912     }
 913
 914     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 915                                               long lportTag, String gwMac,
 916                                               TypedWriteTransaction<Operational> writeOperTxn) {
 917         VpnInterfaceOpDataEntry opInterface =
 918               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac);
 919         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 920             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 921         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
 922         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 923                 interfaceName, dpnId, vpnName);
 924     }
 925
 926     // TODO Clean up the exception handling
 927     @SuppressWarnings("checkstyle:IllegalCatch")
 928     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 929                                            StateTunnelList stateTunnelList,
 930                                            TypedWriteTransaction<Configuration> writeConfigTxn,
 931                                            TypedWriteTransaction<Operational> writeOperTxn) {
 932
 933         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
 934         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 935         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 936         List<Adjacency> adjList =
 937             adjacencies != null && adjacencies.getAdjacency() != null ? adjacencies.getAdjacency() : emptyList();
 938         if (adjList.isEmpty()) {
 939             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 940                     vpnInterface, srcDpnId);
 941             return;
 942         }
 943         String prefix = null;
 944         long label = 0;
 945         List<Adjacency> value = new ArrayList<>();
 946         boolean isNextHopAddReqd = false;
 947         String vpnName = vpnInterface.getVpnInstanceName();
 948         long vpnId = vpnUtil.getVpnId(vpnName);
 949         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 950         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 951                 vpnInterface.getName(), vpnInterface.getDpnId(),
 952                 vpnInterface.getVpnInstanceName(), adjList);
 953         for (Adjacency adj : adjList) {
 954             String rd = adj.getVrfId();
 955             rd = rd != null ? rd : vpnName;
 956             prefix = adj.getIpAddress();
 957             label = adj.getLabel();
 958             List<String> nhList = Collections.singletonList(srcTepIp);
 959             List<String> nextHopList = adj.getNextHopIpList();
 960             // If TEP is added , update the nexthop of primary adjacency.
 961             // Secondary adj nexthop is already pointing to primary adj IP address.
 962             if (nextHopList == null || nextHopList.isEmpty()) {
 963                 isNextHopAddReqd = true;
 964             }
 965
 966             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 967                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 968             } else {
 969                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 970                 if (!vrfEntryOptional.isPresent()) {
 971                     continue;
 972                 }
 973                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 974                 if (!nhList.contains(srcTepIp)) {
 975                     nhList.add(srcTepIp);
 976                     isNextHopAddReqd = true;
 977                 }
 978                 value.add(adj);
 979             }
 980
 981             if (isNextHopAddReqd) {
 982                 updateLabelMapper(label, nhList);
 983                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 984                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 985                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 986                 // Update the VRF entry with nextHop
 987                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 988                         label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
 989
 990                 //Get the list of VPN's importing this route(prefix) .
 991                 // Then update the VRF entry with nhList
 992                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
 993                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 994                     String vpnRd = vpn.getVrfId();
 995                     if (vpnRd != null) {
 996                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
 997                             srcTepIp, label, true, TransactionAdapter.toWriteTransaction(writeConfigTxn));
 998                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 999                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
1000                             vpnInterface.getName(), srcDpnId, vpnName,
1001                             vpn.getVpnInstanceName(), vpnRd);
1002                     }
1003                 }
1004                 // Advertise the prefix to BGP only for external vpn
1005                 // since there is a nexthop change.
1006                 try {
1007                     if (!rd.equalsIgnoreCase(vpnName)) {
1008                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
1009                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
1010                                 null /*gatewayMacAddress*/);
1011                     }
1012                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
1013                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
1014                             srcDpnId, vpnName);
1015                 } catch (Exception ex) {
1016                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
1017                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1018                             vpnInterface.getName(), srcDpnId, vpnName, ex);
1019                 }
1020             }
1021         }
1022         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1023         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1024                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1025                 .addAugmentation(AdjacenciesOp.class, aug).build();
1026         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1027                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1028         writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1029         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
1030                 vpnInterface.getName(), srcDpnId, vpnName);
1031
1032     }
1033
1034     // TODO Clean up the exception handling
1035     @SuppressWarnings("checkstyle:IllegalCatch")
1036     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1037                                               StateTunnelList stateTunnelList,
1038                                               TypedWriteTransaction<Configuration> writeConfigTxn,
1039                                               TypedWriteTransaction<Operational> writeOperTxn) {
1040
1041         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1042         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1043         String prefix = null;
1044         long label = 0;
1045         boolean isNextHopRemoveReqd = false;
1046         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
1047         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
1048         String vpnName = vpnInterface.getVpnInstanceName();
1049         long vpnId = vpnUtil.getVpnId(vpnName);
1050         String primaryRd = vpnUtil.getVpnRd(vpnName);
1051         if (adjList != null) {
1052             List<Adjacency> value = new ArrayList<>();
1053             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1054                     vpnInterface.getName(), vpnInterface.getDpnId(),
1055                     vpnInterface.getVpnInstanceName(), adjList);
1056             for (Adjacency adj : adjList) {
1057                 List<String> nhList = new ArrayList<>();
1058                 String rd = adj.getVrfId();
1059                 rd = rd != null ? rd : vpnName;
1060                 prefix = adj.getIpAddress();
1061                 List<String> nextHopList = adj.getNextHopIpList();
1062                 label = adj.getLabel();
1063                 if (nextHopList != null && !nextHopList.isEmpty()) {
1064                     isNextHopRemoveReqd = true;
1065                 }
1066                 // If TEP is deleted , remove the nexthop from primary adjacency.
1067                 // Secondary adj nexthop will continue to point to primary adj IP address.
1068                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1069                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1070                 } else {
1071                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1072                     if (!vrfEntryOptional.isPresent()) {
1073                         continue;
1074                     }
1075                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1076                     if (nhList.contains(srcTepIp)) {
1077                         nhList.remove(srcTepIp);
1078                         isNextHopRemoveReqd = true;
1079                     }
1080                     value.add(adj);
1081                 }
1082
1083                 if (isNextHopRemoveReqd) {
1084                     updateLabelMapper(label, nhList);
1085                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1086                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1087                             prefix, nhList, vpnName,
1088                             vpnId, rd, vpnInterface.getName());
1089                     // Update the VRF entry with removed nextHop
1090                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1091                             label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1092
1093                     //Get the list of VPN's importing this route(prefix) .
1094                     // Then update the VRF entry with nhList
1095                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1096                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1097                         String vpnRd = vpn.getVrfId();
1098                         if (vpnRd != null) {
1099                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1100                                 srcTepIp, label, false, TransactionAdapter.toWriteTransaction(writeConfigTxn));
1101                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1102                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1103                                     nhList, label, vpnInterface.getName(), srcDpnId,
1104                                     vpnName,
1105                                     vpn.getVpnInstanceName(), vpnRd);
1106                         }
1107                     }
1108
1109                     // Withdraw prefix from BGP only for external vpn.
1110                     try {
1111                         if (!rd.equalsIgnoreCase(vpnName)) {
1112                             bgpManager.withdrawPrefix(rd, prefix);
1113                         }
1114                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1115                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1116                                 vpnInterface.getName(), srcDpnId,
1117                                 vpnName);
1118                     } catch (Exception ex) {
1119                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1120                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1121                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1122                     }
1123                 }
1124             }
1125             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1126             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1127                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1128                     .addAugmentation(AdjacenciesOp.class, aug).build();
1129             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1130                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1131             writeOperTxn.put(interfaceId, opInterface, CREATE_MISSING_PARENTS);
1132             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1133                          vpnInterface.getName(), srcDpnId, vpnName);
1134         }
1135     }
1136
1137     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1138         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1139
1140         String vpnRd = vpnUtil.getVpnRd(vpnName);
1141         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1142         if (vpnInstanceOpDataEntry == null) {
1143             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1144                     + " to check for vpns exporting the routes", vpnName);
1145             return vpnsToExportRoute;
1146         }
1147
1148         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1149             if (input.getVpnInstanceName() == null) {
1150                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1151                         input.getVrfId());
1152                 return false;
1153             }
1154             return !input.getVpnInstanceName().equals(vpnName);
1155         };
1156
1157         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1158             Iterable<String> commonRTs =
1159                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1160                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1161             return Iterators.size(commonRTs.iterator()) > 0;
1162         };
1163
1164         vpnsToExportRoute =
1165                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1166                         Collectors.toList());
1167         return vpnsToExportRoute;
1168     }
1169
1170     // TODO Clean up the exception handling
1171     @SuppressWarnings("checkstyle:IllegalCatch")
1172     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1173         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1174         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1175             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1176             if (vrfEntries != null) {
1177                 ListenableFutures.addErrorLogging(
1178                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1179                         for (VrfEntry vrfEntry : vrfEntries) {
1180                             try {
1181                                 if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1182                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1183                                     LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1184                                             + " is not a controller managed non intervpn link route. Ignoring.",
1185                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1186                                     continue;
1187                                 }
1188                                 String prefix = vrfEntry.getDestPrefix();
1189                                 String gwMac = vrfEntry.getGatewayMacAddress();
1190                                 vrfEntry.nonnullRoutePaths().forEach(routePath -> {
1191                                     String nh = routePath.getNexthopAddress();
1192                                     int label = routePath.getLabel().intValue();
1193                                     if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1194                                         vrfEntry.getOrigin()))) {
1195                                         LOG.info(
1196                                             "handleVpnsExportingRoutesImporting: Importing fib entry rd {}"
1197                                                 + " prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1198                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1199                                         fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1200                                             Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1201                                             0 /*l3vni*/, gwMac, vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1202                                             confTx);
1203                                     } else {
1204                                         LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry"
1205                                                 + " rd {} prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1206                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1207                                         SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1208                                         importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1209                                             confTx);
1210                                     }
1211                                 });
1212                             } catch (RuntimeException e) {
1213                                 LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1214                                         + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1215                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1216                             }
1217                         }
1218                     }), LOG, "Error handing VPN exporting routes");
1219             } else {
1220                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1221                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1222             }
1223         }
1224     }
1225
1226     @Override
1227     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1228         LOG.trace("Received VpnInterface remove event: vpnInterface={}", vpnInterface);
1229         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1230         final String interfaceName = key.getName();
1231         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.nonnullVpnInstanceNames()) {
1232             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1233             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1234         }
1235     }
1236
1237     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1238                                 final VpnInterface vpnInterface, final String vpnName,
1239                                 final String interfaceName) {
1240         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1241             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1242                 ListenableFuture<Void> future =
1243                     txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION, confTx -> {
1244                         deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1245                         LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1246                     });
1247                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1248                         vpnInterface.getName(), vpnName);
1249                 return Collections.singletonList(future);
1250             }, DJC_MAX_RETRIES);
1251         } else {
1252             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1253             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1254         }
1255     }
1256
1257     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1258     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1259                                            final VpnInterface vpnInterface, final String vpnName,
1260                                            final String interfaceName, final Interface interfaceState) {
1261         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1262                 vpnName, vpnInterface.getDpnId());
1263         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1264         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1265             () -> {
1266                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1267                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(CONFIGURATION,
1268                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL,
1269                         writeOperTxn -> futures.add(
1270                             txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION, writeInvTxn -> {
1271                                 LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1272                                         interfaceName, vpnName);
1273                                 BigInteger dpId;
1274                                 int ifIndex;
1275                                 String gwMacAddress;
1276                                 InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1277                                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1278                                 Optional<VpnInterfaceOpDataEntry> optVpnInterface;
1279                                 try {
1280                                     optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1281                                             LogicalDatastoreType.OPERATIONAL, interfaceId);
1282                                 } catch (ReadFailedException e) {
1283                                     LOG.error("remove: Failed to read data store for interface {} vpn {}",
1284                                             interfaceName, vpnName);
1285                                     return;
1286                                 }
1287                                 if (interfaceState != null) {
1288                                     try {
1289                                         dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1290                                     } catch (NumberFormatException | IllegalStateException e) {
1291                                         LOG.error("remove: Unable to retrieve dpnId from interface operational"
1292                                                         + " data store for interface {} on dpn {} for vpn {} Fetching"
1293                                                         + " from vpn interface op data store. ", interfaceName,
1294                                                 vpnInterface.getDpnId(), vpnName, e);
1295                                         dpId = BigInteger.ZERO;
1296                                     }
1297                                     ifIndex = interfaceState.getIfIndex();
1298                                     gwMacAddress = interfaceState.getPhysAddress().getValue();
1299                                 } else {
1300                                     LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1301                                             + " from vpn interface op.", interfaceName);
1302                                     if (optVpnInterface.isPresent()) {
1303                                         VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1304                                         dpId = vpnOpInterface.getDpnId();
1305                                         ifIndex = vpnOpInterface.getLportTag().intValue();
1306                                         gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1307                                     } else {
1308                                         LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1309                                                 + " as interfaceState and vpn interface op is not"
1310                                                 + " available", interfaceName, vpnName);
1311                                         return;
1312                                     }
1313                                 }
1314                                 processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1315                                         optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1316                                         writeConfigTxn, writeOperTxn, writeInvTxn);
1317                                 LOG.info(
1318                                         "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1319                                                 + "successfully",
1320                                         interfaceName, vpnInterface.getDpnId(), vpnName);
1321                             })))));
1322                 futures.add(configFuture);
1323                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1324                 return futures;
1325             }, DJC_MAX_RETRIES);
1326     }
1327
1328     protected void processVpnInterfaceDown(BigInteger dpId,
1329                                            String interfaceName,
1330                                            int lportTag,
1331                                            String gwMac,
1332                                            VpnInterfaceOpDataEntry vpnOpInterface,
1333                                            boolean isInterfaceStateDown,
1334                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1335                                            TypedWriteTransaction<Operational> writeOperTxn,
1336                                            TypedReadWriteTransaction<Configuration> writeInvTxn)
1337             throws ExecutionException, InterruptedException {
1338         if (vpnOpInterface == null) {
1339             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1340                     + " as it is not available in operational data store", interfaceName, dpId);
1341             return;
1342         }
1343         final String vpnName = vpnOpInterface.getVpnInstanceName();
1344         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1345                                                     interfaceName, vpnName);
1346         if (!isInterfaceStateDown) {
1347             final long vpnId = vpnUtil.getVpnId(vpnName);
1348             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName,  null);
1349             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1350             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1351                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1352             if (interfaceManager.isExternalInterface(interfaceName)) {
1353                 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1354                     NwConstants.DEL_FLOW);
1355             }
1356             if (!isBgpVpnInternetVpn) {
1357                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1358             }
1359             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1360                     + " successful", interfaceName, dpId, vpnName);
1361         } else {
1362             // Interface is retained in the DPN, but its Link Down.
1363             // Only withdraw the prefixes for this interface from BGP
1364             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1365         }
1366     }
1367
1368     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1369                                           final String vpnName, final long vpnId, String gwMac,
1370                                           TypedWriteTransaction<Configuration> writeConfigTxn,
1371                                           TypedWriteTransaction<Operational> writeOperTxn,
1372                                           TypedReadWriteTransaction<Configuration> writeInvTxn)
1373             throws ExecutionException, InterruptedException {
1374         //Read NextHops
1375         try {
1376             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1377                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1378             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1379             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1380                     LogicalDatastoreType.OPERATIONAL, path);
1381             boolean isNonPrimaryAdjIp = Boolean.FALSE;
1382             String primaryRd = vpnUtil.getVpnRd(vpnName);
1383             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1384                     interfaceName, dpnId, vpnName, primaryRd);
1385             if (adjacencies.isPresent() && adjacencies.get().getAdjacency() != null
1386                     && !adjacencies.get().getAdjacency().isEmpty()) {
1387                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1388                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1389                         interfaceName, dpnId, vpnName, nextHops);
1390                 for (Adjacency nextHop : nextHops) {
1391                     if (nextHop.isPhysNetworkFunc()) {
1392                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1393                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1394                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1395                                 null/*writeCfgTxn*/);
1396                     } else {
1397                         String rd = nextHop.getVrfId();
1398                         List<String> nhList;
1399                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1400                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1401                             isNonPrimaryAdjIp = Boolean.TRUE;
1402                         } else {
1403                             // This is a primary adjacency
1404                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1405                                     : emptyList();
1406                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1407                                     interfaceName, writeInvTxn);
1408                         }
1409                         if (!nhList.isEmpty()) {
1410                             if (Objects.equals(primaryRd, vpnName)) {
1411                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1412                                 //remove from FIB directly
1413                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1414                                         interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1415                             } else {
1416                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1417                                         interfaceName, writeConfigTxn, writeOperTxn);
1418                             }
1419                         } else {
1420                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1421                                             + " interface {}", nextHop.getIpAddress(), rd,
1422                                     nextHop.getAdjacencyType().toString(), interfaceName);
1423                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1424                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1425                         }
1426                     }
1427                     String ip = nextHop.getIpAddress().split("/")[0];
1428                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1429                     if (vpnVipToPort != null && vpnVipToPort.getPortName().equals(interfaceName)) {
1430                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1431                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1432                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1433                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1434                     }
1435                     // Remove the MIP-IP from VpnPortIpToPort.
1436                     if (isNonPrimaryAdjIp) {
1437                         VpnPortipToPort persistedIp = vpnUtil.getVpnPortipToPort(vpnName, ip);
1438                         if (persistedIp != null && persistedIp.isLearntIp()
1439                                 && persistedIp.getPortName().equals(interfaceName)) {
1440                             VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1441                             LOG.info(
1442                                     "removeAdjacenciesFromVpn: Learnt-IP: {} interface {} of vpn {} removed "
1443                                             + "from VpnPortipToPort",
1444                                     persistedIp.getPortFixedip(), persistedIp.getPortName(), vpnName);
1445                         }
1446                     }
1447                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1448                     if (vpnPortipToPort != null) {
1449                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1450                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1451                                  + "Interface {} ip {} on dpn {} for vpn {}",
1452                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1453                     }
1454                 }
1455             } else {
1456                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1457                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1458                         + " Removing it.", interfaceName, vpnName, dpnId);
1459                 writeOperTxn.delete(identifier);
1460             }
1461         } catch (ReadFailedException e) {
1462             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1463                     interfaceName, dpnId, vpnName);
1464         }
1465     }
1466
1467     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1468                                                             String interfaceName, BigInteger dpnId,
1469                                                             TypedWriteTransaction<Configuration> writeConfigTxn,
1470                                                             TypedWriteTransaction<Operational> writeOperTx) {
1471         return (nh) -> {
1472             String primaryRd = vpnUtil.getVpnRd(vpnName);
1473             String prefix = nextHop.getIpAddress();
1474             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1475             LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1476                     nextHop, vpnName, interfaceName, dpnId);
1477             synchronized (vpnNamePrefixKey.intern()) {
1478                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1479                         prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1480                     //If extra-route is present behind at least one VM, then do not remove or update
1481                     //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1482                     //prefixtointerface DS
1483                     return;
1484                 }
1485                 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1486                         writeConfigTxn);
1487             }
1488             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1489                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1490                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1491         };
1492     }
1493
1494     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1495                                            BigInteger dpnId, String rd, String interfaceName,
1496                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1497                                            TypedWriteTransaction<Operational> writeOperTx) {
1498         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1499                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1500         nhList.forEach((nh) -> {
1501             //IRT: remove routes from other vpns importing it
1502             vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1503                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1504             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1505                 String vpnRd = vpn.getVrfId();
1506                 if (vpnRd != null) {
1507                     fibManager.removeOrUpdateFibEntry(vpnRd,
1508                             nextHop.getIpAddress(), nh, writeConfigTxn);
1509                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1510                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1511                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1512                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1513                 }
1514             }
1515         });
1516     }
1517
1518     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1519                                                  int lportTag, String gwMac, String interfaceName,
1520                                                  TypedReadWriteTransaction<Configuration> writeInvTxn)
1521             throws ExecutionException, InterruptedException {
1522         final Uuid subnetId = nextHop.getSubnetId();
1523         if (nextHop.getSubnetGatewayMacAddress() == null) {
1524             // A valid mac-address was not available for this subnet-gateway-ip
1525             // So a connected-mac-address was used for this subnet and we need
1526             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1527             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1528         }
1529         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1530                 subnetId);
1531     }
1532
1533     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1534                                                   String interfaceName) {
1535         // This is either an extra-route (or) a learned IP via subnet-route
1536         List<String> nhList = null;
1537         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1538         if (nextHopIp == null || nextHopIp.isEmpty()) {
1539             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1540                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1541                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1542                     vpnName);
1543             nhList = emptyList();
1544         } else {
1545             nhList = Collections.singletonList(nextHopIp);
1546         }
1547         return nhList;
1548     }
1549
1550     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1551         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1552         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1553         // else use connected interface
1554         if (gwPort != null && gwPort.isSubnetIp()) {
1555             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1556                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1557             return Optional.of(gwPort.getMacAddress());
1558         }
1559         return Optional.absent();
1560     }
1561
1562     @Override
1563     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1564         final VpnInterface update) {
1565         LOG.trace("Received VpnInterface update event: original={}, update={}", original, update);
1566         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}", update.getName(),
1567                 update.getDpnId(), original.getVpnInstanceNames(), update.getVpnInstanceNames());
1568         final String vpnInterfaceName = update.getName();
1569         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1570         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1571         //handles switching between <internal VPN - external VPN>
1572         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1573             List<ListenableFuture<Void>> futures = new ArrayList<>();
1574             if (handleVpnInstanceUpdateForVpnInterface(identifier, original, update, futures)) {
1575                 LOG.info("update: handled Instance update for VPNInterface {} on dpn {} from oldVpn(s) {} "
1576                                 + "to newVpn(s) {}",
1577                         original.getName(), dpnId,
1578                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1579                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1580                 return emptyList();
1581             }
1582             updateVpnInstanceAdjChange(original, update, vpnInterfaceName, futures);
1583             return futures;
1584         });
1585     }
1586
1587     private boolean handleVpnInstanceUpdateForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1588                                                            VpnInterface original, VpnInterface update,
1589                                                            List<ListenableFuture<Void>> futures) {
1590         boolean isVpnInstanceUpdate = false;
1591         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class);
1592         final String interfaceName = key.getName();
1593         List<String> oldVpnList = VpnUtil.getVpnListForVpnInterface(original);
1594         List<String> oldVpnListCopy = new ArrayList<>();
1595         oldVpnListCopy.addAll(oldVpnList);
1596         List<String> newVpnList = VpnUtil.getVpnListForVpnInterface(update);
1597         List<String> newVpnListCopy = new ArrayList<>();
1598         newVpnListCopy.addAll(newVpnList);
1599
1600         oldVpnList.removeAll(newVpnList);
1601         newVpnList.removeAll(oldVpnListCopy);
1602         //This block will execute only on if there is a change in the VPN Instance.
1603         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1604             /*
1605              * Internet BGP-VPN Instance update with single router:
1606              * ====================================================
1607              * In this case single VPN Interface will be part of maximum 2 VPN Instance only.
1608              *     1st VPN Instance : router VPN or external BGP-VPN.
1609              *     2nd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1610              *
1611              * VPN Instance UPDATE:
1612              * oldVpnList = 0 and newVpnList = 1 (Internet BGP-VPN)
1613              * oldVpnList = 1 and newVpnList = 0 (Internet BGP-VPN)
1614              *
1615              * External BGP-VPN Instance update with single router:
1616              * ====================================================
1617              * In this case single VPN interface will be part of maximum 1 VPN Instance only.
1618              *
1619              * Updated VPN Instance will be always either internal router VPN to
1620              * external BGP-VPN or external BGP-VPN to internal router VPN swap.
1621              *
1622              * VPN Instance UPDATE:
1623              * oldVpnList = 1 and newVpnList = 1 (router VPN to Ext-BGPVPN)
1624              * oldVpnList = 1 and newVpnList = 1 (Ext-BGPVPN to router VPN)
1625              *
1626              * Dual Router VPN Instance Update:
1627              * ================================
1628              * In this case single VPN interface will be part of maximum 3 VPN Instance only.
1629              *
1630              * 1st VPN Instance : router VPN or external BGP-VPN-1.
1631              * 2nd VPN Instance : router VPN or external BGP-VPN-2.
1632              * 3rd VPN Instance : Internet BGP-VPN(router-gw update/delete) for public network access.
1633              *
1634              * Dual Router --> Associated with common external BGP-VPN Instance.
1635              * 1st router and 2nd router are getting associated with single External BGP-VPN
1636              * 1) add 1st router to external bgpvpn --> oldVpnList=1, newVpnList=1;
1637              * 2) add 2nd router to the same external bgpvpn --> oldVpnList=1, newVpnList=0
1638              * In this case, we need to call removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1639              *
1640              *
1641              */
1642             isVpnInstanceUpdate = true;
1643             if (VpnUtil.isDualRouterVpnUpdate(oldVpnListCopy, newVpnListCopy)) {
1644                 if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1645                         && (oldVpnList.size() == 1 && newVpnList.size() == 0)) {
1646                     //Identify the external BGP-VPN Instance and pass that value as newVpnList
1647                     List<String> externalBgpVpnList = new ArrayList<>();
1648                     for (String newVpnName : newVpnListCopy) {
1649                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1650                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1651                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1652                                 .BgpvpnType.BGPVPNExternal) {
1653                             externalBgpVpnList.add(newVpnName);
1654                             break;
1655                         }
1656                     }
1657                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1658                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList,
1659                             externalBgpVpnList, oldVpnListCopy, futures);
1660
1661                 } else if ((oldVpnListCopy.size() == 2 || oldVpnListCopy.size() == 3)
1662                         && (oldVpnList.size() == 0 && newVpnList.size() == 1)) {
1663                     //Identify the router VPN Instance and pass that value as oldVpnList
1664                     List<String> routerVpnList = new ArrayList<>();
1665                     for (String newVpnName : newVpnListCopy) {
1666                         String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1667                         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(primaryRd);
1668                         if (vpnInstanceOpDataEntry.getBgpvpnType() == VpnInstanceOpDataEntry
1669                                 .BgpvpnType.VPN) {
1670                             routerVpnList.add(newVpnName);
1671                             break;
1672                         }
1673                     }
1674                     //This call will execute removeVpnInterfaceCall() followed by addVpnInterfaceCall()
1675                     updateVpnInstanceChange(identifier, interfaceName, original, update, routerVpnList,
1676                             newVpnList, oldVpnListCopy, futures);
1677
1678                 } else {
1679                     //Handle remaining use cases.
1680                     updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1681                             oldVpnListCopy, futures);
1682                 }
1683             } else {
1684                 updateVpnInstanceChange(identifier, interfaceName, original, update, oldVpnList, newVpnList,
1685                         oldVpnListCopy, futures);
1686             }
1687         }
1688         return isVpnInstanceUpdate;
1689     }
1690
1691     private void updateVpnInstanceChange(InstanceIdentifier<VpnInterface> identifier, String interfaceName,
1692                                          VpnInterface original, VpnInterface update, List<String> oldVpnList,
1693                                          List<String> newVpnList, List<String> oldVpnListCopy,
1694                                          List<ListenableFuture<Void>> futures) {
1695         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1696         final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1697                 ? origAdjs.getAdjacency() : new ArrayList<>();
1698         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1699         final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1700                 ? updateAdjs.getAdjacency() : new ArrayList<>();
1701
1702         boolean isOldVpnRemoveCallExecuted = false;
1703         for (String oldVpnName : oldVpnList) {
1704             LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1705                     + "remove from vpnName {} ", interfaceName, oldVpnName);
1706             removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1707             LOG.info("updateVpnInstanceChange: Processed Remove for update on VPNInterface"
1708                             + " {} upon VPN update from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName,
1709                     newVpnList);
1710             isOldVpnRemoveCallExecuted = true;
1711         }
1712         //Wait for previous interface bindings to be removed
1713         if (isOldVpnRemoveCallExecuted && !newVpnList.isEmpty()) {
1714             try {
1715                 Thread.sleep(2000);
1716             } catch (InterruptedException e) {
1717                 //Ignore
1718             }
1719         }
1720         for (String newVpnName : newVpnList) {
1721             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1722             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1723                 LOG.info("updateVpnInstanceChange: VPN Interface update event - intfName {} "
1724                         + "onto vpnName {} ", interfaceName, newVpnName);
1725                 addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1726                 LOG.info("updateVpnInstanceChange: Processed Add for update on VPNInterface {}"
1727                                 + "from oldVpn(s) {} to newVpn {} ",
1728                         interfaceName, oldVpnListCopy, newVpnName);
1729                 /* This block will execute only if V6 subnet is associated with internet BGP-VPN.
1730                  * Use Case:
1731                  *     In Dual stack network, first V4 subnet only attached to router and router is associated
1732                  *     with internet BGP-VPN(router-gw). At this point VPN interface is having only router vpn instance.
1733                  *     Later V6 subnet is added to router, at this point existing VPN interface will get updated
1734                  *     with Internet BGP-VPN instance(Note: Internet BGP-VPN Instance update in vpn interface
1735                  *     is applicable for only on V6 subnet is added to router). newVpnList = Contains only Internet
1736                  *     BGP-VPN Instance. So we required V6 primary adjacency info needs to be populated onto
1737                  *     router VPN as well as Internet BGP-VPN.
1738                  *
1739                  *     addVpnInterfaceCall() --> It will create V6 Adj onto Internet BGP-VPN only.
1740                  *     updateVpnInstanceAdjChange() --> This method call is needed for second primary V6 Adj
1741                  *                                       update in existing router VPN instance.
1742                  */
1743                 if (vpnUtil.isBgpVpnInternet(newVpnName)) {
1744                     LOG.info("updateVpnInstanceChange: VPN Interface {} with new Adjacency {} in existing "
1745                             + "VPN instance {}", interfaceName, newAdjs, original.getVpnInstanceNames());
1746                     updateVpnInstanceAdjChange(original, update, interfaceName, futures);
1747                 }
1748             }
1749         }
1750     }
1751
1752     private List<ListenableFuture<Void>> updateVpnInstanceAdjChange(VpnInterface original, VpnInterface update,
1753                                                                     String vpnInterfaceName,
1754                                                                     List<ListenableFuture<Void>> futures) {
1755         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1756         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1757                 != null ? origAdjs.getAdjacency() : new ArrayList<>();
1758         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1759         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1760                 != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1761
1762         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1763         for (VpnInstanceNames vpnInterfaceVpnInstance : update.nonnullVpnInstanceNames()) {
1764             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1765             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1766             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1767             String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1768             if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1769                 // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1770                 //set of prefix used as entry in prefix-to-interface datastore
1771                 // is prerequisite for refresh Fib to avoid race condition leading to missing remote next hop
1772                 // in bucket actions on bgp-vpn delete
1773                 Set<String> prefixListForRefreshFib = new HashSet<>();
1774                 ListenableFuture<Void> configTxFuture = txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
1775                     confTx -> futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(OPERATIONAL,
1776                         operTx -> {
1777                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1778                                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1779                             LOG.info("VPN Interface update event-intfName {} onto vpnName {} running config-driven",
1780                                     update.getName(), newVpnName);
1781                             //handle both addition and removal of adjacencies
1782                             // currently, new adjacency may be an extra route
1783                             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1784                             if (!oldAdjs.equals(newAdjs)) {
1785                                 for (Adjacency adj : copyNewAdjs) {
1786                                     if (copyOldAdjs.contains(adj)) {
1787                                         copyOldAdjs.remove(adj);
1788                                     } else {
1789                                         // add new adjacency
1790                                         if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1791                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1792                                                     dpnId, operTx, confTx, confTx, prefixListForRefreshFib);
1793                                         }
1794                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} "
1795                                             + " added to vpn interface {} on vpn {} dpnId {}",
1796                                             adj.getIpAddress(), adj.getNextHopIpList(), adj.getLabel(),
1797                                             adj.getSubnetId(), update.getName(), newVpnName, dpnId);
1798                                     }
1799                                 }
1800                                 for (Adjacency adj : copyOldAdjs) {
1801                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1802                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1803                                             && !adj.isPhysNetworkFunc()) {
1804                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId, operTx,
1805                                                 confTx);
1806                                             //remove FIB entry
1807                                             String vpnRd = vpnUtil.getVpnRd(newVpnName);
1808                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1809                                                 + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1810                                             //remove BGP entry
1811                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1812                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1813                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1814                                             }
1815                                         } else {
1816                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1817                                                 operTx, confTx);
1818                                         }
1819                                     }
1820                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1821                                         + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1822                                         adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1823                                 }
1824                             }
1825                         })));
1826                 Futures.addCallback(configTxFuture, new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
1827                     MoreExecutors.directExecutor());
1828                 futures.add(configTxFuture);
1829                 for (ListenableFuture<Void> future : futures) {
1830                     ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1831                             update.getName(), update.getVpnInstanceNames());
1832                 }
1833             } else {
1834                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1835                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1836             }
1837         }
1838         return futures;
1839     }
1840
1841     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1842         try {
1843             Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1844             synchronized (label.toString().intern()) {
1845                 InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1846                         .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1847                 Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1848                         LogicalDatastoreType.OPERATIONAL, lriIid);
1849                 if (opResult.isPresent()) {
1850                     LabelRouteInfo labelRouteInfo =
1851                             new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1852                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1853                             labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1854                 }
1855             }
1856             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1857                     nextHopIpList);
1858         } catch (ReadFailedException e) {
1859             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1860                     nextHopIpList);
1861         } catch (TransactionCommitFailedException e) {
1862             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1863                     nextHopIpList);
1864         }
1865     }
1866
1867     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1868         SubnetRoute route, String parentVpnRd, TypedWriteTransaction<Configuration> writeConfigTxn) {
1869
1870         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1871         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1872                 .addAugmentation(SubnetRoute.class, route).build();
1873         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1874         InstanceIdentifierBuilder<VrfTables> idBuilder =
1875             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1876         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1877         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1878         if (writeConfigTxn != null) {
1879             writeConfigTxn.merge(vrfTableId, vrfTableNew, CREATE_MISSING_PARENTS);
1880         } else {
1881             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1882         }
1883         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1884                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1885     }
1886
1887     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1888                                            Adjacency adj, BigInteger dpnId,
1889                                            TypedWriteTransaction<Operational> writeOperTxn,
1890                                            TypedWriteTransaction<Configuration> writeConfigTxn,
1891                                            TypedReadWriteTransaction<Configuration> writeInvTxn,
1892                                            Set<String> prefixListForRefreshFib)
1893             throws ExecutionException, InterruptedException {
1894         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1895         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1896         try {
1897             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1898                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1899             if (optVpnInterface.isPresent()) {
1900                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1901                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1902                 String vpnName = currVpnIntf.getVpnInstanceName();
1903                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1904                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1905                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1906                         LogicalDatastoreType.OPERATIONAL, adjPath);
1907                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1908                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1909                 long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1910                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1911                 List<Adjacency> adjacencies = new ArrayList<>();
1912                 if (optAdjacencies.isPresent() && optAdjacencies.get().getAdjacency() != null) {
1913                     adjacencies.addAll(optAdjacencies.get().getAdjacency());
1914                 }
1915                 long vpnId = vpnUtil.getVpnId(vpnName);
1916                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1917                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1918                 Adjacency operationalAdjacency = null;
1919                 //Handling dual stack neutron port primary adjacency
1920                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1921                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1922                             currVpnIntf.getName(), vpnName);
1923                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1924                             currVpnIntf.getName());
1925                     if (interfaceState != null) {
1926                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1927                             currVpnIntf.getName(), vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState,
1928                             prefixListForRefreshFib);
1929                     }
1930                 }
1931                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1932                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1933                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.LearntIp ? RouteOrigin.DYNAMIC
1934                             : RouteOrigin.STATIC;
1935                     String nh = adj.getNextHopIpList().get(0);
1936                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1937                     synchronized (vpnPrefixKey.intern()) {
1938                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1939                                 vpnId, null, prefix, vpnName, nh, dpnId);
1940                         if (rdToAllocate.isPresent()) {
1941                             input.setRd(rdToAllocate.get());
1942                             operationalAdjacency = populator.createOperationalAdjacency(input);
1943                             int label = operationalAdjacency.getLabel().intValue();
1944                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1945                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
1946                                     currVpnIntf.getName(), operationalAdjacency, encapType,
1947                                     prefixListForRefreshFib, writeConfigTxn);
1948                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1949                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1950                                     vpnName, label, currVpnIntf.getName(), dpnId);
1951                         } else {
1952                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
1953                                     vpnName, prefix);
1954                             return;
1955                         }
1956                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1957                         // Keeping the MPLS check for now.
1958                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1959                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1960                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1961                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
1962                             vpnsToImportRoute.forEach(vpn -> {
1963                                 if (vpn.getVrfId() != null) {
1964                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(), vpnId, prefix,
1965                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
1966                                             .ifPresent(
1967                                                 rds -> vpnManager.addExtraRoute(
1968                                                         vpnUtil.getVpnName(vpn.getVpnId()), adj.getIpAddress(),
1969                                                         nh, rds, currVpnIntf.getVpnInstanceName(), l3vni,
1970                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(), opAdjacency,
1971                                                         encapType, prefixListForRefreshFib, writeConfigTxn));
1972                                 }
1973                             });
1974                         }
1975                     }
1976                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1977                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1978                             currVpnIntf.getName(), vpnName);
1979
1980                     InstanceIdentifier<VpnInterface> vpnIfaceConfigidentifier = VpnUtil
1981                             .getVpnInterfaceIdentifier(currVpnIntf.getName());
1982                     Optional<VpnInterface> vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1983                             LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier);
1984                     Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1985                             Prefixes.PrefixCue.PhysNetFunc);
1986                     if (vpnIntefaceConfig.isPresent()) {
1987                         pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1988                                 vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(),
1989                                 vpnIntefaceConfig.get().getSegmentationId(), Prefixes.PrefixCue.PhysNetFunc);
1990                     }
1991
1992                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1993
1994                     writeOperTxn.merge(
1995                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
1996                                     prefix), pnfPrefix, true);
1997
1998                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1999                             adj.getIpAddress(), emptyList(), null /* EncapType */, 0 /* label */,
2000                             0 /*l3vni*/, null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
2001
2002                     input.setRd(adj.getVrfId());
2003                 }
2004                 if (operationalAdjacency == null) {
2005                     operationalAdjacency = populator.createOperationalAdjacency(input);
2006                 }
2007                 adjacencies.add(operationalAdjacency);
2008                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
2009                 VpnInterfaceOpDataEntry newVpnIntf =
2010                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
2011                                 aug, dpnId, currVpnIntf.getLportTag(),
2012                                 currVpnIntf.getGatewayMacAddress());
2013
2014                 writeOperTxn.merge(identifier, newVpnIntf, CREATE_MISSING_PARENTS);
2015             }
2016         } catch (ReadFailedException e) {
2017             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
2018                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
2019         }
2020     }
2021
2022     @Nullable
2023     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
2024         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
2025         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
2026     }
2027
2028     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
2029                                             BigInteger dpnId, TypedWriteTransaction<Operational> writeOperTxn,
2030                                             TypedWriteTransaction<Configuration> writeConfigTxn) {
2031         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
2032         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
2033         try {
2034             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
2035                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
2036             if (optVpnInterface.isPresent()) {
2037                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
2038                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
2039                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
2040                         LogicalDatastoreType.OPERATIONAL, path);
2041                 if (optAdjacencies.isPresent()) {
2042                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
2043
2044                     if (adjacencies != null && !adjacencies.isEmpty()) {
2045                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
2046                         for (Adjacency adjacency : adjacencies) {
2047                             if (Objects.equals(adjacency.getIpAddress(), adj.getIpAddress())) {
2048                                 String rd = adjacency.getVrfId();
2049                                 if (adj.getNextHopIpList() != null) {
2050                                     for (String nh : adj.getNextHopIpList()) {
2051                                         deleteExtraRouteFromCurrentAndImportingVpns(
2052                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
2053                                                 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
2054                                     }
2055                                 } else if (adj.isPhysNetworkFunc()) {
2056                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
2057                                             adj.getIpAddress(), adj.getSubnetId());
2058                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
2059                                             writeConfigTxn);
2060                                 }
2061                                 break;
2062                             }
2063
2064                         }
2065                     }
2066                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
2067                             dpnId, adj.getVrfId());
2068                 } else {
2069                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
2070                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
2071                 }
2072             }
2073         } catch (ReadFailedException e) {
2074             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
2075                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
2076         }
2077     }
2078
2079     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
2080                                     String rd, String intfName, TypedWriteTransaction<Configuration> writeConfigTxn,
2081                                     TypedWriteTransaction<Operational> writeOperTx) {
2082         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
2083         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
2084         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
2085             String vpnRd = vpn.getVrfId();
2086             if (vpnRd != null) {
2087                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
2088                         writeOperTx);
2089             }
2090         }
2091     }
2092
2093     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
2094         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2095             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
2096             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
2097     }
2098
2099     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
2100         return InstanceIdentifier.builder(NeutronRouterDpns.class)
2101             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
2102     }
2103
2104     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2105                                                 TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2106         if (vpnInterface == null) {
2107             return;
2108         }
2109         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
2110         if (adjs == null) {
2111             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2112                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2113             return;
2114         }
2115         for (Adjacency adj : adjs) {
2116             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2117                 String primaryInterfaceIp = adj.getIpAddress();
2118                 String macAddress = adj.getMacAddress();
2119                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2120
2121                 long label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
2122                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2123
2124                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2125                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2126                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
2127                         routerInt, label, writeConfigTxn);
2128                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2129                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2130                         macAddress);
2131             } else {
2132                 LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2133                                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}",
2134                         interfaceName, primaryRd, vpnName);
2135             }
2136         }
2137     }
2138
2139     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
2140             TypedWriteTransaction<Configuration> writeConfigTxn, String vpnName) {
2141         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
2142         String rd = vpnUtil.getVpnRd(vpnName);
2143         if (adjs != null) {
2144             List<Adjacency> adjsList = adjs.nonnullAdjacency();
2145             for (Adjacency adj : adjsList) {
2146                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2147                     String primaryInterfaceIp = adj.getIpAddress();
2148                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2149                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
2150                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2151                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2152                     return;
2153                 }
2154             }
2155         } else {
2156             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2157                     vpnInterface.getName(), rd);
2158         }
2159     }
2160
2161     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2162         final VpnInterfaceKey key = intefaceData.identifier.firstKeyOf(VpnInterface.class);
2163         final String interfaceName = key.getName();
2164         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2165                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2166         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2167                   intefaceData.identifier, vpnName);
2168     }
2169
2170     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2171                                               VpnInterface vpnInterface, String vpnName) {
2172         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2173                .get(vpnName);
2174         if (vpnInterfaces == null) {
2175             vpnInterfaces = new ConcurrentLinkedQueue<>();
2176         }
2177         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2178         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2179         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2180                 vpnInterface.getName(), vpnName);
2181     }
2182
2183     public boolean isVpnInstanceReady(String vpnInstanceName) {
2184         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2185         if (vpnRd == null) {
2186             return false;
2187         }
2188         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2189
2190         return vpnInstanceOpDataEntry != null;
2191     }
2192
2193     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2194         synchronized (vpnInstanceName.intern()) {
2195             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2196                     unprocessedVpnInterfaces.get(vpnInstanceName);
2197             if (vpnInterfaces != null) {
2198                 while (!vpnInterfaces.isEmpty()) {
2199                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2200                     if (hasVpnInstanceCreatedSuccessfully) {
2201                         processSavedInterface(savedInterface, vpnInstanceName);
2202                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2203                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2204                     } else {
2205                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2206                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2207                     }
2208                 }
2209             } else {
2210                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2211             }
2212         }
2213     }
2214
2215     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2216             VpnInterface vpnInterface) {
2217         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2218             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2219                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2220             if (vpnInterfaces != null) {
2221                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2222                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2223                             + "unprocessed list", vpnInterface.getName(),
2224                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2225                 }
2226             } else {
2227                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2228                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2229             }
2230         }
2231     }
2232
2233     public void vpnInstanceIsReady(String vpnInstanceName) {
2234         processSavedInterfaces(vpnInstanceName, true);
2235     }
2236
2237     public void vpnInstanceFailed(String vpnInstanceName) {
2238         processSavedInterfaces(vpnInstanceName, false);
2239     }
2240
2241     private static class UnprocessedVpnInterfaceData {
2242         InstanceIdentifier<VpnInterface> identifier;
2243         VpnInterface vpnInterface;
2244
2245         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2246             this.identifier = identifier;
2247             this.vpnInterface = vpnInterface;
2248         }
2249
2250         @Override
2251         public int hashCode() {
2252             final int prime = 31;
2253             int result = 1;
2254             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2255             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2256             return result;
2257         }
2258
2259         @Override
2260         public boolean equals(Object obj) {
2261             if (this == obj) {
2262                 return true;
2263             }
2264             if (obj == null) {
2265                 return false;
2266             }
2267             if (getClass() != obj.getClass()) {
2268                 return false;
2269             }
2270             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2271             if (identifier == null) {
2272                 if (other.identifier != null) {
2273                     return false;
2274                 }
2275             } else if (!identifier.equals(other.identifier)) {
2276                 return false;
2277             }
2278             if (vpnInterface == null) {
2279                 if (other.vpnInterface != null) {
2280                     return false;
2281                 }
2282             } else if (!vpnInterface.equals(other.vpnInterface)) {
2283                 return false;
2284             }
2285             return true;
2286         }
2287     }
2288
2289     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2290         String primaryRd = vpnUtil.getVpnRd(vpnName);
2291         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2292         if (vpnInstanceOpData == null) {
2293             return;
2294         }
2295         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2296         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2297             return;
2298         }
2299         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2300         vpnToDpnLists.forEach(vpnToDpnList -> {
2301             if (vpnToDpnList.getVpnInterfaces() == null) {
2302                 return;
2303             }
2304             vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2305                 try {
2306                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2307                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2308                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2309                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2310                     if (!vpnInterfaceOptional.isPresent()) {
2311                         return;
2312                     }
2313                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2314                             vpnInterface.getInterfaceName());
2315                     if (configVpnAdjacencies == null) {
2316                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2317                         return;
2318                     }
2319                     List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2320                             .augmentation(AdjacenciesOp.class).nonnullAdjacency();
2321                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2322                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2323                     // in operational DS. These unprocessed adjacencies will be handled below.
2324                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2325                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2326                     configVpnAdjacencies.stream()
2327                         .filter(adjacency -> operationVpnAdjacencies.stream()
2328                                 .noneMatch(operationalAdjacency ->
2329                                     Objects.equals(operationalAdjacency.getIpAddress(), adjacency.getIpAddress())))
2330                         .forEach(adjacency -> {
2331                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2332                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2333                                 () -> {
2334                                     // TODO Deal with sequencing — the config tx must only submitted
2335                                     // if the oper tx goes in
2336                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2337                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2338                                         futures.add(
2339                                             txRunner.callWithNewWriteOnlyTransactionAndSubmit(OPERATIONAL, operTx -> {
2340                                                 //set of prefix used, as entry in prefix-to-interface datastore
2341                                                 // is prerequisite for refresh Fib to avoid race condition leading
2342                                                 // to missing remote next hop in bucket actions on bgp-vpn delete
2343                                                 Set<String> prefixListForRefreshFib = new HashSet<>();
2344                                                 ListenableFuture<Void> configTxFuture =
2345                                                     txRunner.callWithNewReadWriteTransactionAndSubmit(CONFIGURATION,
2346                                                         confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2347                                                             primaryRd, adjacency, vpnInterfaceOptional.get().getDpnId(),
2348                                                                 operTx, confTx, confTx, prefixListForRefreshFib));
2349                                                 Futures.addCallback(configTxFuture,
2350                                                     new VpnInterfaceCallBackHandler(primaryRd, prefixListForRefreshFib),
2351                                                     MoreExecutors.directExecutor());
2352                                                 futures.add(configTxFuture);
2353                                             }));
2354                                         return futures;
2355                                     } else {
2356                                         return emptyList();
2357                                     }
2358                                 });
2359                         });
2360                 } catch (ReadFailedException e) {
2361                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2362                             vpnName, primaryRd);
2363                 }
2364             });
2365         });
2366     }
2367
2368     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2369         private final String interfaceName;
2370         private final boolean add;
2371         private final String txnDestination;
2372
2373         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2374             this.interfaceName = interfaceName;
2375             this.add = add;
2376             this.txnDestination = transactionDest;
2377         }
2378
2379         @Override
2380         public void onSuccess(Void voidObj) {
2381             if (add) {
2382                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2383                         interfaceName, txnDestination);
2384             } else {
2385                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2386             }
2387         }
2388
2389         @Override
2390         public void onFailure(Throwable throwable) {
2391             if (add) {
2392                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2393                         interfaceName, txnDestination, throwable);
2394             } else {
2395                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2396                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2397             }
2398         }
2399     }
2400
2401     private class VpnInterfaceCallBackHandler implements FutureCallback<Void> {
2402         private final String primaryRd;
2403         private final Set<String> prefixListForRefreshFib;
2404
2405         VpnInterfaceCallBackHandler(String primaryRd, Set<String> prefixListForRefreshFib) {
2406             this.primaryRd = primaryRd;
2407             this.prefixListForRefreshFib = prefixListForRefreshFib;
2408         }
2409
2410         @Override
2411         public void onSuccess(Void voidObj) {
2412             prefixListForRefreshFib.forEach(prefix -> {
2413                 fibManager.refreshVrfEntry(primaryRd, prefix);
2414             });
2415         }
2416
2417         @Override
2418         public void onFailure(Throwable throwable) {
2419             LOG.debug("write Tx config operation failed {}", throwable);
2420         }
2421     }
2422 }