vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.collect.Iterators;
  13 import com.google.common.util.concurrent.FutureCallback;
  14 import com.google.common.util.concurrent.Futures;
  15 import com.google.common.util.concurrent.ListenableFuture;
  16 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  17 import java.math.BigInteger;
  18 import java.util.ArrayList;
  19 import java.util.Collections;
  20 import java.util.Iterator;
  21 import java.util.List;
  22 import java.util.Map;
  23 import java.util.Objects;
  24 import java.util.concurrent.BlockingQueue;
  25 import java.util.concurrent.ConcurrentHashMap;
  26 import java.util.concurrent.ConcurrentLinkedQueue;
  27 import java.util.concurrent.ExecutionException;
  28 import java.util.concurrent.Executors;
  29 import java.util.concurrent.LinkedBlockingQueue;
  30 import java.util.concurrent.ScheduledExecutorService;
  31 import java.util.concurrent.TimeUnit;
  32 import java.util.function.Consumer;
  33 import java.util.function.Predicate;
  34 import java.util.stream.Collectors;
  35 import javax.annotation.PostConstruct;
  36 import javax.annotation.PreDestroy;
  37 import javax.inject.Inject;
  38 import javax.inject.Singleton;
  39 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  40 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  41 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  42 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  43 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  44 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  46 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  47 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  48 import org.opendaylight.genius.mdsalutil.NwConstants;
  49 import org.opendaylight.genius.mdsalutil.cache.DataObjectCache;
  50 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  51 import org.opendaylight.infrautils.caches.CacheProvider;
  52 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  53 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  54 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  55 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  56 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  57 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  58 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  59 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  60 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  61 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  62 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  63 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  64 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  65 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  66 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  67 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  68 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  69 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  70 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListBuilder;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfaces;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesKey;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
 115 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 116 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 117 import org.slf4j.Logger;
 118 import org.slf4j.LoggerFactory;
 119
 120 @Singleton
 121 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 122
 123     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 124     private static final int VPN_INF_UPDATE_TIMER_TASK_DELAY = 1000;
 125     private static final TimeUnit TIME_UNIT = TimeUnit.MILLISECONDS;
 126     private static final short DJC_MAX_RETRIES = 3;
 127
 128     private final DataBroker dataBroker;
 129     private final ManagedNewTransactionRunner txRunner;
 130     private final IBgpManager bgpManager;
 131     private final IFibManager fibManager;
 132     private final IMdsalApiManager mdsalManager;
 133     private final IdManagerService idManager;
 134     private final OdlInterfaceRpcService ifaceMgrRpcService;
 135     private final VpnFootprintService vpnFootprintService;
 136     private final IInterfaceManager interfaceManager;
 137     private final IVpnManager vpnManager;
 138     private final ArpResponderHandler arpResponderHandler;
 139     private final JobCoordinator jobCoordinator;
 140
 141     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 142
 143     private final BlockingQueue<UpdateData> vpnInterfacesUpdateQueue = new LinkedBlockingQueue<>();
 144     private final ScheduledExecutorService vpnInfUpdateTaskExecutor = Executors.newScheduledThreadPool(1);
 145
 146     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 147             new ConcurrentHashMap<>();
 148
 149     private final DataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 150
 151     @Inject
 152     public VpnInterfaceManager(final DataBroker dataBroker,
 153                                final IBgpManager bgpManager,
 154                                final IdManagerService idManager,
 155                                final IMdsalApiManager mdsalManager,
 156                                final IFibManager fibManager,
 157                                final OdlInterfaceRpcService ifaceMgrRpcService,
 158                                final VpnFootprintService vpnFootprintService,
 159                                final IInterfaceManager interfaceManager,
 160                                final IVpnManager vpnManager,
 161                                final ArpResponderHandler arpResponderHandler,
 162                                final JobCoordinator jobCoordinator,
 163                                final CacheProvider cacheProvider) {
 164         super(VpnInterface.class, VpnInterfaceManager.class);
 165
 166         this.dataBroker = dataBroker;
 167         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 168         this.bgpManager = bgpManager;
 169         this.idManager = idManager;
 170         this.mdsalManager = mdsalManager;
 171         this.fibManager = fibManager;
 172         this.ifaceMgrRpcService = ifaceMgrRpcService;
 173         this.vpnFootprintService = vpnFootprintService;
 174         this.interfaceManager = interfaceManager;
 175         this.vpnManager = vpnManager;
 176         this.arpResponderHandler = arpResponderHandler;
 177         this.jobCoordinator = jobCoordinator;
 178         vpnInfUpdateTaskExecutor.scheduleWithFixedDelay(new VpnInterfaceUpdateTimerTask(),
 179             0, VPN_INF_UPDATE_TIMER_TASK_DELAY, TIME_UNIT);
 180
 181         vpnInstanceOpDataEntryCache = new DataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 182                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 183                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 184     }
 185
 186     public Runnable isNotifyTaskQueued(String intfName) {
 187         return vpnIntfMap.remove(intfName);
 188     }
 189
 190     @PostConstruct
 191     public void start() {
 192         LOG.info("{} start", getClass().getSimpleName());
 193         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 194     }
 195
 196     @Override
 197     @PreDestroy
 198     public void close() {
 199         super.close();
 200         vpnInfUpdateTaskExecutor.shutdown();
 201         vpnInstanceOpDataEntryCache.close();
 202     }
 203
 204     @Override
 205     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 206         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 207     }
 208
 209     @Override
 210     protected VpnInterfaceManager getDataTreeChangeListener() {
 211         return VpnInterfaceManager.this;
 212     }
 213
 214     @Override
 215     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 216         LOG.info("add: intfName {} onto vpnName {}",
 217                  vpnInterface.getName(),
 218                  VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 219         addVpnInterface(identifier, vpnInterface, null, null);
 220     }
 221
 222     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 223                           final VpnInterface vpnInterface, String vpnName) {
 224         synchronized (vpnName.intern()) {
 225             if (isVpnInstanceReady(vpnName)) {
 226                 return true;
 227             }
 228             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 229             return false;
 230         }
 231     }
 232
 233     // TODO Clean up the exception handling
 234     @SuppressWarnings("checkstyle:IllegalCatch")
 235     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 236                              final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs) {
 237         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
 238             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 239             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 240         }
 241     }
 242
 243     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 244                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 245         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 246         final String interfaceName = key.getName();
 247
 248         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 249             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 250                   vpnName, vpnInterface.getName());
 251             return;
 252         }
 253         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 254              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 255         List<Adjacency> copyOldAdjs = null;
 256         if (oldAdjs != null) {
 257             copyOldAdjs = new ArrayList<>();
 258             copyOldAdjs.addAll(oldAdjs);
 259         }
 260         List<Adjacency> copyNewAdjs = null;
 261         if (newAdjs != null) {
 262             copyNewAdjs = new ArrayList<>();
 263             copyNewAdjs.addAll(newAdjs);
 264         }
 265         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 266     }
 267
 268     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 269                     final VpnInterface vpnInterface, final List<Adjacency> oldAdjs,
 270                     final List<Adjacency> newAdjs,
 271                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 272         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 273         final String interfaceName = key.getName();
 274         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 275         if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
 276             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 277             boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
 278             if (interfaceState != null) {
 279                 try {
 280                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 281                     final int ifIndex = interfaceState.getIfIndex();
 282                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName + vpnName, () -> {
 283                         WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 284                         WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
 285                         WriteTransaction writeInvTxn = dataBroker.newWriteOnlyTransaction();
 286                         LOG.info("addVpnInterface: VPN Interface add event - intfName {} vpnName {} on dpn {}" ,
 287                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 288                         processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false, writeConfigTxn,
 289                                      writeOperTxn, writeInvTxn, interfaceState, vpnName);
 290                         if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 291                             LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 292                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 293                             if (newAdjs != null) {
 294                                 for (Adjacency adj : newAdjs) {
 295                                     if (oldAdjs.contains(adj)) {
 296                                         oldAdjs.remove(adj);
 297                                     } else {
 298                                         if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(
 299                                                                   dataBroker, adj)) {
 300                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd,
 301                                                   adj, dpnId, writeOperTxn, writeConfigTxn);
 302                                         }
 303                                     }
 304                                 }
 305                             }
 306                             for (Adjacency adj : oldAdjs) {
 307                                 if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(
 308                                                           dataBroker, adj)) {
 309                                     delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 310                                         writeOperTxn, writeConfigTxn);
 311                                 }
 312                             }
 313                         }
 314                         ListenableFuture<Void> operFuture = writeOperTxn.submit();
 315                         try {
 316                             operFuture.get();
 317                         } catch (ExecutionException e) {
 318                             LOG.error("addVpnInterface: Exception encountered while submitting operational future for"
 319                                     + " addVpnInterface {} on vpn {}: {}", vpnInterface.getName(), vpnName, e);
 320                             return null;
 321                         }
 322                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 323                         ListenableFuture<Void> configFuture = writeConfigTxn.submit();
 324                         futures.add(configFuture);
 325                         Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"));
 326                         futures.add(writeInvTxn.submit());
 327                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 328                                 + " processed successfully", interfaceName, vpnName, dpnId);
 329                         return futures;
 330                     });
 331                 } catch (NumberFormatException | IllegalStateException e) {
 332                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 333                                     + "interface {}. Interface addition on vpn {} failed", interfaceName,
 334                                     vpnName, e);
 335                     return;
 336                 }
 337             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 338                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName() + vpnName,
 339                     () -> {
 340                         WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 341                         createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 342                                        writeConfigTxn, vpnName);
 343                         LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 344                                  vpnName, vpnInterface.getDpnId());
 345                         ListenableFuture<Void> futures = writeConfigTxn.submit();
 346                         String errorText = "addVpnInterfaceCall: Exception encountered while submitting writeConfigTxn"
 347                              + " for interface " + vpnInterface.getName() + " on vpn " + vpnName;
 348                         ListenableFutures.addErrorLogging(futures, LOG, errorText);
 349                         return Collections.singletonList(futures);
 350                     });
 351             } else {
 352                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 353                         + " is not available", interfaceName, vpnName);
 354             }
 355         } else {
 356             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 357                       + " as vpn is pending delete", interfaceName, vpnName,
 358                     vpnInterface.getDpnId());
 359         }
 360     }
 361
 362     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 363     // see comments below.
 364     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 365     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 366             final int lportTag, boolean isInterfaceUp,
 367             WriteTransaction writeConfigTxn,
 368             WriteTransaction writeOperTxn,
 369             WriteTransaction writeInvTxn,
 370             Interface interfaceState,
 371             final String vpnName) {
 372         final String interfaceName = vpnInterface.getName();
 373         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = VpnUtil.getVpnInterfaceOpDataEntry(dataBroker,
 374                                                          interfaceName, vpnName);
 375         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 376         boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
 377         if (!isInterfaceUp) {
 378             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 379                      interfaceName, dpId, vpnName);
 380             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 381             if (vpnId == VpnConstants.INVALID_ID) {
 382                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 383                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 384                         dpId);
 385                 return;
 386             }
 387
 388             boolean waitForVpnInterfaceOpRemoval = false;
 389             if (opVpnInterface != null && !opVpnInterface.isScheduledForRemove()) {
 390                 String opVpnName = opVpnInterface.getVpnInstanceName();
 391                 String primaryInterfaceIp = null;
 392                 if (opVpnName.equals(vpnName)) {
 393                     // Please check if the primary VRF Entry does not exist for VPNInterface
 394                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 395                     // back to back
 396                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 397                     List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
 398                     if (adjs == null) {
 399                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 400                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 401                                 vpnName);
 402                         return;
 403                     }
 404                     for (Adjacency adj : adjs) {
 405                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 406                             primaryInterfaceIp = adj.getIpAddress();
 407                             break;
 408                         }
 409                     }
 410                     if (primaryInterfaceIp == null) {
 411                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 412                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 413                                 dpId, vpnName);
 414                         return;
 415                     }
 416                     // Get the rd of the vpn instance
 417                     VrfEntry vrf = VpnUtil.getVrfEntry(dataBroker, primaryRd, primaryInterfaceIp);
 418                     if (vrf != null) {
 419                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 420                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 421                         return;
 422                     }
 423                     waitForVpnInterfaceOpRemoval = true;
 424                 } else {
 425                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 426                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 427                 }
 428             }
 429             if (!waitForVpnInterfaceOpRemoval) {
 430                 // Add the VPNInterface and quit
 431                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 432                         null/*ipAddressSourceValuePair*/,
 433                         true /* add */);
 434                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 435                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 436                 if (!isBgpVpnInternetVpn) {
 437                     VpnUtil.bindService(vpnName, interfaceName, dataBroker, false /*isTunnelInterface*/,
 438                             jobCoordinator);
 439                 }
 440                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 441                         dpId, vpnName);
 442                 if (interfaceManager.isExternalInterface(interfaceName)) {
 443                     processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
 444                         NwConstants.ADD_FLOW);
 445                 }
 446                 return;
 447             }
 448
 449             // FIB didn't get a chance yet to clean up this VPNInterface
 450             // Let us give it a chance here !
 451             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 452                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 453             try {
 454                 Runnable notifyTask = new VpnNotifyTask();
 455                 synchronized (notifyTask) {
 456                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 457                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 458                     // published here while holding the lock on it so this path will hit the wait before notify can be
 459                     // invoked.
 460                     vpnIntfMap.put(interfaceName, notifyTask);
 461                     try {
 462                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 463                     } catch (InterruptedException e) {
 464                         // Ignored
 465                     }
 466                 }
 467             } finally {
 468                 vpnIntfMap.remove(interfaceName);
 469             }
 470
 471             if (opVpnInterface != null) {
 472                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 473                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 474                         dpId, vpnName);
 475                 VpnUtil.unsetScheduledToRemoveForVpnInterface(dataBroker, interfaceName);
 476                 return;
 477             }
 478             // VPNInterface got removed, proceed with Add
 479             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 480                     interfaceName, dpId, vpnName);
 481             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 482                     null/*ipAddressSourceValuePair*/,
 483                     true /* add */);
 484             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 485                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 486             if (!isBgpVpnInternetVpn) {
 487                 VpnUtil.bindService(vpnName, interfaceName, dataBroker, false/*isTunnelInterface*/, jobCoordinator);
 488             }
 489             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 490                     + " FIB to clean up", interfaceName, dpId, vpnName);
 491             if (interfaceManager.isExternalInterface(interfaceName)) {
 492                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId,
 493                                lportTag, writeInvTxn, NwConstants.ADD_FLOW);
 494             }
 495         } else {
 496             // Interface is retained in the DPN, but its Link Up.
 497             // Advertise prefixes again for this interface to BGP
 498             InstanceIdentifier<VpnInterface> identifier =
 499                 VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 500             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 501                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 502             advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 503             // Perform similar operation as interface add event for extraroutes.
 504             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 505             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 506             if (!optAdjacencies.isPresent()) {
 507                 LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 508                 return;
 509             }
 510             List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
 511             for (Adjacency adjacency : adjacencies) {
 512                 if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 513                     continue;
 514                 }
 515                 // if BGPVPN Internet, filter only IPv6 Adjacencies
 516                 if (isBgpVpnInternetVpn && !VpnUtil.isAdjacencyEligibleToVpnInternet(
 517                                 dataBroker, adjacency)) {
 518                     continue;
 519                 }
 520                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 521                           dpId, writeOperTxn, writeConfigTxn);
 522             }
 523         }
 524     }
 525
 526     private void processExternalVpnInterface(String interfaceName, String vpnName, long vpnId, BigInteger dpId,
 527             int lportTag, WriteTransaction writeInvTxn, int addOrRemove) {
 528         Uuid extNetworkId;
 529         try {
 530             // vpn instance of ext-net interface is the network-id
 531             extNetworkId = new Uuid(vpnName);
 532         } catch (IllegalArgumentException e) {
 533             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 534                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 535             return;
 536         }
 537
 538         List<Uuid> routerIds = VpnUtil.getExternalNetworkRouterIds(dataBroker, extNetworkId);
 539         if (routerIds == null || routerIds.isEmpty()) {
 540             LOG.info("processExternalVpnInterface: No router is associated with {}."
 541                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 542                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 543             return;
 544         }
 545
 546         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 547                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 548         for (Uuid routerId : routerIds) {
 549             String routerName = routerId.getValue();
 550             BigInteger primarySwitch = VpnUtil.getPrimarySwitchForRouter(dataBroker, routerName);
 551             if (Objects.equals(primarySwitch, dpId)) {
 552                 Routers router = VpnUtil.getExternalRouter(dataBroker, routerName);
 553                 if (router != null) {
 554                     if (addOrRemove == NwConstants.ADD_FLOW) {
 555                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 556                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 557                                 dpId, interfaceName, lportTag, writeInvTxn);
 558                     } else {
 559                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 560                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 561                                 dpId, interfaceName, lportTag);
 562                     }
 563                 } else {
 564                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 565                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 566                             interfaceName, dpId, vpnName);
 567                 }
 568             }
 569         }
 570     }
 571
 572     // TODO Clean up the exception handling
 573     @SuppressWarnings("checkstyle:IllegalCatch")
 574     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 575                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 576                                                   String vpnName, String interfaceName) {
 577         if (rd == null) {
 578             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 579                   interfaceName, dpnId, vpnName);
 580             return;
 581         } else {
 582             if (rd.equals(vpnName)) {
 583                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 584                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 585                 return;
 586             }
 587         }
 588         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 589                 interfaceName, dpnId, vpnName, rd);
 590
 591         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 592         if (nextHopIp == null) {
 593             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 594                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 595                     rd, vpnName);
 596             return;
 597         }
 598
 599         //Read NextHops
 600         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 601         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 602         if (adjacencies.isPresent()) {
 603             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 604
 605             if (!nextHops.isEmpty()) {
 606                 LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 607                         + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 608                 VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, rd);
 609                 long l3vni = vpnInstanceOpData.getL3vni();
 610                 VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 611                         ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 612                 for (Adjacency nextHop : nextHops) {
 613                     if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 614                         continue;
 615                     }
 616                     String gatewayMac = null;
 617                     long label = 0;
 618                     if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 619                         final VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker,
 620                             vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 621                         gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName).get();
 622                     } else {
 623                         label = nextHop.getLabel();
 624                     }
 625                     try {
 626                         LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 627                                 + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 628                         bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 629                                 encapType, (int)label, l3vni, 0 /*l2vni*/,
 630                                 gatewayMac);
 631                         LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 632                                 + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 633                                 nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 634                     } catch (Exception e) {
 635                         LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {} with rd {}"
 636                                 + " for interface {} on dpn {}", nextHop.getIpAddress(),
 637                                   vpnName, rd, interfaceName, dpnId, e);
 638                     }
 639                 }
 640             }
 641         }
 642     }
 643
 644     // TODO Clean up the exception handling
 645     @SuppressWarnings("checkstyle:IllegalCatch")
 646     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 647                         String vpnName, String interfaceName, WriteTransaction writeConfigTxn) {
 648         //Read NextHops
 649         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 650         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 651
 652         String rd = VpnUtil.getVpnRd(dataBroker, interfaceName);
 653         if (rd == null) {
 654             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 655                 interfaceName, vpnName);
 656             return;
 657         } else {
 658             if (rd.equals(vpnName)) {
 659                 LOG.info(
 660                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 661                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 662                 return;
 663             }
 664         }
 665         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 666                vpnName, rd);
 667         if (adjacencies.isPresent()) {
 668             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 669
 670             if (!nextHops.isEmpty()) {
 671                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 672                         nextHops, interfaceName, vpnName, rd);
 673                 for (Adjacency nextHop : nextHops) {
 674                     try {
 675                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 676                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 677                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 678                                     interfaceName, vpnName);
 679                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 680                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 681                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 682                                     interfaceName, vpnName);
 683                         } else {
 684                             // Perform similar operation as interface delete event for extraroutes.
 685                             String allocatedRd = nextHop.getVrfId();
 686                             for (String nh : nextHop.getNextHopIpList()) {
 687                                 deleteExtraRouteFromCurrentAndImportingVpns(
 688                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn);
 689                             }
 690                         }
 691                     } catch (Exception e) {
 692                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 693                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 694                     }
 695                 }
 696             }
 697         }
 698     }
 699
 700     @SuppressWarnings("checkstyle:IllegalCatch")
 701     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 702                                                   String primaryRd, String interfaceName, final long vpnId,
 703                                                   WriteTransaction writeConfigTxn,
 704                                                   WriteTransaction writeOperTxn,
 705                                                   final WriteTransaction writeInvTxn,
 706                                                   Interface interfaceState) {
 707         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 708         // Read NextHops
 709         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 710         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 711         if (!adjacencies.isPresent()) {
 712             addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 713                     null/*gwMac*/, writeOperTxn);
 714             return;
 715         }
 716
 717         // Get the rd of the vpn instance
 718         String nextHopIp = null;
 719         try {
 720             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 721         } catch (Exception e) {
 722             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 723                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 724         }
 725         List<String> nhList = new ArrayList<>();
 726         if (nextHopIp != null) {
 727             nhList.add(nextHopIp);
 728             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 729                     interfaceName, dpnId, vpnName, nhList);
 730         }
 731         Optional<String> gwMac = Optional.absent();
 732         String vpnInterfaceSubnetGwMacAddress = null;
 733         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
 734         Long l3vni = vpnInstanceOpData.getL3vni();
 735         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 736         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 737         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 738         List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 739         List<Adjacency> value = new ArrayList<>();
 740         for (Adjacency nextHop : nextHops) {
 741             String rd = primaryRd;
 742             Subnetmap sn = VpnUtil.getSubnetmapFromItsUuid(dataBroker, nextHop.getSubnetId());
 743             if (!VpnUtil.isSubnetPartOfVpn(sn, vpnName)) {
 744                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 745                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 746                 LOG.debug("processVpnInterfaceAdjacencies: Not Adding prefix {} to interface {}"
 747                       + " as the subnet is not part of vpn {}", prefix, interfaceName, vpnName);
 748                 continue;
 749             }
 750             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 751                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 752                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 753                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 754                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 755                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 756                 writeOperTxn.merge(
 757                     LogicalDatastoreType.OPERATIONAL,
 758                     VpnUtil.getPrefixToInterfaceIdentifier(
 759                         VpnUtil.getVpnId(dataBroker, vpnName), prefix),
 760                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, nextHop.getSubnetId(),
 761                             prefixCue), true);
 762                 final Uuid subnetId = nextHop.getSubnetId();
 763                 final Optional<String> gatewayIp = VpnUtil.getVpnSubnetGatewayIp(dataBroker, subnetId);
 764                 if (gatewayIp.isPresent()) {
 765                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp.get());
 766                     if (gwMac.isPresent()) {
 767                         // A valid mac-address is available for this subnet-gateway-ip
 768                         // Use this for programming ARP_RESPONDER table here.  And save this
 769                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 770                         // to populate L3_GW_MAC_TABLE there.
 771                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 772                                 gatewayIp.get(), gwMac.get());
 773                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 774                     } else {
 775                         // A valid mac-address is not available for this subnet-gateway-ip
 776                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 777                         // Save this connected-mac-address as gateway-mac-address for the
 778                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 779                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 780                         if (gwMac.isPresent()) {
 781                             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
 782                                     vpnId, writeInvTxn, NwConstants.ADD_FLOW, gwMac.get());
 783                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 784                                     gatewayIp.get(), gwMac.get());
 785                         } else {
 786                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 787                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 788                                 + "gwIp {}",
 789                                 interfaceName, vpnName, gatewayIp.get());
 790                         }
 791                     }
 792                 } else {
 793                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 794                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 795                         subnetId, interfaceName, vpnName);
 796                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 797                 }
 798                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 799                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 800             } else {
 801                 //Extra route adjacency
 802                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 803                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 804                 synchronized (vpnPrefixKey.intern()) {
 805                     java.util.Optional<String> rdToAllocate = VpnUtil
 806                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(dataBroker, vpnId, null,
 807                             prefix, vpnName, nextHop.getNextHopIpList().get(0), dpnId);
 808                     if (rdToAllocate.isPresent()) {
 809                         rd = rdToAllocate.get();
 810                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 811                             rd, prefix);
 812                     } else {
 813                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 814                         continue;
 815                     }
 816                 }
 817                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 818                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 819                         interfaceName, dpnId);
 820             }
 821             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 822             // can be different from the gateway-mac-address within the VRFEntry as the
 823             // gateway-mac-address is a superset.
 824             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 825                     : RouteOrigin.STATIC;
 826             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 827                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 828                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 829             Adjacency operationalAdjacency = null;
 830             try {
 831                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 832             } catch (NullPointerException e) {
 833                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 834                     input, e.getMessage());
 835                 return;
 836             }
 837             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 838                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 839                         vpnName, l3vni, origin,
 840                         interfaceName, operationalAdjacency, encapType, writeConfigTxn);
 841             }
 842             value.add(operationalAdjacency);
 843         }
 844
 845         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 846         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 847                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 848
 849         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 850                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 851                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 852
 853         for (Adjacency nextHop : aug.getAdjacency()) {
 854             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 855             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 856                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 857                         : RouteOrigin.STATIC;
 858                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 859                 registeredPopulator.populateFib(input, writeConfigTxn);
 860             }
 861         }
 862     }
 863
 864     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 865                                               long lportTag, String gwMac, WriteTransaction writeOperTxn) {
 866         VpnInterfaceOpDataEntry opInterface =
 867               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, Boolean.FALSE, lportTag, gwMac);
 868         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 869             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 870         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 871                 WriteTransaction.CREATE_MISSING_PARENTS);
 872         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 873                 interfaceName, dpnId, vpnName);
 874     }
 875
 876     // TODO Clean up the exception handling
 877     @SuppressWarnings("checkstyle:IllegalCatch")
 878     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 879                                            StateTunnelList stateTunnelList,
 880                                            WriteTransaction writeConfigTxn,
 881                                            WriteTransaction writeOperTxn) {
 882
 883         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
 884         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 885         AdjacenciesOp adjacencies = vpnInterface.getAugmentation(AdjacenciesOp.class);
 886         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 887         if (adjList.isEmpty()) {
 888             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 889                     vpnInterface, srcDpnId);
 890             return;
 891         }
 892         String prefix = null;
 893         long label = 0;
 894         List<Adjacency> value = new ArrayList<>();
 895         boolean isNextHopAddReqd = false;
 896         String vpnName = vpnInterface.getVpnInstanceName();
 897         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 898         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 899         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 900                 vpnInterface.getName(), vpnInterface.getDpnId(),
 901                 vpnInterface.getVpnInstanceName(), adjList);
 902         for (Adjacency adj : adjList) {
 903             String rd = adj.getVrfId();
 904             rd = rd != null ? rd : vpnName;
 905             prefix = adj.getIpAddress();
 906             label = adj.getLabel();
 907             List<String> nhList = Collections.singletonList(srcTepIp);
 908             List<String> nextHopList = adj.getNextHopIpList();
 909             // If TEP is added , update the nexthop of primary adjacency.
 910             // Secondary adj nexthop is already pointing to primary adj IP address.
 911             if (nextHopList != null && !nextHopList.isEmpty() && nextHopList.get(0).equalsIgnoreCase(srcTepIp)) {
 912                 /* everything right already */
 913             } else {
 914                 isNextHopAddReqd = true;
 915             }
 916
 917             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 918                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 919             } else {
 920                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 921                 if (!vrfEntryOptional.isPresent()) {
 922                     continue;
 923                 }
 924                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 925                 if (!nhList.contains(srcTepIp)) {
 926                     nhList.add(srcTepIp);
 927                     isNextHopAddReqd = true;
 928                 }
 929                 value.add(adj);
 930             }
 931
 932             if (isNextHopAddReqd) {
 933                 updateLabelMapper(label, nhList);
 934                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 935                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 936                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 937                 // Update the VRF entry with nextHop
 938                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 939                         label, true, writeConfigTxn);
 940
 941                 //Get the list of VPN's importing this route(prefix) .
 942                 // Then update the VRF entry with nhList
 943                 List<VpnInstanceOpDataEntry> vpnsToImportRoute =
 944                            VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
 945                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 946                     String vpnRd = vpn.getVrfId();
 947                     if (vpnRd != null) {
 948                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
 949                                 srcTepIp, label, true, writeConfigTxn);
 950                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 951                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
 952                             vpnInterface.getName(), srcDpnId, vpnName,
 953                             vpn.getVpnInstanceName(), vpnRd);
 954                     }
 955                 }
 956                 // Advertise the prefix to BGP only for external vpn
 957                 // since there is a nexthop change.
 958                 try {
 959                     if (!rd.equalsIgnoreCase(vpnName)) {
 960                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
 961                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
 962                                 null /*gatewayMacAddress*/);
 963                     }
 964                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
 965                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
 966                             srcDpnId, vpnName);
 967                 } catch (Exception ex) {
 968                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
 969                             + " on rd {} for interface {} on dpn {} vpn {} as {}", prefix, nhList, label, rd,
 970                             vpnInterface.getName(), srcDpnId,
 971                             vpnName, ex);
 972                 }
 973             }
 974         }
 975         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 976         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
 977                 .setKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
 978                 .addAugmentation(AdjacenciesOp.class, aug).build();
 979         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
 980                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
 981         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 982                 WriteTransaction.CREATE_MISSING_PARENTS);
 983         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
 984                 vpnInterface.getName(), srcDpnId, vpnName);
 985
 986     }
 987
 988     // TODO Clean up the exception handling
 989     @SuppressWarnings("checkstyle:IllegalCatch")
 990     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
 991                                               StateTunnelList stateTunnelList,
 992                                               WriteTransaction writeConfigTxn,
 993                                               WriteTransaction writeOperTxn) {
 994
 995         AdjacenciesOp adjacencies = vpnInterface.getAugmentation(AdjacenciesOp.class);
 996         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 997         String prefix = null;
 998         long label = 0;
 999         boolean isNextHopRemoveReqd = false;
1000         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
1001         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
1002         String vpnName = vpnInterface.getVpnInstanceName();
1003         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1004         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1005         if (adjList != null) {
1006             List<Adjacency> value = new ArrayList<>();
1007             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1008                     vpnInterface.getName(), vpnInterface.getDpnId(),
1009                     vpnInterface.getVpnInstanceName(), adjList);
1010             for (Adjacency adj : adjList) {
1011                 List<String> nhList = new ArrayList<>();
1012                 String rd = adj.getVrfId();
1013                 rd = rd != null ? rd : vpnName;
1014                 prefix = adj.getIpAddress();
1015                 List<String> nextHopList = adj.getNextHopIpList();
1016                 label = adj.getLabel();
1017                 if (nextHopList != null && !nextHopList.isEmpty()) {
1018                     isNextHopRemoveReqd = true;
1019                 }
1020                 // If TEP is deleted , remove the nexthop from primary adjacency.
1021                 // Secondary adj nexthop will continue to point to primary adj IP address.
1022                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1023                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1024                 } else {
1025                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1026                     if (!vrfEntryOptional.isPresent()) {
1027                         continue;
1028                     }
1029                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1030                     if (nhList.contains(srcTepIp)) {
1031                         nhList.remove(srcTepIp);
1032                         isNextHopRemoveReqd = true;
1033                     }
1034                     value.add(adj);
1035                 }
1036
1037                 if (isNextHopRemoveReqd) {
1038                     updateLabelMapper(label, nhList);
1039                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1040                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1041                             prefix, nhList, vpnName,
1042                             vpnId, rd, vpnInterface.getName());
1043                     // Update the VRF entry with removed nextHop
1044                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1045                             label, false, writeConfigTxn);
1046
1047                     //Get the list of VPN's importing this route(prefix) .
1048                     // Then update the VRF entry with nhList
1049                     List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1050                           VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1051                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1052                         String vpnRd = vpn.getVrfId();
1053                         if (vpnRd != null) {
1054                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1055                                     srcTepIp, label, false, writeConfigTxn);
1056                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1057                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1058                                     nhList, label, vpnInterface.getName(), srcDpnId,
1059                                     vpnName,
1060                                     vpn.getVpnInstanceName(), vpnRd);
1061                         }
1062                     }
1063
1064                     // Withdraw prefix from BGP only for external vpn.
1065                     try {
1066                         if (!rd.equalsIgnoreCase(vpnName)) {
1067                             bgpManager.withdrawPrefix(rd, prefix);
1068                         }
1069                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1070                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1071                                 vpnInterface.getName(), srcDpnId,
1072                                 vpnName);
1073                     } catch (Exception ex) {
1074                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1075                                 + " on rd {} for interface {} on dpn {} vpn {} as {}", prefix, nhList, label, rd,
1076                                 vpnInterface.getName(), srcDpnId,
1077                                 vpnName, ex);
1078                     }
1079                 }
1080             }
1081             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1082             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1083                     .setKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1084                     .addAugmentation(AdjacenciesOp.class, aug).build();
1085             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1086                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1087             writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
1088                     WriteTransaction.CREATE_MISSING_PARENTS);
1089             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1090                          vpnInterface.getName(), srcDpnId, vpnName);
1091         }
1092     }
1093
1094     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1095         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1096
1097         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1098         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
1099         if (vpnInstanceOpDataEntry == null) {
1100             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1101                     + " to check for vpns exporting the routes", vpnName);
1102             return vpnsToExportRoute;
1103         }
1104
1105         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1106             if (input.getVpnInstanceName() == null) {
1107                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1108                         input.getVrfId());
1109                 return false;
1110             }
1111             return !input.getVpnInstanceName().equals(vpnName);
1112         };
1113
1114         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1115             Iterable<String> commonRTs =
1116                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1117                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1118             return Iterators.size(commonRTs.iterator()) > 0;
1119         };
1120
1121         vpnsToExportRoute =
1122                 VpnUtil.getAllVpnInstanceOpData(dataBroker).stream().filter(excludeVpn).filter(matchRTs).collect(
1123                         Collectors.toList());
1124         return vpnsToExportRoute;
1125     }
1126
1127     // TODO Clean up the exception handling
1128     @SuppressWarnings("checkstyle:IllegalCatch")
1129     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1130         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1131         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1132             List<VrfEntry> vrfEntries = VpnUtil.getAllVrfEntries(dataBroker, vpn.getVrfId());
1133             WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1134             if (vrfEntries != null) {
1135                 for (VrfEntry vrfEntry : vrfEntries) {
1136                     try {
1137                         if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1138                                 RouteOrigin.value(vrfEntry.getOrigin()))) {
1139                             LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1140                                     + " is not a controller managed non intervpn link route. Ignoring.",
1141                                     vpn.getVrfId(), vrfEntry.getDestPrefix());
1142                             continue;
1143                         }
1144                         String prefix = vrfEntry.getDestPrefix();
1145                         String gwMac = vrfEntry.getGatewayMacAddress();
1146                         vrfEntry.getRoutePaths().forEach(routePath -> {
1147                             String nh = routePath.getNexthopAddress();
1148                             int label = routePath.getLabel().intValue();
1149                             if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1150                                     vrfEntry.getOrigin()))) {
1151                                 LOG.info("handleVpnsExportingRoutesImporting: Importing fib entry rd {} prefix {}"
1152                                         + " nexthop {} label {} to vpn {} vpnRd {}", vpn.getVrfId(), prefix, nh, label,
1153                                         vpnName, vpnRd);
1154                                 fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1155                                         Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1156                                         0 /*l3vni*/, gwMac,  null /*parentVpnRd*/, RouteOrigin.SELF_IMPORTED,
1157                                         writeConfigTxn);
1158                             } else {
1159                                 LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry rd {} prefix {}"
1160                                         + " nexthop {} label {} to vpn {} vpnRd {}", vpn.getVrfId(), prefix, nh, label,
1161                                         vpnName, vpnRd);
1162                                 SubnetRoute route = vrfEntry.getAugmentation(SubnetRoute.class);
1163                                 importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, writeConfigTxn);
1164                             }
1165                         });
1166                     } catch (RuntimeException e) {
1167                         LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1168                                 + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1169                                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1170                     }
1171                 }
1172                 writeConfigTxn.submit();
1173             } else {
1174                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1175                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1176             }
1177         }
1178     }
1179
1180     @SuppressWarnings("checkstyle:IllegalCatch")
1181     @Override
1182     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1183         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1184         final String interfaceName = key.getName();
1185         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
1186             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1187             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1188         }
1189     }
1190
1191     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1192                                 final VpnInterface vpnInterface, final String vpnName,
1193                                 final String interfaceName) {
1194         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1195             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName() + vpnName, () -> {
1196                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1197                 deleteFibEntryForRouterInterface(vpnInterface, writeConfigTxn, vpnName);
1198                 LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1199                 ListenableFuture<Void> futures = writeConfigTxn.submit();
1200                 String errorText = "removeVpnInterfaceCall: Exception encountered while submitting writeConfigTxn"
1201                         + " for interface " + vpnInterface.getName() + " on vpn " + vpnName;
1202                 ListenableFutures.addErrorLogging(futures, LOG, errorText);
1203                 return Collections.singletonList(futures);
1204             }, DJC_MAX_RETRIES);
1205         } else {
1206             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1207             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1208         }
1209     }
1210
1211     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1212     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1213                                 final VpnInterface vpnInterface, final String vpnName,
1214                                 final String interfaceName, final Interface interfaceState) {
1215         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1216                 vpnName, vpnInterface.getDpnId());
1217         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1218         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName + vpnName,
1219             () -> {
1220                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1221                 ListenableFuture<Void> configFuture = txRunner
1222                         .callWithNewWriteOnlyTransactionAndSubmit(writeConfigTxn -> {
1223                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeOperTxn -> {
1224                                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeInvTxn -> {
1225                                     LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1226                                             interfaceName, vpnName);
1227                                     BigInteger dpId = BigInteger.ZERO;
1228                                     int ifIndex = 0;
1229                                     String gwMacAddress = null;
1230                                     InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1231                                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1232                                     final Optional<VpnInterfaceOpDataEntry> optVpnInterface =
1233                                             VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, interfaceId);
1234                                     if (interfaceState != null) {
1235                                         try {
1236                                             dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1237                                         } catch (NumberFormatException | IllegalStateException e) {
1238                                             LOG.error("remove: Unable to retrieve dpnId from interface operational"
1239                                                     + " data store for interface {} on dpn {} for vpn {} Fetching"
1240                                                             + " from vpn interface op data store. ", interfaceName,
1241                                                     vpnInterface.getDpnId(), vpnName, e);
1242                                             dpId = BigInteger.ZERO;
1243                                         }
1244                                         ifIndex = interfaceState.getIfIndex();
1245                                         gwMacAddress = interfaceState.getPhysAddress().getValue();
1246                                     } else {
1247                                         LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1248                                                 + " from vpn interface op.", interfaceName);
1249                                         if (optVpnInterface.isPresent()) {
1250                                             VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1251                                             dpId = vpnOpInterface.getDpnId();
1252                                             ifIndex = vpnOpInterface.getLportTag().intValue();
1253                                             gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1254                                         } else {
1255                                             LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1256                                                             + " as interfaceState and vpn interface op is not"
1257                                                     + " available", interfaceName, vpnName);
1258                                             return;
1259                                         }
1260                                     }
1261                                     processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1262                                             optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1263                                             writeConfigTxn, writeOperTxn, writeInvTxn);
1264                                     LOG.info(
1265                                             "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1266                                                     + "successfully",
1267                                             interfaceName, vpnInterface.getDpnId(), vpnName);
1268                                 }));
1269                             }));
1270                         });
1271                 futures.add(configFuture);
1272                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1273                 return futures;
1274             }, DJC_MAX_RETRIES);
1275     }
1276
1277     protected void processVpnInterfaceDown(BigInteger dpId,
1278                                            String interfaceName,
1279                                            int lportTag,
1280                                            String gwMac,
1281                                            VpnInterfaceOpDataEntry vpnOpInterface,
1282                                            boolean isInterfaceStateDown,
1283                                            WriteTransaction writeConfigTxn,
1284                                            WriteTransaction writeOperTxn,
1285                                            WriteTransaction writeInvTxn) {
1286         if (vpnOpInterface == null) {
1287             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1288                     + " as it is not available in operational data store", interfaceName, dpId);
1289             return;
1290         }
1291         final String vpnName = vpnOpInterface.getVpnInstanceName();
1292         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1293                                                     interfaceName, vpnName);
1294         if (!isInterfaceStateDown) {
1295             final long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1296             VpnUtil.scheduleVpnInterfaceForRemoval(dataBroker, interfaceName, dpId, vpnName, Boolean.TRUE,
1297                     null);
1298             final boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
1299             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1300                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1301             if (interfaceManager.isExternalInterface(interfaceName)) {
1302                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
1303                         NwConstants.DEL_FLOW);
1304             }
1305             if (!isBgpVpnInternetVpn) {
1306                 VpnUtil.unbindService(dataBroker, interfaceName, isInterfaceStateDown, jobCoordinator);
1307             }
1308             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1309                     + " successful", interfaceName, dpId, vpnName);
1310         } else {
1311             // Interface is retained in the DPN, but its Link Down.
1312             // Only withdraw the prefixes for this interface from BGP
1313             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn);
1314         }
1315     }
1316
1317     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1318                                           final String vpnName, final long vpnId, String gwMac,
1319                                           WriteTransaction writeConfigTxn, final WriteTransaction writeOperTxn,
1320                                           final WriteTransaction writeInvTxn) {
1321         //Read NextHops
1322         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1323                 .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1324         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1325         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1326
1327         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1328         LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1329                 interfaceName, dpnId, vpnName, primaryRd);
1330         if (adjacencies.isPresent() && !adjacencies.get().getAdjacency().isEmpty()) {
1331             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1332             LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1333                     interfaceName, dpnId, vpnName, nextHops);
1334             for (Adjacency nextHop : nextHops) {
1335                 if (nextHop.isPhysNetworkFunc()) {
1336                     LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1337                             nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1338                     fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1339                             null/*writeCfgTxn*/);
1340                 } else {
1341                     String rd = nextHop.getVrfId();
1342                     List<String> nhList;
1343                     if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1344                         nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1345                     } else {
1346                         // This is a primary adjacency
1347                         nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1348                                 : Collections.emptyList();
1349                         removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1350                                 interfaceName, writeInvTxn);
1351                     }
1352                     if (!nhList.isEmpty()) {
1353                         if (rd.equals(vpnName)) {
1354                             //this is an internal vpn - the rd is assigned to the vpn instance name;
1355                             //remove from FIB directly
1356                             nhList.forEach((nh) -> removeAdjacencyFromInternalVpn(nextHop, vpnName,
1357                                     interfaceName, dpnId, writeConfigTxn));
1358                         } else {
1359                             removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd, interfaceName,
1360                                     writeConfigTxn);
1361                         }
1362                     } else {
1363                         LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1364                                         + " interface {}", nextHop.getIpAddress(), rd,
1365                                 nextHop.getAdjacencyType().toString(), interfaceName);
1366                         bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
1367                         fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1368                     }
1369                 }
1370                 String ip = nextHop.getIpAddress().split("/")[0];
1371                 LearntVpnVipToPort vpnVipToPort = VpnUtil.getLearntVpnVipToPort(dataBroker, vpnName, ip);
1372                 if (vpnVipToPort != null) {
1373                     VpnUtil.removeLearntVpnVipToPort(dataBroker, vpnName, ip);
1374                     LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed adjacency for Interface {}"
1375                                     + " ip {} on dpn {} for vpn {} from VpnPortData Entry", vpnVipToPort.getPortName(),
1376                             ip, dpnId, vpnName);
1377                 }
1378             }
1379         } else {
1380             // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1381             LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1382                     + " Removing it.", interfaceName, vpnName, dpnId);
1383             writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, identifier);
1384         }
1385     }
1386
1387     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1388                                                                    String interfaceName, BigInteger dpnId,
1389                                                                    WriteTransaction writeConfigTxn) {
1390         return (nh) -> {
1391             fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1392                     writeConfigTxn);
1393             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1394                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1395                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1396         };
1397     }
1398
1399     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1400                                            BigInteger dpnId, String rd, String interfaceName,
1401                                            WriteTransaction writeConfigTxn) {
1402         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1403                 VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1404         nhList.forEach((nh) -> {
1405             //IRT: remove routes from other vpns importing it
1406             vpnManager.removePrefixFromBGP(primaryRd, rd, vpnName, nextHop.getIpAddress(),
1407                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn);
1408             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1409                 String vpnRd = vpn.getVrfId();
1410                 if (vpnRd != null) {
1411                     fibManager.removeOrUpdateFibEntry(vpnRd,
1412                             nextHop.getIpAddress(), nh, writeConfigTxn);
1413                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1414                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1415                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1416                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1417                 }
1418             }
1419         });
1420     }
1421
1422     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1423                                                  int lportTag, String gwMac, String interfaceName,
1424                                                  WriteTransaction writeInvTxn) {
1425         final Uuid subnetId = nextHop.getSubnetId();
1426         if (nextHop.getSubnetGatewayMacAddress() == null) {
1427             // A valid mac-address was not available for this subnet-gateway-ip
1428             // So a connected-mac-address was used for this subnet and we need
1429             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1430             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
1431                     vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1432         }
1433         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, subnetId);
1434     }
1435
1436     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1437                                                   String interfaceName) {
1438         // This is either an extra-route (or) a learned IP via subnet-route
1439         List<String> nhList = null;
1440         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1441         if (nextHopIp == null || nextHopIp.isEmpty()) {
1442             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1443                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1444                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1445                     vpnName);
1446             nhList = Collections.emptyList();
1447         } else {
1448             nhList = Collections.singletonList(nextHopIp);
1449         }
1450         return nhList;
1451     }
1452
1453     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1454         VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker, vpnName, ipAddress);
1455         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1456         // else use connected interface
1457         if (gwPort != null && gwPort.isSubnetIp()) {
1458             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1459                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1460             return Optional.of(gwPort.getMacAddress());
1461         }
1462         return Optional.absent();
1463     }
1464
1465     @Override
1466     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1467         final VpnInterface update) {
1468         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}" ,update.getName(),
1469                 update.getDpnId(), original.getVpnInstanceNames(),
1470                 update.getVpnInstanceNames());
1471         final String vpnInterfaceName = update.getName();
1472         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1473         final Adjacencies origAdjs = original.getAugmentation(Adjacencies.class);
1474         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1475             != null ? origAdjs.getAdjacency() : new ArrayList<>();
1476         final Adjacencies updateAdjs = update.getAugmentation(Adjacencies.class);
1477         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1478             != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1479
1480         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1481         //handles switching between <internal VPN - external VPN>
1482         for (VpnInstanceNames vpnInterfaceVpnInstance : original.getVpnInstanceNames()) {
1483             String oldVpnName = vpnInterfaceVpnInstance.getVpnName();
1484             if (oldVpnName != null && (update.getVpnInstanceNames() == null
1485                 || !VpnHelper.doesVpnInterfaceBelongToVpnInstance(oldVpnName, update.getVpnInstanceNames()))) {
1486                 UpdateData updateData = new UpdateData(identifier, original, update);
1487                 vpnInterfacesUpdateQueue.add(updateData);
1488                 LOG.info("update: UpdateData on VPNInterface {} on dpn {} update upon VPN swap from oldVpn(s) {}"
1489                         + "to newVpn(s) {} added to update queue",
1490                         updateData.getOriginal().getName(), dpnId,
1491                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1492                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1493                 return;
1494             }
1495         }
1496         for (VpnInstanceNames vpnInterfaceVpnInstance : update.getVpnInstanceNames()) {
1497             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1498             String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName);
1499             if (newVpnName != null && (original.getVpnInstanceNames() == null
1500                 || !VpnHelper.doesVpnInterfaceBelongToVpnInstance(newVpnName,
1501                                 original.getVpnInstanceNames()))) {
1502                 if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
1503                     InstanceIdentifier<VpnInterfaceOpDataEntry> opIdentifier = VpnUtil
1504                         .getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1505                     if (canHandleNewVpnInterface(identifier, update, newVpnName)) {
1506                         List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1507                         List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1508                         addVpnInterfaceToVpn(opIdentifier, update, copyOldAdjs, copyNewAdjs, identifier, newVpnName);
1509                     }
1510                 } else {
1511                     UpdateData updateData = new UpdateData(identifier, original, update);
1512                     vpnInterfacesUpdateQueue.add(updateData);
1513                     LOG.info("update: UpdateData on VPNInterface {} on dpn {} update upon VPN swap from oldVpn(s) {}"
1514                             + "to newVpn(s) {} added to update queue",
1515                             updateData.getOriginal().getName(), dpnId,
1516                             VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1517                             VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1518                     return;
1519                 }
1520             }
1521         }
1522         for (VpnInstanceNames vpnInterfaceVpnInstance : update.getVpnInstanceNames()) {
1523             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1524             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1525             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1526             String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName);
1527             if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
1528                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName + newVpnName, () -> {
1529                     WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1530                     WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
1531                     InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1532                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1533                     LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven",
1534                             update.getName(), newVpnName);
1535                     //handle both addition and removal of adjacencies
1536                     //currently, new adjacency may be an extra route
1537                     boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, newVpnName);
1538                     if (!oldAdjs.equals(newAdjs)) {
1539                         for (Adjacency adj : copyNewAdjs) {
1540                             if (copyOldAdjs.contains(adj)) {
1541                                 copyOldAdjs.remove(adj);
1542                             } else {
1543                                 // add new adjacency - right now only extra route will hit this path
1544                                 if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker, adj)) {
1545                                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1546                                              dpnId, writeOperTxn, writeConfigTxn);
1547                                 }
1548                                 LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to vpn "
1549                                                 + "interface {} on vpn {} dpnId {}",
1550                                         adj.getIpAddress(), adj.getNextHopIpList(),
1551                                         adj.getLabel(), adj.getSubnetId(), update.getName(),
1552                                         newVpnName, dpnId);
1553                             }
1554                         }
1555                         for (Adjacency adj : copyOldAdjs) {
1556                             if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker, adj)) {
1557                                 delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1558                                        writeOperTxn, writeConfigTxn);
1559                             }
1560                             LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1561                                 + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1562                                 adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1563                         }
1564                     }
1565                     ListenableFuture<Void> operFuture = writeOperTxn.submit();
1566                     try {
1567                         operFuture.get();
1568                     } catch (ExecutionException e) {
1569                         LOG.error("Exception encountered while submitting operational future for update"
1570                                 + " VpnInterface {} on vpn {}: {}", vpnInterfaceName, newVpnName, e);
1571                         return null;
1572                     }
1573                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1574                     futures.add(writeConfigTxn.submit());
1575                     LOG.info("update: vpn interface updated for interface {} oldVpn(s) {} newVpn {}"
1576                         + "processed successfully", update.getName(),
1577                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()), newVpnName);
1578                     return futures;
1579                 });
1580             } else {
1581                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1582                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1583             }
1584         }
1585     }
1586
1587     @SuppressWarnings("checkstyle:IllegalCatch")
1588     private class VpnInterfaceUpdateTimerTask implements Runnable {
1589         private final Logger log = LoggerFactory.getLogger(VpnInterfaceUpdateTimerTask.class);
1590
1591         @Override
1592         public void run() {
1593             try {
1594                 doRun();
1595             } catch (ReadFailedException | RuntimeException e) {
1596                 LOG.error("VpnInterfaceUpdateTimerTask failed", e);
1597             }
1598         }
1599
1600         private void doRun() throws ReadFailedException {
1601             List<UpdateData> processQueue = new ArrayList<>();
1602             List<UpdateData> updateDataList = new ArrayList<>();
1603             vpnInterfacesUpdateQueue.drainTo(processQueue);
1604             int maxInterfaceList = 0;
1605
1606             for (UpdateData updData : processQueue) {
1607                 final VpnInterfaceKey key = updData.getIdentifier()
1608                     .firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1609                 final String interfaceName = key.getName();
1610                 Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1611                 for (VpnInstanceNames vpnInterfaceVpnInstance : updData.getOriginal().getVpnInstanceNames()) {
1612                     String oldVpnName = vpnInterfaceVpnInstance.getVpnName();
1613                     if (updData.getUpdate().getVpnInstanceNames() != null
1614                         && VpnHelper.doesVpnInterfaceBelongToVpnInstance(oldVpnName,
1615                                     updData.getUpdate().getVpnInstanceNames())) {
1616                         continue;
1617                     }
1618                     log.info("run: VPN Interface update event - intfName {} remove vpnName {} running"
1619                         + " config-driven swap removal", updData.getOriginal().getName(), oldVpnName);
1620                     maxInterfaceList ++;
1621                     removeVpnInterfaceCall(updData.getIdentifier(), updData.getOriginal(),
1622                                        oldVpnName, interfaceName);
1623                     log.info("run: Processed Remove for update on VPNInterface {} upon VPN swap from old vpn {}"
1624                             + " to newVpn(s) {}", updData.getOriginal().getName(),
1625                             oldVpnName, VpnHelper.getVpnInterfaceVpnInstanceNamesString(updData
1626                                        .getUpdate().getVpnInstanceNames()));
1627                 }
1628                 updateDataList.add(updData);
1629             }
1630             /* Decide the max-wait time based on number of VpnInterfaces.
1631             *  max-wait-time is num-of-interface * 4seconds (random choice).
1632             *  Every 2sec poll VpnToDpnList. If VpnInterface is removed ,
1633             *  remove it from vpnInterfaceList.
1634             */
1635             int maxWaitTime =
1636                 maxInterfaceList * (int) (VpnConstants.PER_INTERFACE_MAX_WAIT_TIME_IN_MILLISECONDS / 1000);
1637             int waitTime = 2;
1638             Iterator<UpdateData> updateDataIterator = updateDataList.iterator();
1639             UpdateData updateDataSet;
1640             while (waitTime < maxWaitTime) {
1641                 try {
1642                     Thread.sleep(2000); // sleep for 2sec
1643                 } catch (InterruptedException e) {
1644                     // Ignored
1645                 }
1646
1647                 while (updateDataIterator.hasNext()) {
1648                     boolean interfaceIsRemoved = true;
1649                     updateDataSet = updateDataIterator.next();
1650                     for (VpnInstanceNames vpnInterfaceVpnInstance : updateDataSet.getOriginal().getVpnInstanceNames()) {
1651                         String oldVpnName = vpnInterfaceVpnInstance.getVpnName();
1652                         if (oldVpnName != null && updateDataSet.getUpdate().getVpnInstanceNames() != null
1653                             && VpnHelper.doesVpnInterfaceBelongToVpnInstance(oldVpnName,
1654                                     updateDataSet.getUpdate().getVpnInstanceNames())) {
1655                             continue;
1656                         }
1657
1658                         boolean isPresent = isVpnIntfPresentInVpnToDpnList(updateDataSet.getOriginal(), oldVpnName);
1659                         if (isPresent) {
1660                             interfaceIsRemoved = false;
1661                         }
1662                     }
1663                     if (interfaceIsRemoved) {
1664                         updateDataIterator.remove();
1665                     }
1666                 }
1667                 if (updateDataList.isEmpty()) {
1668                     log.info("run: All VpnInterfaces are successfully removed from OLD VPN after time {}", waitTime);
1669                     break;
1670                 }
1671                 waitTime += 2; //Increment linearly by 2sec.
1672             }
1673
1674             if (updateDataList.size() > 0) {
1675                 log.error("run: VpnInterfacesList not removed from old Vpn even after waiting {}", waitTime);
1676             }
1677             for (UpdateData updData : processQueue) {
1678                 if (updateDataList.contains(updData)) {
1679                     log.error("run: Failed to swap VpnInterface {} from oldVpn {} to target VPN {}"
1680                              + "as it has not been cleaned up from the oldVpn", updData.getOriginal().getName(),
1681                              VpnHelper.getVpnInterfaceVpnInstanceNamesString(updData.getOriginal()
1682                                             .getVpnInstanceNames()),
1683                              VpnHelper.getVpnInterfaceVpnInstanceNamesString(updData.getUpdate()
1684                                             .getVpnInstanceNames()));
1685                     continue;
1686                 }
1687                 for (VpnInstanceNames vpnInterfaceVpnInstance : updData.getUpdate().getVpnInstanceNames()) {
1688                     String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1689                     if (updData.getOriginal().getVpnInstanceNames() != null
1690                         && VpnHelper.doesVpnInterfaceBelongToVpnInstance(newVpnName,
1691                                               updData.getOriginal().getVpnInstanceNames())) {
1692                         continue;
1693                     }
1694                     log.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven"
1695                            + " swap addition", updData.getUpdate().getName(), newVpnName);
1696                     final Adjacencies origAdjs = updData.getOriginal().getAugmentation(Adjacencies.class);
1697                     final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency() != null
1698                         ? origAdjs.getAdjacency() : new ArrayList<>();
1699                     final Adjacencies updateAdjs = updData.getUpdate().getAugmentation(Adjacencies.class);
1700                     final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency() != null
1701                         ? updateAdjs.getAdjacency() : new ArrayList<>();
1702
1703                     addVpnInterfaceCall(updData.getIdentifier(), updData.getUpdate(),
1704                                       oldAdjs, newAdjs, newVpnName);
1705                     log.info("run: Processed Add for update on VPNInterface {} from oldVpn(s) {} to newVpn {}"
1706                             + " upon VPN swap", updData.getUpdate().getName(),
1707                             VpnHelper.getVpnInterfaceVpnInstanceNamesString(updData.getOriginal()
1708                                        .getVpnInstanceNames()), newVpnName);
1709                 }
1710             }
1711         }
1712
1713         private boolean isVpnIntfPresentInVpnToDpnList(VpnInterface vpnInterface, String vpnName)
1714                 throws ReadFailedException {
1715             BigInteger dpnId = vpnInterface.getDpnId();
1716             String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
1717             log.trace("isVpnIntfPresentInVpnToDpnList: GOT rd {} for VpnInterface {}  VpnInstance {} ", rd ,
1718                     vpnInterface.getName(), vpnName);
1719
1720             if (rd == null) {
1721                 return false;
1722             }
1723
1724             Optional<VpnInstanceOpDataEntry> maybeVpnInstanceOpData = vpnInstanceOpDataEntryCache.get(
1725                     VpnUtil.getVpnInstanceOpDataIdentifier(rd));
1726             if (maybeVpnInstanceOpData.isPresent()) {
1727                 log.trace("isVpnIntfPresentInVpnToDpnList: GOT VpnInstanceOp {} for rd {} ",
1728                         maybeVpnInstanceOpData.get(), rd);
1729                 List<VpnToDpnList> dpnToVpns = maybeVpnInstanceOpData.get().getVpnToDpnList();
1730                 if (dpnToVpns != null) {
1731                     for (VpnToDpnList dpn : dpnToVpns) {
1732                         if (dpn.getDpnId().equals(dpnId)) {
1733                             return dpn.getVpnInterfaces().stream().anyMatch(
1734                                 vpnInterfaces -> vpnInterface.getName().equals(vpnInterfaces.getInterfaceName()));
1735                         }
1736                         log.info("isVpnIntfPresentInVpnToDpnList: VpnInterface {} not present in DpnId {} vpn {}",
1737                                 vpnInterface.getName(), dpn.getDpnId(), vpnName);
1738                     }
1739                 }
1740             }
1741
1742             return false;
1743         }
1744     }
1745
1746     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1747         Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1748         synchronized (label.toString().intern()) {
1749             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1750                     .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1751             Optional<LabelRouteInfo> opResult = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
1752             if (opResult.isPresent()) {
1753                 LabelRouteInfo labelRouteInfo =
1754                     new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1755                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid, labelRouteInfo);
1756             }
1757         }
1758         LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label, nextHopIpList);
1759     }
1760
1761     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1762         SubnetRoute route, WriteTransaction writeConfigTxn) {
1763
1764         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1765         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, null /* parentVpnRd */)
1766                 .addAugmentation(SubnetRoute.class, route).build();
1767         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1768         InstanceIdentifierBuilder<VrfTables> idBuilder =
1769             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1770         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1771         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1772         if (writeConfigTxn != null) {
1773             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1774         } else {
1775             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1776         }
1777         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1778                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1779     }
1780
1781     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1782                                            Adjacency adj, BigInteger dpnId, WriteTransaction writeOperTxn,
1783                                            WriteTransaction writeConfigTxn) {
1784
1785         Optional<VpnInterfaceOpDataEntry> optVpnInterface = VpnUtil.read(dataBroker,
1786                                                 LogicalDatastoreType.OPERATIONAL, identifier);
1787
1788         if (optVpnInterface.isPresent()) {
1789             VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1790             String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1791             String vpnName = currVpnIntf.getVpnInstanceName();
1792             VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
1793             InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1794             Optional<AdjacenciesOp> optAdjacencies = VpnUtil.read(dataBroker,
1795                                             LogicalDatastoreType.OPERATIONAL, adjPath);
1796             boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1797             VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1798             long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1799             VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1800             List<Adjacency> adjacencies;
1801             if (optAdjacencies.isPresent()) {
1802                 adjacencies = optAdjacencies.get().getAdjacency();
1803             } else {
1804                 // This code will be hit in case of first PNF adjacency
1805                 adjacencies = new ArrayList<>();
1806             }
1807             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1808             L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1809                     .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1810             Adjacency operationalAdjacency = null;
1811             if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()) {
1812                 RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
1813                         : RouteOrigin.STATIC;
1814                 String nh = adj.getNextHopIpList().get(0);
1815                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1816                 synchronized (vpnPrefixKey.intern()) {
1817                     java.util.Optional<String> rdToAllocate = VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1818                             dataBroker, vpnId, null, prefix, vpnName, nh, dpnId);
1819                     if (rdToAllocate.isPresent()) {
1820                         input.setRd(rdToAllocate.get());
1821                         operationalAdjacency = populator.createOperationalAdjacency(input);
1822                         int label = operationalAdjacency.getLabel().intValue();
1823                         vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1824                                 currVpnIntf.getVpnInstanceName(), l3vni, origin,
1825                                 currVpnIntf.getName(), operationalAdjacency, encapType, writeConfigTxn);
1826                         LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1827                                 + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1828                                 vpnName, label, currVpnIntf.getName(), dpnId);
1829                     } else {
1830                         LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}", vpnName,
1831                                 prefix);
1832                         return;
1833                     }
1834                     // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1835                     // Keeping the MPLS check for now.
1836                     if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1837                         final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1838                         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1839                                 VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1840                         vpnsToImportRoute.forEach(vpn -> {
1841                             if (vpn.getVrfId() != null) {
1842                                 VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1843                                         dataBroker, vpn.getVpnId(), vpnId, prefix,
1844                                         VpnUtil.getVpnName(dataBroker, vpn.getVpnId()), nh, dpnId)
1845                                         .ifPresent(
1846                                             rds -> vpnManager.addExtraRoute(
1847                                                     VpnUtil.getVpnName(dataBroker, vpn.getVpnId()),
1848                                                     adj.getIpAddress(), nh, rds,
1849                                                     currVpnIntf.getVpnInstanceName(),
1850                                                     l3vni, RouteOrigin.SELF_IMPORTED,
1851                                                     currVpnIntf.getName(), opAdjacency, encapType, writeConfigTxn));
1852                             }
1853                         });
1854                     }
1855                 }
1856             } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1857                 LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1858                         currVpnIntf.getName(), vpnName);
1859
1860                 String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1861
1862                 writeOperTxn.merge(
1863                         LogicalDatastoreType.OPERATIONAL,
1864                         VpnUtil.getPrefixToInterfaceIdentifier(VpnUtil.getVpnId(dataBroker,
1865                                 adj.getSubnetId().getValue()), prefix),
1866                         VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1867                                 adj.getSubnetId(), Prefixes.PrefixCue.PhysNetFunc), true);
1868
1869                 fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1870                         adj.getIpAddress(), Collections.emptyList(), null /* EncapType */, 0 /* label */, 0 /*l3vni*/,
1871                       null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1872
1873                 input.setRd(adj.getVrfId());
1874             }
1875             if (operationalAdjacency == null) {
1876                 operationalAdjacency = populator.createOperationalAdjacency(input);
1877             }
1878             adjacencies.add(operationalAdjacency);
1879             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1880             VpnInterfaceOpDataEntry newVpnIntf =
1881                 VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
1882                     aug, dpnId, currVpnIntf.isScheduledForRemove(), currVpnIntf.getLportTag(),
1883                         currVpnIntf.getGatewayMacAddress());
1884
1885             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1886         }
1887     }
1888
1889     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
1890         Subnets subnets = VpnUtil.getExternalSubnet(dataBroker, adj.getSubnetId());
1891         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
1892     }
1893
1894     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
1895             BigInteger dpnId, WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) {
1896         Optional<VpnInterfaceOpDataEntry> optVpnInterface = VpnUtil.read(dataBroker,
1897                                           LogicalDatastoreType.OPERATIONAL, identifier);
1898
1899         if (optVpnInterface.isPresent()) {
1900             VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1901
1902             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1903             Optional<AdjacenciesOp> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1904             if (optAdjacencies.isPresent()) {
1905                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1906
1907                 if (!adjacencies.isEmpty()) {
1908                     LOG.trace("delAdjFromVpnInterface: Adjacencies are " + adjacencies);
1909                     Iterator<Adjacency> adjIt = adjacencies.iterator();
1910                     while (adjIt.hasNext()) {
1911                         Adjacency adjElem = adjIt.next();
1912                         if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1913                             String rd = adjElem.getVrfId();
1914                             adjIt.remove();
1915
1916                             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1917                             VpnInterfaceOpDataEntry newVpnIntf = VpnUtil
1918                                     .getVpnInterfaceOpDataEntry(currVpnIntf.getName(),
1919                                     currVpnIntf.getVpnInstanceName(), aug, dpnId, currVpnIntf.isScheduledForRemove(),
1920                                             currVpnIntf.getLportTag(), currVpnIntf.getGatewayMacAddress());
1921
1922                             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1923                             if (adj.getNextHopIpList() != null) {
1924                                 for (String nh : adj.getNextHopIpList()) {
1925                                     deleteExtraRouteFromCurrentAndImportingVpns(
1926                                         currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
1927                                         currVpnIntf.getName(), writeConfigTxn);
1928                                 }
1929                             } else if (adj.isPhysNetworkFunc()) {
1930                                 LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet [}",
1931                                         adj.getIpAddress(), adj.getSubnetId());
1932                                 fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
1933                                         writeConfigTxn);
1934                             }
1935                             break;
1936                         }
1937
1938                     }
1939                 }
1940                 LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
1941                         dpnId, adj.getVrfId());
1942             } else {
1943                 LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
1944                         + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
1945             }
1946         }
1947     }
1948
1949     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
1950         String rd, String intfName, WriteTransaction writeConfigTxn) {
1951         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn);
1952         List<VpnInstanceOpDataEntry> vpnsToImportRoute = VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1953         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1954             String vpnRd = vpn.getVrfId();
1955             if (vpnRd != null) {
1956                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn);
1957             }
1958         }
1959     }
1960
1961     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1962         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1963             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1964             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1965     }
1966
1967     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1968         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1969             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1970     }
1971
1972     protected void addToNeutronRouterDpnsMap(String routerName, String vpnInterfaceName,
1973         WriteTransaction writeOperTxn) {
1974         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1975         if (dpId.equals(BigInteger.ZERO)) {
1976             LOG.error("addToNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
1977                     + " association model", vpnInterfaceName, routerName);
1978
1979             return;
1980         }
1981         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1982
1983         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1984             .OPERATIONAL, routerDpnListIdentifier);
1985         RouterInterfaces routerInterface =
1986             new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
1987                 vpnInterfaceName).build();
1988         if (optionalRouterDpnList.isPresent()) {
1989             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1990                 RouterInterfaces.class, new RouterInterfacesKey(vpnInterfaceName)), routerInterface, true);
1991         } else {
1992             RouterDpnListBuilder builder = new RouterDpnListBuilder();
1993             builder.setRouterId(routerName);
1994             DpnVpninterfacesListBuilder dpnVpnList = new DpnVpninterfacesListBuilder().setDpnId(dpId);
1995             builder.setDpnVpninterfacesList(Collections.singletonList(dpnVpnList.build()));
1996             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL,
1997                 getRouterId(routerName),
1998                 builder.build(), true);
1999         }
2000     }
2001
2002     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName,
2003         WriteTransaction writeOperTxn) {
2004         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
2005         if (dpId.equals(BigInteger.ZERO)) {
2006             LOG.error("removeFromNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
2007                     + " dissociation model", vpnInterfaceName, routerName);
2008
2009             return;
2010         }
2011         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
2012         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
2013             .OPERATIONAL, routerDpnListIdentifier);
2014         if (optionalRouterDpnList.isPresent()) {
2015             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
2016             RouterInterfaces routerInterface =
2017                 new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
2018                     vpnInterfaceName).build();
2019
2020             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
2021                 if (routerInterfaces.isEmpty()) {
2022                     if (writeOperTxn != null) {
2023                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
2024                     } else {
2025                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
2026                     }
2027                 } else {
2028                     if (writeOperTxn != null) {
2029                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
2030                             RouterInterfaces.class,
2031                             new RouterInterfacesKey(vpnInterfaceName)));
2032                     } else {
2033                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
2034                             routerDpnListIdentifier.child(
2035                                 RouterInterfaces.class,
2036                                 new RouterInterfacesKey(vpnInterfaceName)));
2037                     }
2038                 }
2039             }
2040         }
2041     }
2042
2043     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, BigInteger dpId,
2044         WriteTransaction writeOperTxn) {
2045         if (dpId.equals(BigInteger.ZERO)) {
2046             LOG.error("removeFromNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
2047                     + " dissociation model", vpnInterfaceName, routerName);
2048
2049             return;
2050         }
2051         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
2052         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
2053             .OPERATIONAL, routerDpnListIdentifier);
2054         if (optionalRouterDpnList.isPresent()) {
2055             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
2056             RouterInterfaces routerInterface =
2057                 new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
2058                     vpnInterfaceName).build();
2059             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
2060                 if (routerInterfaces.isEmpty()) {
2061                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
2062                 } else {
2063                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
2064                         RouterInterfaces.class,
2065                         new RouterInterfacesKey(vpnInterfaceName)));
2066                 }
2067             }
2068         }
2069     }
2070
2071     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2072                                                     WriteTransaction writeConfigTxn, String vpnName) {
2073         if (vpnInterface == null) {
2074             return;
2075         }
2076         List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
2077         if (adjs == null) {
2078             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2079                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2080             return;
2081         }
2082         for (Adjacency adj : adjs) {
2083             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2084                 String primaryInterfaceIp = adj.getIpAddress();
2085                 String macAddress = adj.getMacAddress();
2086                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2087
2088                 long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
2089                     VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2090
2091                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2092                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2093                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
2094                         routerInt, label, writeConfigTxn);
2095                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2096                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2097                         macAddress);
2098                 return;
2099             }
2100         }
2101         LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2102                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}", interfaceName,
2103                 primaryRd, vpnName);
2104     }
2105
2106     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
2107                                                     WriteTransaction writeConfigTxn, String vpnName) {
2108         Adjacencies adjs = vpnInterface.getAugmentation(Adjacencies.class);
2109         String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
2110         if (adjs != null) {
2111             List<Adjacency> adjsList = adjs.getAdjacency();
2112             for (Adjacency adj : adjsList) {
2113                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2114                     String primaryInterfaceIp = adj.getIpAddress();
2115                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2116                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
2117                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2118                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2119                     return;
2120                 }
2121             }
2122         } else {
2123             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2124                     vpnInterface.getName(), rd);
2125         }
2126     }
2127
2128     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2129         if (!canHandleNewVpnInterface(intefaceData.identifier, intefaceData.vpnInterface, vpnName)) {
2130             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
2131                   vpnName, intefaceData.vpnInterface.getName());
2132             return;
2133         }
2134         final VpnInterfaceKey key = intefaceData.identifier
2135                .firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
2136         final String interfaceName = key.getName();
2137         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2138                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2139         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2140                   intefaceData.identifier, vpnName);
2141         return;
2142     }
2143
2144     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2145                                               VpnInterface vpnInterface, String vpnName) {
2146         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2147                .get(vpnName);
2148         if (vpnInterfaces == null) {
2149             vpnInterfaces = new ConcurrentLinkedQueue<>();
2150         }
2151         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2152         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2153         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2154                 vpnInterface.getName(), vpnName);
2155     }
2156
2157     public boolean isVpnInstanceReady(String vpnInstanceName) {
2158         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnInstanceName);
2159         if (vpnRd == null) {
2160             return false;
2161         }
2162         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
2163
2164         return vpnInstanceOpDataEntry != null;
2165     }
2166
2167     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2168         synchronized (vpnInstanceName.intern()) {
2169             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2170                     unprocessedVpnInterfaces.get(vpnInstanceName);
2171             if (vpnInterfaces != null) {
2172                 while (!vpnInterfaces.isEmpty()) {
2173                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2174                     if (hasVpnInstanceCreatedSuccessfully) {
2175                         processSavedInterface(savedInterface, vpnInstanceName);
2176                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2177                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2178                     } else {
2179                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2180                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2181                     }
2182                 }
2183             } else {
2184                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2185             }
2186         }
2187     }
2188
2189     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2190             VpnInterface vpnInterface) {
2191         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2192             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2193                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2194             if (vpnInterfaces != null) {
2195                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2196                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2197                             + "unprocessed list", vpnInterface.getName(),
2198                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2199                 }
2200             } else {
2201                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2202                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2203             }
2204         }
2205     }
2206
2207     public void vpnInstanceIsReady(String vpnInstanceName) {
2208         processSavedInterfaces(vpnInstanceName, true);
2209     }
2210
2211     public void vpnInstanceFailed(String vpnInstanceName) {
2212         processSavedInterfaces(vpnInstanceName, false);
2213     }
2214
2215     private static class UnprocessedVpnInterfaceData {
2216         InstanceIdentifier<VpnInterface> identifier;
2217         VpnInterface vpnInterface;
2218
2219         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2220             this.identifier = identifier;
2221             this.vpnInterface = vpnInterface;
2222         }
2223
2224         @Override
2225         public int hashCode() {
2226             final int prime = 31;
2227             int result = 1;
2228             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2229             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2230             return result;
2231         }
2232
2233         @Override
2234         public boolean equals(Object obj) {
2235             if (this == obj) {
2236                 return true;
2237             }
2238             if (obj == null) {
2239                 return false;
2240             }
2241             if (getClass() != obj.getClass()) {
2242                 return false;
2243             }
2244             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2245             if (identifier == null) {
2246                 if (other.identifier != null) {
2247                     return false;
2248                 }
2249             } else if (!identifier.equals(other.identifier)) {
2250                 return false;
2251             }
2252             if (vpnInterface == null) {
2253                 if (other.vpnInterface != null) {
2254                     return false;
2255                 }
2256             } else if (!vpnInterface.equals(other.vpnInterface)) {
2257                 return false;
2258             }
2259             return true;
2260         }
2261     }
2262
2263     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2264         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
2265         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
2266         if (vpnInstanceOpData == null) {
2267             return;
2268         }
2269         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2270         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2271             return;
2272         }
2273         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2274         vpnToDpnLists.forEach(vpnToDpnList -> vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2275             InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2276                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2277             Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = VpnUtil.read(dataBroker,
2278                     LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2279             if (!vpnInterfaceOptional.isPresent()) {
2280                 return;
2281             }
2282             List<Adjacency> configVpnAdjacencies = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker,
2283                     vpnInterface.getInterfaceName());
2284             if (configVpnAdjacencies == null) {
2285                 LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2286                 return;
2287             }
2288             List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2289                     .getAugmentation(AdjacenciesOp.class).getAdjacency();
2290             // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2291             // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2292             // in operational DS. These unprocessed adjacencies will be handled below.
2293             // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in operational
2294             // DS are retrieved which is used to call addNewAdjToVpnInterface method.
2295             configVpnAdjacencies.stream()
2296                     .filter(adjacency -> operationVpnAdjacencies.stream()
2297                             .noneMatch(operationalAdjacency ->
2298                                     operationalAdjacency.getIpAddress().equals(adjacency.getIpAddress())))
2299                     .forEach(adjacency -> {
2300                         LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2301                         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName() + vpnName,
2302                             () -> {
2303                                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
2304                                 WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
2305                                 if (VpnUtil.isAdjacencyEligibleToVpn(dataBroker, adjacency, vpnName)) {
2306                                     addNewAdjToVpnInterface(existingVpnInterfaceId, primaryRd, adjacency,
2307                                             vpnInterfaceOptional.get().getDpnId(), writeConfigTxn, writeOperTxn);
2308                                     ListenableFuture<Void> operFuture = writeOperTxn.submit();
2309                                     try {
2310                                         operFuture.get();
2311                                     } catch (ExecutionException | InterruptedException e) {
2312                                         LOG.error("Exception encountered while submitting operational"
2313                                                 + " future for vpnInterface {}", vpnInterface, e);
2314                                     }
2315                                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2316                                     futures.add(writeConfigTxn.submit());
2317                                     return futures;
2318                                 }  else {
2319                                     return Collections.emptyList();
2320                                 }
2321                             });
2322                     });
2323         }));
2324     }
2325
2326     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2327         private final String interfaceName;
2328         private final boolean add;
2329         private final String txnDestination;
2330
2331         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2332             this.interfaceName = interfaceName;
2333             this.add = add;
2334             this.txnDestination = transactionDest;
2335         }
2336
2337         @Override
2338         public void onSuccess(Void voidObj) {
2339             if (add) {
2340                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2341                         interfaceName, txnDestination);
2342             } else {
2343                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2344             }
2345         }
2346
2347         @Override
2348         public void onFailure(Throwable throwable) {
2349             if (add) {
2350                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}"
2351                         + " with exception: {}", interfaceName, txnDestination, throwable);
2352             } else {
2353                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed with exception: {}", interfaceName,
2354                         throwable);
2355                 VpnUtil.unsetScheduledToRemoveForVpnInterface(dataBroker, interfaceName);
2356             }
2357         }
2358     }
2359 }