vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.collect.Iterators;
  13 import com.google.common.util.concurrent.FutureCallback;
  14 import com.google.common.util.concurrent.Futures;
  15 import com.google.common.util.concurrent.ListenableFuture;
  16 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  17 import java.math.BigInteger;
  18 import java.util.ArrayList;
  19 import java.util.Collections;
  20 import java.util.Iterator;
  21 import java.util.List;
  22 import java.util.Map;
  23 import java.util.Objects;
  24 import java.util.concurrent.BlockingQueue;
  25 import java.util.concurrent.ConcurrentHashMap;
  26 import java.util.concurrent.ConcurrentLinkedQueue;
  27 import java.util.concurrent.ExecutionException;
  28 import java.util.concurrent.Executors;
  29 import java.util.concurrent.LinkedBlockingQueue;
  30 import java.util.concurrent.ScheduledExecutorService;
  31 import java.util.concurrent.TimeUnit;
  32 import java.util.function.Consumer;
  33 import java.util.function.Predicate;
  34 import java.util.stream.Collectors;
  35 import javax.annotation.PostConstruct;
  36 import javax.annotation.PreDestroy;
  37 import javax.inject.Inject;
  38 import javax.inject.Singleton;
  39 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  40 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  41 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  42 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  43 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  44 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  46 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  47 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  48 import org.opendaylight.genius.mdsalutil.NwConstants;
  49 import org.opendaylight.genius.mdsalutil.cache.DataObjectCache;
  50 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  51 import org.opendaylight.infrautils.caches.CacheProvider;
  52 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  53 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  54 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  55 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  56 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  57 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  58 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  59 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  60 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  61 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  62 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  63 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  64 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  65 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  66 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  67 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  68 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  69 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  70 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListBuilder;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListBuilder;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfaces;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesBuilder;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesKey;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.subnetmaps.Subnetmap;
 115 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 116 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 117 import org.slf4j.Logger;
 118 import org.slf4j.LoggerFactory;
 119
 120 @Singleton
 121 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 122
 123     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 124     private static final int VPN_INF_UPDATE_TIMER_TASK_DELAY = 1000;
 125     private static final TimeUnit TIME_UNIT = TimeUnit.MILLISECONDS;
 126     private static final short DJC_MAX_RETRIES = 3;
 127
 128     private final DataBroker dataBroker;
 129     private final ManagedNewTransactionRunner txRunner;
 130     private final IBgpManager bgpManager;
 131     private final IFibManager fibManager;
 132     private final IMdsalApiManager mdsalManager;
 133     private final IdManagerService idManager;
 134     private final OdlInterfaceRpcService ifaceMgrRpcService;
 135     private final VpnFootprintService vpnFootprintService;
 136     private final IInterfaceManager interfaceManager;
 137     private final IVpnManager vpnManager;
 138     private final ArpResponderHandler arpResponderHandler;
 139     private final JobCoordinator jobCoordinator;
 140
 141     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 142
 143     private final BlockingQueue<UpdateData> vpnInterfacesUpdateQueue = new LinkedBlockingQueue<>();
 144     private final ScheduledExecutorService vpnInfUpdateTaskExecutor = Executors.newScheduledThreadPool(1);
 145
 146     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 147             new ConcurrentHashMap<>();
 148
 149     private final DataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 150
 151     @Inject
 152     public VpnInterfaceManager(final DataBroker dataBroker,
 153                                final IBgpManager bgpManager,
 154                                final IdManagerService idManager,
 155                                final IMdsalApiManager mdsalManager,
 156                                final IFibManager fibManager,
 157                                final OdlInterfaceRpcService ifaceMgrRpcService,
 158                                final VpnFootprintService vpnFootprintService,
 159                                final IInterfaceManager interfaceManager,
 160                                final IVpnManager vpnManager,
 161                                final ArpResponderHandler arpResponderHandler,
 162                                final JobCoordinator jobCoordinator,
 163                                final CacheProvider cacheProvider) {
 164         super(VpnInterface.class, VpnInterfaceManager.class);
 165
 166         this.dataBroker = dataBroker;
 167         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 168         this.bgpManager = bgpManager;
 169         this.idManager = idManager;
 170         this.mdsalManager = mdsalManager;
 171         this.fibManager = fibManager;
 172         this.ifaceMgrRpcService = ifaceMgrRpcService;
 173         this.vpnFootprintService = vpnFootprintService;
 174         this.interfaceManager = interfaceManager;
 175         this.vpnManager = vpnManager;
 176         this.arpResponderHandler = arpResponderHandler;
 177         this.jobCoordinator = jobCoordinator;
 178         vpnInfUpdateTaskExecutor.scheduleWithFixedDelay(new VpnInterfaceUpdateTimerTask(),
 179             0, VPN_INF_UPDATE_TIMER_TASK_DELAY, TIME_UNIT);
 180
 181         vpnInstanceOpDataEntryCache = new DataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 182                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 183                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 184     }
 185
 186     public Runnable isNotifyTaskQueued(String intfName) {
 187         return vpnIntfMap.remove(intfName);
 188     }
 189
 190     @PostConstruct
 191     public void start() {
 192         LOG.info("{} start", getClass().getSimpleName());
 193         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 194     }
 195
 196     @Override
 197     @PreDestroy
 198     public void close() {
 199         super.close();
 200         vpnInfUpdateTaskExecutor.shutdown();
 201         vpnInstanceOpDataEntryCache.close();
 202     }
 203
 204     @Override
 205     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 206         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 207     }
 208
 209     @Override
 210     protected VpnInterfaceManager getDataTreeChangeListener() {
 211         return VpnInterfaceManager.this;
 212     }
 213
 214     @Override
 215     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 216         LOG.info("add: intfName {} onto vpnName {}",
 217                  vpnInterface.getName(),
 218                  VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 219         addVpnInterface(identifier, vpnInterface, null, null);
 220     }
 221
 222     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 223                           final VpnInterface vpnInterface, String vpnName) {
 224         synchronized (vpnName.intern()) {
 225             if (isVpnInstanceReady(vpnName)) {
 226                 return true;
 227             }
 228             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 229             return false;
 230         }
 231     }
 232
 233     // TODO Clean up the exception handling
 234     @SuppressWarnings("checkstyle:IllegalCatch")
 235     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 236                              final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs) {
 237         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
 238             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 239             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 240         }
 241     }
 242
 243     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 244                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 245         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 246         final String interfaceName = key.getName();
 247
 248         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 249             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 250                   vpnName, vpnInterface.getName());
 251             return;
 252         }
 253         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 254              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 255         List<Adjacency> copyOldAdjs = null;
 256         if (oldAdjs != null) {
 257             copyOldAdjs = new ArrayList<>();
 258             copyOldAdjs.addAll(oldAdjs);
 259         }
 260         List<Adjacency> copyNewAdjs = null;
 261         if (newAdjs != null) {
 262             copyNewAdjs = new ArrayList<>();
 263             copyNewAdjs.addAll(newAdjs);
 264         }
 265         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 266     }
 267
 268     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 269                     final VpnInterface vpnInterface, final List<Adjacency> oldAdjs,
 270                     final List<Adjacency> newAdjs,
 271                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 272         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 273         final String interfaceName = key.getName();
 274         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 275         if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
 276             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 277             boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
 278             if (interfaceState != null) {
 279                 try {
 280                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 281                     final int ifIndex = interfaceState.getIfIndex();
 282                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName + vpnName, () -> {
 283                         WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 284                         WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
 285                         WriteTransaction writeInvTxn = dataBroker.newWriteOnlyTransaction();
 286                         LOG.info("addVpnInterface: VPN Interface add event - intfName {} vpnName {} on dpn {}" ,
 287                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 288                         processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false, writeConfigTxn,
 289                                      writeOperTxn, writeInvTxn, interfaceState, vpnName);
 290                         if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 291                             LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 292                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 293                             if (newAdjs != null) {
 294                                 for (Adjacency adj : newAdjs) {
 295                                     if (oldAdjs.contains(adj)) {
 296                                         oldAdjs.remove(adj);
 297                                     } else {
 298                                         if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(
 299                                                                   dataBroker, adj)) {
 300                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd,
 301                                                   adj, dpnId, writeOperTxn, writeConfigTxn);
 302                                         }
 303                                     }
 304                                 }
 305                             }
 306                             for (Adjacency adj : oldAdjs) {
 307                                 if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(
 308                                                           dataBroker, adj)) {
 309                                     delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 310                                         writeOperTxn, writeConfigTxn);
 311                                 }
 312                             }
 313                         }
 314                         ListenableFuture<Void> operFuture = writeOperTxn.submit();
 315                         try {
 316                             operFuture.get();
 317                         } catch (ExecutionException e) {
 318                             LOG.error("addVpnInterface: Exception encountered while submitting operational future for"
 319                                     + " addVpnInterface {} on vpn {}: {}", vpnInterface.getName(), vpnName, e);
 320                             return null;
 321                         }
 322                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 323                         ListenableFuture<Void> configFuture = writeConfigTxn.submit();
 324                         futures.add(configFuture);
 325                         Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"));
 326                         futures.add(writeInvTxn.submit());
 327                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 328                                 + " processed successfully", interfaceName, vpnName, dpnId);
 329                         return futures;
 330                     });
 331                 } catch (NumberFormatException | IllegalStateException e) {
 332                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 333                                     + "interface {}. Interface addition on vpn {} failed", interfaceName,
 334                                     vpnName, e);
 335                     return;
 336                 }
 337             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 338                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName() + vpnName,
 339                     () -> {
 340                         WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 341                         createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 342                                        writeConfigTxn, vpnName);
 343                         LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 344                                  vpnName, vpnInterface.getDpnId());
 345                         ListenableFuture<Void> futures = writeConfigTxn.submit();
 346                         String errorText = "addVpnInterfaceCall: Exception encountered while submitting writeConfigTxn"
 347                              + " for interface " + vpnInterface.getName() + " on vpn " + vpnName;
 348                         ListenableFutures.addErrorLogging(futures, LOG, errorText);
 349                         return Collections.singletonList(futures);
 350                     });
 351             } else {
 352                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 353                         + " is not available", interfaceName, vpnName);
 354             }
 355         } else {
 356             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 357                       + " as vpn is pending delete", interfaceName, vpnName,
 358                     vpnInterface.getDpnId());
 359         }
 360     }
 361
 362     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 363     // see comments below.
 364     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 365     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 366             final int lportTag, boolean isInterfaceUp,
 367             WriteTransaction writeConfigTxn,
 368             WriteTransaction writeOperTxn,
 369             WriteTransaction writeInvTxn,
 370             Interface interfaceState,
 371             final String vpnName) {
 372         final String interfaceName = vpnInterface.getName();
 373         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = VpnUtil.getVpnInterfaceOpDataEntry(dataBroker,
 374                                                          interfaceName, vpnName);
 375         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 376         boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
 377         if (!isInterfaceUp) {
 378             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 379                      interfaceName, dpId, vpnName);
 380             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 381             if (vpnId == VpnConstants.INVALID_ID) {
 382                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 383                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 384                         dpId);
 385                 return;
 386             }
 387
 388             boolean waitForVpnInterfaceOpRemoval = false;
 389             if (opVpnInterface != null && !opVpnInterface.isScheduledForRemove()) {
 390                 String opVpnName = opVpnInterface.getVpnInstanceName();
 391                 String primaryInterfaceIp = null;
 392                 if (opVpnName.equals(vpnName)) {
 393                     // Please check if the primary VRF Entry does not exist for VPNInterface
 394                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 395                     // back to back
 396                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 397                     List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
 398                     if (adjs == null) {
 399                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 400                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 401                                 vpnName);
 402                         return;
 403                     }
 404                     for (Adjacency adj : adjs) {
 405                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 406                             primaryInterfaceIp = adj.getIpAddress();
 407                             break;
 408                         }
 409                     }
 410                     if (primaryInterfaceIp == null) {
 411                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 412                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 413                                 dpId, vpnName);
 414                         return;
 415                     }
 416                     // Get the rd of the vpn instance
 417                     VrfEntry vrf = VpnUtil.getVrfEntry(dataBroker, primaryRd, primaryInterfaceIp);
 418                     if (vrf != null) {
 419                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 420                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 421                         return;
 422                     }
 423                     waitForVpnInterfaceOpRemoval = true;
 424                 } else {
 425                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 426                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 427                 }
 428             }
 429             if (!waitForVpnInterfaceOpRemoval) {
 430                 // Add the VPNInterface and quit
 431                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 432                         null/*ipAddressSourceValuePair*/,
 433                         true /* add */);
 434                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 435                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 436                 if (!isBgpVpnInternetVpn) {
 437                     VpnUtil.bindService(vpnName, interfaceName, dataBroker, false /*isTunnelInterface*/,
 438                             jobCoordinator);
 439                 }
 440                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 441                         dpId, vpnName);
 442                 if (interfaceManager.isExternalInterface(interfaceName)) {
 443                     processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
 444                         NwConstants.ADD_FLOW);
 445                 }
 446                 return;
 447             }
 448
 449             // FIB didn't get a chance yet to clean up this VPNInterface
 450             // Let us give it a chance here !
 451             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 452                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 453             try {
 454                 Runnable notifyTask = new VpnNotifyTask();
 455                 synchronized (notifyTask) {
 456                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 457                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 458                     // published here while holding the lock on it so this path will hit the wait before notify can be
 459                     // invoked.
 460                     vpnIntfMap.put(interfaceName, notifyTask);
 461                     try {
 462                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 463                     } catch (InterruptedException e) {
 464                         // Ignored
 465                     }
 466                 }
 467             } finally {
 468                 vpnIntfMap.remove(interfaceName);
 469             }
 470
 471             if (opVpnInterface != null) {
 472                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 473                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 474                         dpId, vpnName);
 475                 VpnUtil.unsetScheduledToRemoveForVpnInterface(dataBroker, interfaceName);
 476                 return;
 477             }
 478             // VPNInterface got removed, proceed with Add
 479             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 480                     interfaceName, dpId, vpnName);
 481             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 482                     null/*ipAddressSourceValuePair*/,
 483                     true /* add */);
 484             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 485                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 486             if (!isBgpVpnInternetVpn) {
 487                 VpnUtil.bindService(vpnName, interfaceName, dataBroker, false/*isTunnelInterface*/, jobCoordinator);
 488             }
 489             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 490                     + " FIB to clean up", interfaceName, dpId, vpnName);
 491             if (interfaceManager.isExternalInterface(interfaceName)) {
 492                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId,
 493                                lportTag, writeInvTxn, NwConstants.ADD_FLOW);
 494             }
 495         } else {
 496             // Interface is retained in the DPN, but its Link Up.
 497             // Advertise prefixes again for this interface to BGP
 498             InstanceIdentifier<VpnInterface> identifier =
 499                 VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 500             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 501                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 502             advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 503             // Perform similar operation as interface add event for extraroutes.
 504             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 505             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 506             if (!optAdjacencies.isPresent()) {
 507                 LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 508                 return;
 509             }
 510             List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
 511             for (Adjacency adjacency : adjacencies) {
 512                 if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 513                     continue;
 514                 }
 515                 // if BGPVPN Internet, filter only IPv6 Adjacencies
 516                 if (isBgpVpnInternetVpn && !VpnUtil.isAdjacencyEligibleToVpnInternet(
 517                                 dataBroker, adjacency)) {
 518                     continue;
 519                 }
 520                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 521                           dpId, writeOperTxn, writeConfigTxn);
 522             }
 523         }
 524     }
 525
 526     private void processExternalVpnInterface(String interfaceName, String vpnName, long vpnId, BigInteger dpId,
 527             int lportTag, WriteTransaction writeInvTxn, int addOrRemove) {
 528         Uuid extNetworkId;
 529         try {
 530             // vpn instance of ext-net interface is the network-id
 531             extNetworkId = new Uuid(vpnName);
 532         } catch (IllegalArgumentException e) {
 533             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 534                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 535             return;
 536         }
 537
 538         List<Uuid> routerIds = VpnUtil.getExternalNetworkRouterIds(dataBroker, extNetworkId);
 539         if (routerIds == null || routerIds.isEmpty()) {
 540             LOG.info("processExternalVpnInterface: No router is associated with {}."
 541                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 542                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 543             return;
 544         }
 545
 546         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 547                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 548         for (Uuid routerId : routerIds) {
 549             String routerName = routerId.getValue();
 550             BigInteger primarySwitch = VpnUtil.getPrimarySwitchForRouter(dataBroker, routerName);
 551             if (Objects.equals(primarySwitch, dpId)) {
 552                 Routers router = VpnUtil.getExternalRouter(dataBroker, routerName);
 553                 if (router != null) {
 554                     if (addOrRemove == NwConstants.ADD_FLOW) {
 555                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 556                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 557                                 dpId, vpnId, interfaceName, lportTag, writeInvTxn);
 558                     } else {
 559                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 560                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 561                                 dpId, interfaceName, lportTag, writeInvTxn);
 562                     }
 563                 } else {
 564                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 565                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 566                             interfaceName, dpId, vpnName);
 567                 }
 568             }
 569         }
 570     }
 571
 572     // TODO Clean up the exception handling
 573     @SuppressWarnings("checkstyle:IllegalCatch")
 574     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 575                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 576                                                   String vpnName, String interfaceName) {
 577         if (rd == null) {
 578             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 579                   interfaceName, dpnId, vpnName);
 580             return;
 581         } else {
 582             if (rd.equals(vpnName)) {
 583                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 584                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 585                 return;
 586             }
 587         }
 588         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 589                 interfaceName, dpnId, vpnName, rd);
 590
 591         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 592         if (nextHopIp == null) {
 593             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 594                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 595                     rd, vpnName);
 596             return;
 597         }
 598
 599         //Read NextHops
 600         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 601         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 602         if (adjacencies.isPresent()) {
 603             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 604
 605             if (!nextHops.isEmpty()) {
 606                 LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 607                         + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 608                 VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, rd);
 609                 long l3vni = vpnInstanceOpData.getL3vni();
 610                 VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 611                         ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 612                 for (Adjacency nextHop : nextHops) {
 613                     if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 614                         continue;
 615                     }
 616                     String gatewayMac = null;
 617                     long label = 0;
 618                     if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 619                         final VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker,
 620                             vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 621                         gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName).get();
 622                     } else {
 623                         label = nextHop.getLabel();
 624                     }
 625                     try {
 626                         LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 627                                 + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 628                         bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 629                                 encapType, (int)label, l3vni, 0 /*l2vni*/,
 630                                 gatewayMac);
 631                         LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 632                                 + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 633                                 nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 634                     } catch (Exception e) {
 635                         LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {} with rd {}"
 636                                 + " for interface {} on dpn {}", nextHop.getIpAddress(),
 637                                   vpnName, rd, interfaceName, dpnId, e);
 638                     }
 639                 }
 640             }
 641         }
 642     }
 643
 644     // TODO Clean up the exception handling
 645     @SuppressWarnings("checkstyle:IllegalCatch")
 646     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 647                         String vpnName, String interfaceName, WriteTransaction writeConfigTxn) {
 648         //Read NextHops
 649         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 650         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 651
 652         String rd = VpnUtil.getVpnRd(dataBroker, interfaceName);
 653         if (rd == null) {
 654             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 655                 interfaceName, vpnName);
 656             return;
 657         } else {
 658             if (rd.equals(vpnName)) {
 659                 LOG.info(
 660                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 661                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 662                 return;
 663             }
 664         }
 665         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 666                vpnName, rd);
 667         if (adjacencies.isPresent()) {
 668             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 669
 670             if (!nextHops.isEmpty()) {
 671                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 672                         nextHops, interfaceName, vpnName, rd);
 673                 for (Adjacency nextHop : nextHops) {
 674                     try {
 675                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 676                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 677                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 678                                     interfaceName, vpnName);
 679                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 680                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 681                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 682                                     interfaceName, vpnName);
 683                         } else {
 684                             // Perform similar operation as interface delete event for extraroutes.
 685                             String allocatedRd = nextHop.getVrfId();
 686                             for (String nh : nextHop.getNextHopIpList()) {
 687                                 deleteExtraRouteFromCurrentAndImportingVpns(
 688                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn);
 689                             }
 690                         }
 691                     } catch (Exception e) {
 692                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 693                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 694                     }
 695                 }
 696             }
 697         }
 698     }
 699
 700     @SuppressWarnings("checkstyle:IllegalCatch")
 701     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 702                                                   String primaryRd, String interfaceName, final long vpnId,
 703                                                   WriteTransaction writeConfigTxn,
 704                                                   WriteTransaction writeOperTxn,
 705                                                   final WriteTransaction writeInvTxn,
 706                                                   Interface interfaceState) {
 707         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 708         // Read NextHops
 709         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 710         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 711         if (!adjacencies.isPresent()) {
 712             addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 713                     null/*gwMac*/, writeOperTxn);
 714             return;
 715         }
 716
 717         // Get the rd of the vpn instance
 718         String nextHopIp = null;
 719         try {
 720             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 721         } catch (Exception e) {
 722             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 723                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 724         }
 725         List<String> nhList = new ArrayList<>();
 726         if (nextHopIp != null) {
 727             nhList.add(nextHopIp);
 728             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 729                     interfaceName, dpnId, vpnName, nhList);
 730         }
 731         Optional<String> gwMac = Optional.absent();
 732         String vpnInterfaceSubnetGwMacAddress = null;
 733         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
 734         Long l3vni = vpnInstanceOpData.getL3vni();
 735         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 736         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 737         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 738         List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 739         List<Adjacency> value = new ArrayList<>();
 740         for (Adjacency nextHop : nextHops) {
 741             String rd = primaryRd;
 742             Subnetmap sn = VpnUtil.getSubnetmapFromItsUuid(dataBroker, nextHop.getSubnetId());
 743             if (!VpnUtil.isSubnetPartOfVpn(sn, vpnName)) {
 744                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 745                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 746                 LOG.debug("processVpnInterfaceAdjacencies: Not Adding prefix {} to interface {}"
 747                       + " as the subnet is not part of vpn {}", prefix, interfaceName, vpnName);
 748                 continue;
 749             }
 750             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 751                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 752                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 753                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 754                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 755                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 756                 writeOperTxn.merge(
 757                     LogicalDatastoreType.OPERATIONAL,
 758                     VpnUtil.getPrefixToInterfaceIdentifier(
 759                         VpnUtil.getVpnId(dataBroker, vpnName), prefix),
 760                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, nextHop.getSubnetId(),
 761                             prefixCue), true);
 762                 final Uuid subnetId = nextHop.getSubnetId();
 763                 final Optional<String> gatewayIp = VpnUtil.getVpnSubnetGatewayIp(dataBroker, subnetId);
 764                 if (gatewayIp.isPresent()) {
 765                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp.get());
 766                     if (gwMac.isPresent()) {
 767                         // A valid mac-address is available for this subnet-gateway-ip
 768                         // Use this for programming ARP_RESPONDER table here.  And save this
 769                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 770                         // to populate L3_GW_MAC_TABLE there.
 771                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, vpnName, vpnId, interfaceName,
 772                                 subnetId, gatewayIp.get(), gwMac.get());
 773                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 774                     } else {
 775                         // A valid mac-address is not available for this subnet-gateway-ip
 776                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 777                         // Save this connected-mac-address as gateway-mac-address for the
 778                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 779                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 780                         if (gwMac.isPresent()) {
 781                             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
 782                                     vpnId, writeInvTxn, NwConstants.ADD_FLOW, gwMac.get());
 783                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, vpnName, vpnId, interfaceName,
 784                                     subnetId, gatewayIp.get(), gwMac.get());
 785                         } else {
 786                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 787                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 788                                 + "gwIp {}",
 789                                 interfaceName, vpnName, gatewayIp.get());
 790                         }
 791                     }
 792                 } else {
 793                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 794                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 795                         subnetId, interfaceName, vpnName);
 796                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 797                 }
 798                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 799                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 800             } else {
 801                 //Extra route adjacency
 802                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 803                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 804                 synchronized (vpnPrefixKey.intern()) {
 805                     java.util.Optional<String> rdToAllocate = VpnUtil
 806                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(dataBroker, vpnId, null,
 807                             prefix, vpnName, nextHop.getNextHopIpList().get(0), dpnId, writeOperTxn);
 808                     if (rdToAllocate.isPresent()) {
 809                         rd = rdToAllocate.get();
 810                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 811                             rd, prefix);
 812                     } else {
 813                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 814                         continue;
 815                     }
 816                 }
 817                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 818                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 819                         interfaceName, dpnId);
 820             }
 821             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 822             // can be different from the gateway-mac-address within the VRFEntry as the
 823             // gateway-mac-address is a superset.
 824             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 825                     : RouteOrigin.STATIC;
 826             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 827                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 828                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 829             Adjacency operationalAdjacency = null;
 830             try {
 831                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 832             } catch (NullPointerException e) {
 833                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 834                     input, e.getMessage());
 835                 return;
 836             }
 837             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 838                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 839                         vpnName, operationalAdjacency.getLabel().intValue(), l3vni, origin,
 840                         interfaceName, operationalAdjacency, encapType, writeConfigTxn);
 841             }
 842             value.add(operationalAdjacency);
 843         }
 844
 845         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 846         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 847                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 848
 849         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 850                 .setGatewayMac(gwMac.isPresent() ? gwMac.get() : null).setInterfaceName(interfaceName)
 851                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 852
 853         for (Adjacency nextHop : aug.getAdjacency()) {
 854             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 855             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 856                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 857                         : RouteOrigin.STATIC;
 858                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 859                 registeredPopulator.populateFib(input, writeConfigTxn, writeOperTxn);
 860             }
 861         }
 862     }
 863
 864     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 865                                               long lportTag, String gwMac, WriteTransaction writeOperTxn) {
 866         VpnInterfaceOpDataEntry opInterface =
 867               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, Boolean.FALSE, lportTag, gwMac);
 868         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 869             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 870         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 871                 WriteTransaction.CREATE_MISSING_PARENTS);
 872         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 873                 interfaceName, dpnId, vpnName);
 874     }
 875
 876     // TODO Clean up the exception handling
 877     @SuppressWarnings("checkstyle:IllegalCatch")
 878     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 879                                            StateTunnelList stateTunnelList,
 880                                            WriteTransaction writeConfigTxn,
 881                                            WriteTransaction writeOperTxn) {
 882
 883         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
 884         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 885         AdjacenciesOp adjacencies = vpnInterface.getAugmentation(AdjacenciesOp.class);
 886         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 887         if (adjList.isEmpty()) {
 888             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 889                     vpnInterface, srcDpnId);
 890             return;
 891         }
 892         String prefix = null;
 893         long label = 0;
 894         List<Adjacency> value = new ArrayList<>();
 895         boolean isNextHopAddReqd = false;
 896         String vpnName = vpnInterface.getVpnInstanceName();
 897         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 898         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 899         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 900                 vpnInterface.getName(), vpnInterface.getDpnId(),
 901                 vpnInterface.getVpnInstanceName(), adjList);
 902         for (Adjacency adj : adjList) {
 903             String rd = adj.getVrfId();
 904             rd = rd != null ? rd : vpnName;
 905             prefix = adj.getIpAddress();
 906             label = adj.getLabel();
 907             List<String> nhList = Collections.singletonList(srcTepIp);
 908             List<String> nextHopList = adj.getNextHopIpList();
 909             // If TEP is added , update the nexthop of primary adjacency.
 910             // Secondary adj nexthop is already pointing to primary adj IP address.
 911             if (nextHopList != null && !nextHopList.isEmpty() && nextHopList.get(0).equalsIgnoreCase(srcTepIp)) {
 912                 /* everything right already */
 913             } else {
 914                 isNextHopAddReqd = true;
 915             }
 916
 917             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 918                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 919             } else {
 920                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 921                 if (!vrfEntryOptional.isPresent()) {
 922                     continue;
 923                 }
 924                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 925                 if (!nhList.contains(srcTepIp)) {
 926                     nhList.add(srcTepIp);
 927                     isNextHopAddReqd = true;
 928                 }
 929                 value.add(adj);
 930             }
 931
 932             if (isNextHopAddReqd) {
 933                 updateLabelMapper(label, nhList);
 934                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 935                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 936                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 937                 // Update the VRF entry with nextHop
 938                 fibManager.updateRoutePathForFibEntry(dataBroker, primaryRd, prefix, srcTepIp,
 939                         label, true, writeConfigTxn);
 940
 941                 //Get the list of VPN's importing this route(prefix) .
 942                 // Then update the VRF entry with nhList
 943                 List<VpnInstanceOpDataEntry> vpnsToImportRoute =
 944                            VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
 945                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 946                     String vpnRd = vpn.getVrfId();
 947                     if (vpnRd != null) {
 948                         fibManager.updateRoutePathForFibEntry(dataBroker, vpnRd, prefix,
 949                                 srcTepIp, label, true, writeConfigTxn);
 950                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 951                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
 952                             vpnInterface.getName(), srcDpnId, vpnName,
 953                             vpn.getVpnInstanceName(), vpnRd);
 954                     }
 955                 }
 956                 // Advertise the prefix to BGP only for external vpn
 957                 // since there is a nexthop change.
 958                 try {
 959                     if (!rd.equalsIgnoreCase(vpnName)) {
 960                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
 961                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
 962                                 null /*gatewayMacAddress*/);
 963                     }
 964                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
 965                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
 966                             srcDpnId, vpnName);
 967                 } catch (Exception ex) {
 968                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
 969                             + " on rd {} for interface {} on dpn {} vpn {} as {}", prefix, nhList, label, rd,
 970                             vpnInterface.getName(), srcDpnId,
 971                             vpnName, ex);
 972                 }
 973             }
 974         }
 975         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 976         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
 977                 .setKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
 978                 .addAugmentation(AdjacenciesOp.class, aug).build();
 979         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
 980                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
 981         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 982                 WriteTransaction.CREATE_MISSING_PARENTS);
 983         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
 984                 vpnInterface.getName(), srcDpnId, vpnName);
 985
 986     }
 987
 988     // TODO Clean up the exception handling
 989     @SuppressWarnings("checkstyle:IllegalCatch")
 990     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
 991                                               StateTunnelList stateTunnelList,
 992                                               WriteTransaction writeConfigTxn,
 993                                               WriteTransaction writeOperTxn) {
 994
 995         AdjacenciesOp adjacencies = vpnInterface.getAugmentation(AdjacenciesOp.class);
 996         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 997         String prefix = null;
 998         long label = 0;
 999         boolean isNextHopRemoveReqd = false;
1000         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
1001         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
1002         String vpnName = vpnInterface.getVpnInstanceName();
1003         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1004         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1005         if (adjList != null) {
1006             List<Adjacency> value = new ArrayList<>();
1007             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1008                     vpnInterface.getName(), vpnInterface.getDpnId(),
1009                     vpnInterface.getVpnInstanceName(), adjList);
1010             for (Adjacency adj : adjList) {
1011                 List<String> nhList = new ArrayList<>();
1012                 String rd = adj.getVrfId();
1013                 rd = rd != null ? rd : vpnName;
1014                 prefix = adj.getIpAddress();
1015                 List<String> nextHopList = adj.getNextHopIpList();
1016                 label = adj.getLabel();
1017                 if (nextHopList != null && !nextHopList.isEmpty()) {
1018                     isNextHopRemoveReqd = true;
1019                 }
1020                 // If TEP is deleted , remove the nexthop from primary adjacency.
1021                 // Secondary adj nexthop will continue to point to primary adj IP address.
1022                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1023                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1024                 } else {
1025                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1026                     if (!vrfEntryOptional.isPresent()) {
1027                         continue;
1028                     }
1029                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1030                     if (nhList.contains(srcTepIp)) {
1031                         nhList.remove(srcTepIp);
1032                         isNextHopRemoveReqd = true;
1033                     }
1034                     value.add(adj);
1035                 }
1036
1037                 if (isNextHopRemoveReqd) {
1038                     updateLabelMapper(label, nhList);
1039                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1040                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1041                             prefix, nhList, vpnName,
1042                             vpnId, rd, vpnInterface.getName());
1043                     // Update the VRF entry with removed nextHop
1044                     fibManager.updateRoutePathForFibEntry(dataBroker, primaryRd, prefix, srcTepIp,
1045                             label, false, writeConfigTxn);
1046
1047                     //Get the list of VPN's importing this route(prefix) .
1048                     // Then update the VRF entry with nhList
1049                     List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1050                           VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1051                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1052                         String vpnRd = vpn.getVrfId();
1053                         if (vpnRd != null) {
1054                             fibManager.updateRoutePathForFibEntry(dataBroker, vpnRd, prefix,
1055                                     srcTepIp, label, false, writeConfigTxn);
1056                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1057                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1058                                     nhList, label, vpnInterface.getName(), srcDpnId,
1059                                     vpnName,
1060                                     vpn.getVpnInstanceName(), vpnRd);
1061                         }
1062                     }
1063
1064                     // Withdraw prefix from BGP only for external vpn.
1065                     try {
1066                         if (!rd.equalsIgnoreCase(vpnName)) {
1067                             bgpManager.withdrawPrefix(rd, prefix);
1068                         }
1069                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1070                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1071                                 vpnInterface.getName(), srcDpnId,
1072                                 vpnName);
1073                     } catch (Exception ex) {
1074                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1075                                 + " on rd {} for interface {} on dpn {} vpn {} as {}", prefix, nhList, label, rd,
1076                                 vpnInterface.getName(), srcDpnId,
1077                                 vpnName, ex);
1078                     }
1079                 }
1080             }
1081             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1082             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1083                     .setKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1084                     .addAugmentation(AdjacenciesOp.class, aug).build();
1085             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1086                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1087             writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
1088                     WriteTransaction.CREATE_MISSING_PARENTS);
1089             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1090                          vpnInterface.getName(), srcDpnId, vpnName);
1091         }
1092     }
1093
1094     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1095         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1096
1097         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1098         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
1099         if (vpnInstanceOpDataEntry == null) {
1100             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1101                     + " to check for vpns exporting the routes", vpnName);
1102             return vpnsToExportRoute;
1103         }
1104
1105         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1106             if (input.getVpnInstanceName() == null) {
1107                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1108                         input.getVrfId());
1109                 return false;
1110             }
1111             return !input.getVpnInstanceName().equals(vpnName);
1112         };
1113
1114         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1115             Iterable<String> commonRTs =
1116                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1117                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1118             return Iterators.size(commonRTs.iterator()) > 0;
1119         };
1120
1121         vpnsToExportRoute =
1122                 VpnUtil.getAllVpnInstanceOpData(dataBroker).stream().filter(excludeVpn).filter(matchRTs).collect(
1123                         Collectors.toList());
1124         return vpnsToExportRoute;
1125     }
1126
1127     // TODO Clean up the exception handling
1128     @SuppressWarnings("checkstyle:IllegalCatch")
1129     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1130         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1131         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1132             List<VrfEntry> vrfEntries = VpnUtil.getAllVrfEntries(dataBroker, vpn.getVrfId());
1133             WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1134             if (vrfEntries != null) {
1135                 for (VrfEntry vrfEntry : vrfEntries) {
1136                     try {
1137                         if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1138                                 RouteOrigin.value(vrfEntry.getOrigin()))) {
1139                             LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1140                                     + " is not a controller managed non intervpn link route. Ignoring.",
1141                                     vpn.getVrfId(), vrfEntry.getDestPrefix());
1142                             continue;
1143                         }
1144                         String prefix = vrfEntry.getDestPrefix();
1145                         String gwMac = vrfEntry.getGatewayMacAddress();
1146                         vrfEntry.getRoutePaths().forEach(routePath -> {
1147                             String nh = routePath.getNexthopAddress();
1148                             int label = routePath.getLabel().intValue();
1149                             if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1150                                     vrfEntry.getOrigin()))) {
1151                                 LOG.info("handleVpnsExportingRoutesImporting: Importing fib entry rd {} prefix {}"
1152                                         + " nexthop {} label {} to vpn {} vpnRd {}", vpn.getVrfId(), prefix, nh, label,
1153                                         vpnName, vpnRd);
1154                                 fibManager.addOrUpdateFibEntry(dataBroker, vpnRd, null /*macAddress*/, prefix,
1155                                         Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1156                                         0 /*l3vni*/, gwMac,  null /*parentVpnRd*/, RouteOrigin.SELF_IMPORTED,
1157                                         writeConfigTxn);
1158                             } else {
1159                                 LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry rd {} prefix {}"
1160                                         + " nexthop {} label {} to vpn {} vpnRd {}", vpn.getVrfId(), prefix, nh, label,
1161                                         vpnName, vpnRd);
1162                                 SubnetRoute route = vrfEntry.getAugmentation(SubnetRoute.class);
1163                                 importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, writeConfigTxn);
1164                             }
1165                         });
1166                     } catch (RuntimeException e) {
1167                         LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1168                                 + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1169                                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1170                     }
1171                 }
1172                 writeConfigTxn.submit();
1173             } else {
1174                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1175                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1176             }
1177         }
1178     }
1179
1180     @SuppressWarnings("checkstyle:IllegalCatch")
1181     @Override
1182     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1183         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1184         final String interfaceName = key.getName();
1185         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
1186             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1187             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1188         }
1189     }
1190
1191     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1192                                 final VpnInterface vpnInterface, final String vpnName,
1193                                 final String interfaceName) {
1194         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1195             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName() + vpnName, () -> {
1196                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1197                 deleteFibEntryForRouterInterface(vpnInterface, writeConfigTxn, vpnName);
1198                 LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1199                 ListenableFuture<Void> futures = writeConfigTxn.submit();
1200                 String errorText = "removeVpnInterfaceCall: Exception encountered while submitting writeConfigTxn"
1201                         + " for interface " + vpnInterface.getName() + " on vpn " + vpnName;
1202                 ListenableFutures.addErrorLogging(futures, LOG, errorText);
1203                 return Collections.singletonList(futures);
1204             }, DJC_MAX_RETRIES);
1205         } else {
1206             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1207             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1208         }
1209     }
1210
1211     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1212     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1213                                 final VpnInterface vpnInterface, final String vpnName,
1214                                 final String interfaceName, final Interface interfaceState) {
1215         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1216                 vpnName, vpnInterface.getDpnId());
1217         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1218         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName + vpnName,
1219             () -> {
1220                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1221                 ListenableFuture<Void> configFuture = txRunner
1222                         .callWithNewWriteOnlyTransactionAndSubmit(writeConfigTxn -> {
1223                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeOperTxn -> {
1224                                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeInvTxn -> {
1225                                     LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1226                                             interfaceName, vpnName);
1227                                     BigInteger dpId = BigInteger.ZERO;
1228                                     int ifIndex = 0;
1229                                     String gwMacAddress = null;
1230                                     InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1231                                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1232                                     final Optional<VpnInterfaceOpDataEntry> optVpnInterface =
1233                                             VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, interfaceId);
1234                                     if (interfaceState != null) {
1235                                         try {
1236                                             dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1237                                         } catch (NumberFormatException | IllegalStateException e) {
1238                                             LOG.error("remove: Unable to retrieve dpnId from interface operational"
1239                                                     + " data store for interface {} on dpn {} for vpn {} Fetching"
1240                                                             + " from vpn interface op data store. ", interfaceName,
1241                                                     vpnInterface.getDpnId(), vpnName, e);
1242                                             dpId = BigInteger.ZERO;
1243                                         }
1244                                         ifIndex = interfaceState.getIfIndex();
1245                                         gwMacAddress = interfaceState.getPhysAddress().getValue();
1246                                     } else {
1247                                         LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1248                                                 + " from vpn interface op.", interfaceName);
1249                                         if (optVpnInterface.isPresent()) {
1250                                             VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1251                                             dpId = vpnOpInterface.getDpnId();
1252                                             ifIndex = vpnOpInterface.getLportTag().intValue();
1253                                             gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1254                                         } else {
1255                                             LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1256                                                             + " as interfaceState and vpn interface op is not"
1257                                                     + " available", interfaceName, vpnName);
1258                                             return;
1259                                         }
1260                                     }
1261                                     processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1262                                             optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1263                                             writeConfigTxn, writeOperTxn, writeInvTxn);
1264                                     LOG.info(
1265                                             "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1266                                                     + "successfully",
1267                                             interfaceName, vpnInterface.getDpnId(), vpnName);
1268                                 }));
1269                             }));
1270                         });
1271                 futures.add(configFuture);
1272                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1273                 return futures;
1274             }, DJC_MAX_RETRIES);
1275     }
1276
1277     protected void processVpnInterfaceDown(BigInteger dpId,
1278                                            String interfaceName,
1279                                            int lportTag,
1280                                            String gwMac,
1281                                            VpnInterfaceOpDataEntry vpnOpInterface,
1282                                            boolean isInterfaceStateDown,
1283                                            WriteTransaction writeConfigTxn,
1284                                            WriteTransaction writeOperTxn,
1285                                            WriteTransaction writeInvTxn) {
1286         if (vpnOpInterface == null) {
1287             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1288                     + " as it is not available in operational data store", interfaceName, dpId);
1289             return;
1290         }
1291         final String vpnName = vpnOpInterface.getVpnInstanceName();
1292         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1293                                                     interfaceName, vpnName);
1294         if (!isInterfaceStateDown) {
1295             final long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1296             VpnUtil.scheduleVpnInterfaceForRemoval(dataBroker, interfaceName, dpId, vpnName, Boolean.TRUE,
1297                     null);
1298             final boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
1299             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1300                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1301             if (interfaceManager.isExternalInterface(interfaceName)) {
1302                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
1303                         NwConstants.DEL_FLOW);
1304             }
1305             if (!isBgpVpnInternetVpn) {
1306                 VpnUtil.unbindService(dataBroker, interfaceName, isInterfaceStateDown, jobCoordinator);
1307             }
1308             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1309                     + " successful", interfaceName, dpId, vpnName);
1310         } else {
1311             // Interface is retained in the DPN, but its Link Down.
1312             // Only withdraw the prefixes for this interface from BGP
1313             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn);
1314         }
1315     }
1316
1317     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1318                                           final String vpnName, final long vpnId, String gwMac,
1319                                           WriteTransaction writeConfigTxn, final WriteTransaction writeOperTxn,
1320                                           final WriteTransaction writeInvTxn) {
1321         //Read NextHops
1322         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1323                 .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1324         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1325         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1326
1327         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1328         LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1329                 interfaceName, dpnId, vpnName, primaryRd);
1330         if (adjacencies.isPresent() && !adjacencies.get().getAdjacency().isEmpty()) {
1331             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1332             LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1333                     interfaceName, dpnId, vpnName, nextHops);
1334             for (Adjacency nextHop : nextHops) {
1335                 if (nextHop.isPhysNetworkFunc()) {
1336                     LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1337                             nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1338                     fibManager.removeFibEntry(dataBroker, nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1339                             null/*writeCfgTxn*/);
1340                 } else {
1341                     String rd = nextHop.getVrfId();
1342                     List<String> nhList;
1343                     if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1344                         nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1345                     } else {
1346                         // This is a primary adjacency
1347                         nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1348                                 : Collections.emptyList();
1349                         removeGwMacAndArpResponderFlows(nextHop, vpnId, vpnName, dpnId, lportTag, gwMac,
1350                                 interfaceName, writeInvTxn);
1351                     }
1352                     if (!nhList.isEmpty()) {
1353                         if (rd.equals(vpnName)) {
1354                             //this is an internal vpn - the rd is assigned to the vpn instance name;
1355                             //remove from FIB directly
1356                             nhList.forEach((nh) -> removeAdjacencyFromInternalVpn(nextHop, vpnName,
1357                                     interfaceName, dpnId, writeConfigTxn));
1358                         } else {
1359                             removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd, interfaceName,
1360                                     writeConfigTxn);
1361                         }
1362                     } else {
1363                         LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1364                                         + " interface {}", nextHop.getIpAddress(), rd,
1365                                 nextHop.getAdjacencyType().toString(), interfaceName);
1366                         bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
1367                         fibManager.removeFibEntry(dataBroker, primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1368                     }
1369                 }
1370                 String ip = nextHop.getIpAddress().split("/")[0];
1371                 LearntVpnVipToPort vpnVipToPort = VpnUtil.getLearntVpnVipToPort(dataBroker, vpnName, ip);
1372                 if (vpnVipToPort != null) {
1373                     VpnUtil.removeLearntVpnVipToPort(dataBroker, vpnName, ip);
1374                     LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed adjacency for Interface {}"
1375                                     + " ip {} on dpn {} for vpn {} from VpnPortData Entry", vpnVipToPort.getPortName(),
1376                             ip, dpnId, vpnName);
1377                 }
1378             }
1379         } else {
1380             // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1381             LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1382                     + " Removing it.", interfaceName, vpnName, dpnId);
1383             writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, identifier);
1384         }
1385     }
1386
1387     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1388                                                                    String interfaceName, BigInteger dpnId,
1389                                                                    WriteTransaction writeConfigTxn) {
1390         return (nh) -> {
1391             fibManager.removeOrUpdateFibEntry(dataBroker, vpnName, nextHop.getIpAddress(), nh,
1392                     writeConfigTxn);
1393             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1394                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1395                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1396         };
1397     }
1398
1399     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1400                                            BigInteger dpnId, String rd, String interfaceName,
1401                                            WriteTransaction writeConfigTxn) {
1402         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1403                 VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1404         nhList.forEach((nh) -> {
1405             //IRT: remove routes from other vpns importing it
1406             vpnManager.removePrefixFromBGP(primaryRd, rd, vpnName, nextHop.getIpAddress(),
1407                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn);
1408             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1409                 String vpnRd = vpn.getVrfId();
1410                 if (vpnRd != null) {
1411                     fibManager.removeOrUpdateFibEntry(dataBroker, vpnRd,
1412                             nextHop.getIpAddress(), nh, writeConfigTxn);
1413                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1414                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1415                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1416                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1417                 }
1418             }
1419         });
1420     }
1421
1422     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, String vpnName, BigInteger dpnId,
1423                                                  int lportTag, String gwMac, String interfaceName,
1424                                                  WriteTransaction writeInvTxn) {
1425         final Uuid subnetId = nextHop.getSubnetId();
1426         if (nextHop.getSubnetGatewayMacAddress() == null) {
1427             // A valid mac-address was not available for this subnet-gateway-ip
1428             // So a connected-mac-address was used for this subnet and we need
1429             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1430             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
1431                     vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1432         }
1433         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, vpnName, vpnId,
1434                 subnetId);
1435     }
1436
1437     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1438                                                   String interfaceName) {
1439         // This is either an extra-route (or) a learned IP via subnet-route
1440         List<String> nhList = null;
1441         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1442         if (nextHopIp == null || nextHopIp.isEmpty()) {
1443             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1444                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1445                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1446                     vpnName);
1447             nhList = Collections.emptyList();
1448         } else {
1449             nhList = Collections.singletonList(nextHopIp);
1450         }
1451         return nhList;
1452     }
1453
1454     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1455         VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker, vpnName, ipAddress);
1456         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1457         // else use connected interface
1458         if (gwPort != null && gwPort.isSubnetIp()) {
1459             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1460                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1461             return Optional.of(gwPort.getMacAddress());
1462         }
1463         return Optional.absent();
1464     }
1465
1466     @Override
1467     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1468         final VpnInterface update) {
1469         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}" ,update.getName(),
1470                 update.getDpnId(), original.getVpnInstanceNames(),
1471                 update.getVpnInstanceNames());
1472         final String vpnInterfaceName = update.getName();
1473         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1474         final Adjacencies origAdjs = original.getAugmentation(Adjacencies.class);
1475         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1476             != null ? origAdjs.getAdjacency() : new ArrayList<>();
1477         final Adjacencies updateAdjs = update.getAugmentation(Adjacencies.class);
1478         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1479             != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1480
1481         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1482         //handles switching between <internal VPN - external VPN>
1483         for (VpnInstanceNames vpnInterfaceVpnInstance : original.getVpnInstanceNames()) {
1484             String oldVpnName = vpnInterfaceVpnInstance.getVpnName();
1485             if (oldVpnName != null && (update.getVpnInstanceNames() == null
1486                 || !VpnHelper.doesVpnInterfaceBelongToVpnInstance(oldVpnName, update.getVpnInstanceNames()))) {
1487                 UpdateData updateData = new UpdateData(identifier, original, update);
1488                 vpnInterfacesUpdateQueue.add(updateData);
1489                 LOG.info("update: UpdateData on VPNInterface {} on dpn {} update upon VPN swap from oldVpn(s) {}"
1490                         + "to newVpn(s) {} added to update queue",
1491                         updateData.getOriginal().getName(), dpnId,
1492                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1493                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1494                 return;
1495             }
1496         }
1497         for (VpnInstanceNames vpnInterfaceVpnInstance : update.getVpnInstanceNames()) {
1498             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1499             String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName);
1500             if (newVpnName != null && (original.getVpnInstanceNames() == null
1501                 || !VpnHelper.doesVpnInterfaceBelongToVpnInstance(newVpnName,
1502                                 original.getVpnInstanceNames()))) {
1503                 if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
1504                     InstanceIdentifier<VpnInterfaceOpDataEntry> opIdentifier = VpnUtil
1505                         .getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1506                     if (canHandleNewVpnInterface(identifier, update, newVpnName)) {
1507                         List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1508                         List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1509                         addVpnInterfaceToVpn(opIdentifier, update, copyOldAdjs, copyNewAdjs, identifier, newVpnName);
1510                     }
1511                 } else {
1512                     UpdateData updateData = new UpdateData(identifier, original, update);
1513                     vpnInterfacesUpdateQueue.add(updateData);
1514                     LOG.info("update: UpdateData on VPNInterface {} on dpn {} update upon VPN swap from oldVpn(s) {}"
1515                             + "to newVpn(s) {} added to update queue",
1516                             updateData.getOriginal().getName(), dpnId,
1517                             VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1518                             VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1519                     return;
1520                 }
1521             }
1522         }
1523         for (VpnInstanceNames vpnInterfaceVpnInstance : update.getVpnInstanceNames()) {
1524             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1525             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1526             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1527             String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName);
1528             if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
1529                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName + newVpnName, () -> {
1530                     WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1531                     WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
1532                     InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1533                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1534                     LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven",
1535                             update.getName(), newVpnName);
1536                     //handle both addition and removal of adjacencies
1537                     //currently, new adjacency may be an extra route
1538                     boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, newVpnName);
1539                     if (!oldAdjs.equals(newAdjs)) {
1540                         for (Adjacency adj : copyNewAdjs) {
1541                             if (copyOldAdjs.contains(adj)) {
1542                                 copyOldAdjs.remove(adj);
1543                             } else {
1544                                 // add new adjacency - right now only extra route will hit this path
1545                                 if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker, adj)) {
1546                                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1547                                              dpnId, writeOperTxn, writeConfigTxn);
1548                                 }
1549                                 LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to vpn "
1550                                                 + "interface {} on vpn {} dpnId {}",
1551                                         adj.getIpAddress(), adj.getNextHopIpList(),
1552                                         adj.getLabel(), adj.getSubnetId(), update.getName(),
1553                                         newVpnName, dpnId);
1554                             }
1555                         }
1556                         for (Adjacency adj : copyOldAdjs) {
1557                             if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker, adj)) {
1558                                 delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1559                                        writeOperTxn, writeConfigTxn);
1560                             }
1561                             LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1562                                 + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1563                                 adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1564                         }
1565                     }
1566                     ListenableFuture<Void> operFuture = writeOperTxn.submit();
1567                     try {
1568                         operFuture.get();
1569                     } catch (ExecutionException e) {
1570                         LOG.error("Exception encountered while submitting operational future for update"
1571                                 + " VpnInterface {} on vpn {}: {}", vpnInterfaceName, newVpnName, e);
1572                         return null;
1573                     }
1574                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1575                     futures.add(writeConfigTxn.submit());
1576                     LOG.info("update: vpn interface updated for interface {} oldVpn(s) {} newVpn {}"
1577                         + "processed successfully", update.getName(),
1578                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()), newVpnName);
1579                     return futures;
1580                 });
1581             } else {
1582                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1583                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1584             }
1585         }
1586     }
1587
1588     @SuppressWarnings("checkstyle:IllegalCatch")
1589     private class VpnInterfaceUpdateTimerTask implements Runnable {
1590         private final Logger log = LoggerFactory.getLogger(VpnInterfaceUpdateTimerTask.class);
1591
1592         @Override
1593         public void run() {
1594             try {
1595                 doRun();
1596             } catch (ReadFailedException | RuntimeException e) {
1597                 LOG.error("VpnInterfaceUpdateTimerTask failed", e);
1598             }
1599         }
1600
1601         private void doRun() throws ReadFailedException {
1602             List<UpdateData> processQueue = new ArrayList<>();
1603             List<UpdateData> updateDataList = new ArrayList<>();
1604             vpnInterfacesUpdateQueue.drainTo(processQueue);
1605             int maxInterfaceList = 0;
1606
1607             for (UpdateData updData : processQueue) {
1608                 final VpnInterfaceKey key = updData.getIdentifier()
1609                     .firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1610                 final String interfaceName = key.getName();
1611                 Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1612                 for (VpnInstanceNames vpnInterfaceVpnInstance : updData.getOriginal().getVpnInstanceNames()) {
1613                     String oldVpnName = vpnInterfaceVpnInstance.getVpnName();
1614                     if (updData.getUpdate().getVpnInstanceNames() != null
1615                         && VpnHelper.doesVpnInterfaceBelongToVpnInstance(oldVpnName,
1616                                     updData.getUpdate().getVpnInstanceNames())) {
1617                         continue;
1618                     }
1619                     log.info("run: VPN Interface update event - intfName {} remove vpnName {} running"
1620                         + " config-driven swap removal", updData.getOriginal().getName(), oldVpnName);
1621                     maxInterfaceList ++;
1622                     removeVpnInterfaceCall(updData.getIdentifier(), updData.getOriginal(),
1623                                        oldVpnName, interfaceName);
1624                     log.info("run: Processed Remove for update on VPNInterface {} upon VPN swap from old vpn {}"
1625                             + " to newVpn(s) {}", updData.getOriginal().getName(),
1626                             oldVpnName, VpnHelper.getVpnInterfaceVpnInstanceNamesString(updData
1627                                        .getUpdate().getVpnInstanceNames()));
1628                 }
1629                 updateDataList.add(updData);
1630             }
1631             /* Decide the max-wait time based on number of VpnInterfaces.
1632             *  max-wait-time is num-of-interface * 4seconds (random choice).
1633             *  Every 2sec poll VpnToDpnList. If VpnInterface is removed ,
1634             *  remove it from vpnInterfaceList.
1635             */
1636             int maxWaitTime =
1637                 maxInterfaceList * (int) (VpnConstants.PER_INTERFACE_MAX_WAIT_TIME_IN_MILLISECONDS / 1000);
1638             int waitTime = 2;
1639             Iterator<UpdateData> updateDataIterator = updateDataList.iterator();
1640             UpdateData updateDataSet;
1641             while (waitTime < maxWaitTime) {
1642                 try {
1643                     Thread.sleep(2000); // sleep for 2sec
1644                 } catch (InterruptedException e) {
1645                     // Ignored
1646                 }
1647
1648                 while (updateDataIterator.hasNext()) {
1649                     boolean interfaceIsRemoved = true;
1650                     updateDataSet = updateDataIterator.next();
1651                     for (VpnInstanceNames vpnInterfaceVpnInstance : updateDataSet.getOriginal().getVpnInstanceNames()) {
1652                         String oldVpnName = vpnInterfaceVpnInstance.getVpnName();
1653                         if (oldVpnName != null && updateDataSet.getUpdate().getVpnInstanceNames() != null
1654                             && VpnHelper.doesVpnInterfaceBelongToVpnInstance(oldVpnName,
1655                                     updateDataSet.getUpdate().getVpnInstanceNames())) {
1656                             continue;
1657                         }
1658
1659                         boolean isPresent = isVpnIntfPresentInVpnToDpnList(updateDataSet.getOriginal(), oldVpnName);
1660                         if (isPresent) {
1661                             interfaceIsRemoved = false;
1662                         }
1663                     }
1664                     if (interfaceIsRemoved) {
1665                         updateDataIterator.remove();
1666                     }
1667                 }
1668                 if (updateDataList.isEmpty()) {
1669                     log.info("run: All VpnInterfaces are successfully removed from OLD VPN after time {}", waitTime);
1670                     break;
1671                 }
1672                 waitTime += 2; //Increment linearly by 2sec.
1673             }
1674
1675             if (updateDataList.size() > 0) {
1676                 log.error("run: VpnInterfacesList not removed from old Vpn even after waiting {}", waitTime);
1677             }
1678             for (UpdateData updData : processQueue) {
1679                 if (updateDataList.contains(updData)) {
1680                     log.error("run: Failed to swap VpnInterface {} from oldVpn {} to target VPN {}"
1681                              + "as it has not been cleaned up from the oldVpn", updData.getOriginal().getName(),
1682                              VpnHelper.getVpnInterfaceVpnInstanceNamesString(updData.getOriginal()
1683                                             .getVpnInstanceNames()),
1684                              VpnHelper.getVpnInterfaceVpnInstanceNamesString(updData.getUpdate()
1685                                             .getVpnInstanceNames()));
1686                     continue;
1687                 }
1688                 for (VpnInstanceNames vpnInterfaceVpnInstance : updData.getUpdate().getVpnInstanceNames()) {
1689                     String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1690                     if (updData.getOriginal().getVpnInstanceNames() != null
1691                         && VpnHelper.doesVpnInterfaceBelongToVpnInstance(newVpnName,
1692                                               updData.getOriginal().getVpnInstanceNames())) {
1693                         continue;
1694                     }
1695                     log.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven"
1696                            + " swap addition", updData.getUpdate().getName(), newVpnName);
1697                     final Adjacencies origAdjs = updData.getOriginal().getAugmentation(Adjacencies.class);
1698                     final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency() != null
1699                         ? origAdjs.getAdjacency() : new ArrayList<>();
1700                     final Adjacencies updateAdjs = updData.getUpdate().getAugmentation(Adjacencies.class);
1701                     final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency() != null
1702                         ? updateAdjs.getAdjacency() : new ArrayList<>();
1703
1704                     addVpnInterfaceCall(updData.getIdentifier(), updData.getUpdate(),
1705                                       oldAdjs, newAdjs, newVpnName);
1706                     log.info("run: Processed Add for update on VPNInterface {} from oldVpn(s) {} to newVpn {}"
1707                             + " upon VPN swap", updData.getUpdate().getName(),
1708                             VpnHelper.getVpnInterfaceVpnInstanceNamesString(updData.getOriginal()
1709                                        .getVpnInstanceNames()), newVpnName);
1710                 }
1711             }
1712         }
1713
1714         private boolean isVpnIntfPresentInVpnToDpnList(VpnInterface vpnInterface, String vpnName)
1715                 throws ReadFailedException {
1716             BigInteger dpnId = vpnInterface.getDpnId();
1717             String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
1718             log.trace("isVpnIntfPresentInVpnToDpnList: GOT rd {} for VpnInterface {}  VpnInstance {} ", rd ,
1719                     vpnInterface.getName(), vpnName);
1720
1721             if (rd == null) {
1722                 return false;
1723             }
1724
1725             Optional<VpnInstanceOpDataEntry> maybeVpnInstanceOpData = vpnInstanceOpDataEntryCache.get(
1726                     VpnUtil.getVpnInstanceOpDataIdentifier(rd));
1727             if (maybeVpnInstanceOpData.isPresent()) {
1728                 log.trace("isVpnIntfPresentInVpnToDpnList: GOT VpnInstanceOp {} for rd {} ",
1729                         maybeVpnInstanceOpData.get(), rd);
1730                 List<VpnToDpnList> dpnToVpns = maybeVpnInstanceOpData.get().getVpnToDpnList();
1731                 if (dpnToVpns != null) {
1732                     for (VpnToDpnList dpn : dpnToVpns) {
1733                         if (dpn.getDpnId().equals(dpnId)) {
1734                             return dpn.getVpnInterfaces().stream().anyMatch(
1735                                 vpnInterfaces -> vpnInterface.getName().equals(vpnInterfaces.getInterfaceName()));
1736                         }
1737                         log.info("isVpnIntfPresentInVpnToDpnList: VpnInterface {} not present in DpnId {} vpn {}",
1738                                 vpnInterface.getName(), dpn.getDpnId(), vpnName);
1739                     }
1740                 }
1741             }
1742
1743             return false;
1744         }
1745     }
1746
1747     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1748         Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1749         synchronized (label.toString().intern()) {
1750             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1751                     .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1752             Optional<LabelRouteInfo> opResult = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
1753             if (opResult.isPresent()) {
1754                 LabelRouteInfo labelRouteInfo =
1755                     new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1756                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid, labelRouteInfo);
1757             }
1758         }
1759         LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label, nextHopIpList);
1760     }
1761
1762     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1763         SubnetRoute route, WriteTransaction writeConfigTxn) {
1764
1765         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1766         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, null /* parentVpnRd */)
1767                 .addAugmentation(SubnetRoute.class, route).build();
1768         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1769         InstanceIdentifierBuilder<VrfTables> idBuilder =
1770             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1771         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1772         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1773         if (writeConfigTxn != null) {
1774             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1775         } else {
1776             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1777         }
1778         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1779                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1780     }
1781
1782     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1783                                            Adjacency adj, BigInteger dpnId, WriteTransaction writeOperTxn,
1784                                            WriteTransaction writeConfigTxn) {
1785
1786         Optional<VpnInterfaceOpDataEntry> optVpnInterface = VpnUtil.read(dataBroker,
1787                                                 LogicalDatastoreType.OPERATIONAL, identifier);
1788
1789         if (optVpnInterface.isPresent()) {
1790             VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1791             String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1792             String vpnName = currVpnIntf.getVpnInstanceName();
1793             VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
1794             InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1795             Optional<AdjacenciesOp> optAdjacencies = VpnUtil.read(dataBroker,
1796                                             LogicalDatastoreType.OPERATIONAL, adjPath);
1797             boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1798             VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1799             long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1800             VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1801             List<Adjacency> adjacencies;
1802             if (optAdjacencies.isPresent()) {
1803                 adjacencies = optAdjacencies.get().getAdjacency();
1804             } else {
1805                 // This code will be hit in case of first PNF adjacency
1806                 adjacencies = new ArrayList<>();
1807             }
1808             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1809             L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1810                     .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1811             Adjacency operationalAdjacency = null;
1812             if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()) {
1813                 RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
1814                         : RouteOrigin.STATIC;
1815                 String nh = adj.getNextHopIpList().get(0);
1816                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1817                 synchronized (vpnPrefixKey.intern()) {
1818                     java.util.Optional<String> rdToAllocate = VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1819                             dataBroker, vpnId, null, prefix, vpnName, nh, dpnId, writeOperTxn);
1820                     if (rdToAllocate.isPresent()) {
1821                         input.setRd(rdToAllocate.get());
1822                         operationalAdjacency = populator.createOperationalAdjacency(input);
1823                         int label = operationalAdjacency.getLabel().intValue();
1824                         vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1825                                 currVpnIntf.getVpnInstanceName(), label, l3vni, origin,
1826                                 currVpnIntf.getName(), operationalAdjacency, encapType, writeConfigTxn);
1827                         LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1828                                 + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1829                                 vpnName, label, currVpnIntf.getName(), dpnId);
1830                     } else {
1831                         LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}", vpnName,
1832                                 prefix);
1833                         return;
1834                     }
1835                     // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1836                     // Keeping the MPLS check for now.
1837                     if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1838                         final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1839                         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1840                                 VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1841                         vpnsToImportRoute.forEach(vpn -> {
1842                             if (vpn.getVrfId() != null) {
1843                                 VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1844                                         dataBroker, vpn.getVpnId(), vpnId, prefix,
1845                                         VpnUtil.getVpnName(dataBroker, vpn.getVpnId()), nh, dpnId,
1846                                         writeOperTxn)
1847                                         .ifPresent(
1848                                             rds -> vpnManager.addExtraRoute(
1849                                                     VpnUtil.getVpnName(dataBroker, vpn.getVpnId()),
1850                                                     adj.getIpAddress(), nh, rds,
1851                                                     currVpnIntf.getVpnInstanceName(),
1852                                                     opAdjacency.getLabel().intValue(),
1853                                                     l3vni, RouteOrigin.SELF_IMPORTED,
1854                                                     currVpnIntf.getName(), opAdjacency, encapType, writeConfigTxn));
1855                             }
1856                         });
1857                     }
1858                 }
1859             } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1860                 LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1861                         currVpnIntf.getName(), vpnName);
1862
1863                 String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1864
1865                 writeOperTxn.merge(
1866                         LogicalDatastoreType.OPERATIONAL,
1867                         VpnUtil.getPrefixToInterfaceIdentifier(VpnUtil.getVpnId(dataBroker,
1868                                 adj.getSubnetId().getValue()), prefix),
1869                         VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1870                                 adj.getSubnetId(), Prefixes.PrefixCue.PhysNetFunc), true);
1871
1872                 fibManager.addOrUpdateFibEntry(dataBroker, adj.getSubnetId().getValue(), adj.getMacAddress(),
1873                         adj.getIpAddress(), Collections.emptyList(), null /* EncapType */, 0 /* label */, 0 /*l3vni*/,
1874                       null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1875
1876                 input.setRd(adj.getVrfId());
1877             }
1878             if (operationalAdjacency == null) {
1879                 operationalAdjacency = populator.createOperationalAdjacency(input);
1880             }
1881             adjacencies.add(operationalAdjacency);
1882             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1883             VpnInterfaceOpDataEntry newVpnIntf =
1884                 VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
1885                     aug, dpnId, currVpnIntf.isScheduledForRemove(), currVpnIntf.getLportTag(),
1886                         currVpnIntf.getGatewayMacAddress());
1887
1888             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1889         }
1890     }
1891
1892     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
1893         Subnets subnets = VpnUtil.getExternalSubnet(dataBroker, adj.getSubnetId());
1894         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
1895     }
1896
1897     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
1898             BigInteger dpnId, WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) {
1899         Optional<VpnInterfaceOpDataEntry> optVpnInterface = VpnUtil.read(dataBroker,
1900                                           LogicalDatastoreType.OPERATIONAL, identifier);
1901
1902         if (optVpnInterface.isPresent()) {
1903             VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1904
1905             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1906             Optional<AdjacenciesOp> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1907             if (optAdjacencies.isPresent()) {
1908                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1909
1910                 if (!adjacencies.isEmpty()) {
1911                     LOG.trace("delAdjFromVpnInterface: Adjacencies are " + adjacencies);
1912                     Iterator<Adjacency> adjIt = adjacencies.iterator();
1913                     while (adjIt.hasNext()) {
1914                         Adjacency adjElem = adjIt.next();
1915                         if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1916                             String rd = adjElem.getVrfId();
1917                             adjIt.remove();
1918
1919                             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1920                             VpnInterfaceOpDataEntry newVpnIntf = VpnUtil
1921                                     .getVpnInterfaceOpDataEntry(currVpnIntf.getName(),
1922                                     currVpnIntf.getVpnInstanceName(), aug, dpnId, currVpnIntf.isScheduledForRemove(),
1923                                             currVpnIntf.getLportTag(), currVpnIntf.getGatewayMacAddress());
1924
1925                             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1926                             if (adj.getNextHopIpList() != null) {
1927                                 for (String nh : adj.getNextHopIpList()) {
1928                                     deleteExtraRouteFromCurrentAndImportingVpns(
1929                                         currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
1930                                         currVpnIntf.getName(), writeConfigTxn);
1931                                 }
1932                             } else if (adj.isPhysNetworkFunc()) {
1933                                 LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet [}",
1934                                         adj.getIpAddress(), adj.getSubnetId());
1935                                 fibManager.removeFibEntry(dataBroker, adj.getSubnetId().getValue(), adj.getIpAddress(),
1936                                         writeConfigTxn);
1937                             }
1938                             break;
1939                         }
1940
1941                     }
1942                 }
1943                 LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
1944                         dpnId, adj.getVrfId());
1945             } else {
1946                 LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
1947                         + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
1948             }
1949         }
1950     }
1951
1952     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
1953         String rd, String intfName, WriteTransaction writeConfigTxn) {
1954         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn);
1955         List<VpnInstanceOpDataEntry> vpnsToImportRoute = VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1956         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1957             String vpnRd = vpn.getVrfId();
1958             if (vpnRd != null) {
1959                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn);
1960             }
1961         }
1962     }
1963
1964     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1965         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1966             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1967             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1968     }
1969
1970     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1971         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1972             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1973     }
1974
1975     protected void addToNeutronRouterDpnsMap(String routerName, String vpnInterfaceName,
1976         WriteTransaction writeOperTxn) {
1977         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1978         if (dpId.equals(BigInteger.ZERO)) {
1979             LOG.error("addToNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
1980                     + " association model", vpnInterfaceName, routerName);
1981
1982             return;
1983         }
1984         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1985
1986         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1987             .OPERATIONAL, routerDpnListIdentifier);
1988         RouterInterfaces routerInterface =
1989             new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
1990                 vpnInterfaceName).build();
1991         if (optionalRouterDpnList.isPresent()) {
1992             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
1993                 RouterInterfaces.class, new RouterInterfacesKey(vpnInterfaceName)), routerInterface, true);
1994         } else {
1995             RouterDpnListBuilder builder = new RouterDpnListBuilder();
1996             builder.setRouterId(routerName);
1997             DpnVpninterfacesListBuilder dpnVpnList = new DpnVpninterfacesListBuilder().setDpnId(dpId);
1998             builder.setDpnVpninterfacesList(Collections.singletonList(dpnVpnList.build()));
1999             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL,
2000                 getRouterId(routerName),
2001                 builder.build(), true);
2002         }
2003     }
2004
2005     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName,
2006         WriteTransaction writeOperTxn) {
2007         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
2008         if (dpId.equals(BigInteger.ZERO)) {
2009             LOG.error("removeFromNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
2010                     + " dissociation model", vpnInterfaceName, routerName);
2011
2012             return;
2013         }
2014         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
2015         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
2016             .OPERATIONAL, routerDpnListIdentifier);
2017         if (optionalRouterDpnList.isPresent()) {
2018             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
2019             RouterInterfaces routerInterface =
2020                 new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
2021                     vpnInterfaceName).build();
2022
2023             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
2024                 if (routerInterfaces.isEmpty()) {
2025                     if (writeOperTxn != null) {
2026                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
2027                     } else {
2028                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
2029                     }
2030                 } else {
2031                     if (writeOperTxn != null) {
2032                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
2033                             RouterInterfaces.class,
2034                             new RouterInterfacesKey(vpnInterfaceName)));
2035                     } else {
2036                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
2037                             routerDpnListIdentifier.child(
2038                                 RouterInterfaces.class,
2039                                 new RouterInterfacesKey(vpnInterfaceName)));
2040                     }
2041                 }
2042             }
2043         }
2044     }
2045
2046     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, BigInteger dpId,
2047         WriteTransaction writeOperTxn) {
2048         if (dpId.equals(BigInteger.ZERO)) {
2049             LOG.error("removeFromNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
2050                     + " dissociation model", vpnInterfaceName, routerName);
2051
2052             return;
2053         }
2054         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
2055         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
2056             .OPERATIONAL, routerDpnListIdentifier);
2057         if (optionalRouterDpnList.isPresent()) {
2058             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
2059             RouterInterfaces routerInterface =
2060                 new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
2061                     vpnInterfaceName).build();
2062             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
2063                 if (routerInterfaces.isEmpty()) {
2064                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
2065                 } else {
2066                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
2067                         RouterInterfaces.class,
2068                         new RouterInterfacesKey(vpnInterfaceName)));
2069                 }
2070             }
2071         }
2072     }
2073
2074     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2075                                                     WriteTransaction writeConfigTxn, String vpnName) {
2076         if (vpnInterface == null) {
2077             return;
2078         }
2079         List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
2080         if (adjs == null) {
2081             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2082                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2083             return;
2084         }
2085         for (Adjacency adj : adjs) {
2086             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2087                 String primaryInterfaceIp = adj.getIpAddress();
2088                 String macAddress = adj.getMacAddress();
2089                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2090
2091                 long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
2092                     VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2093
2094                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2095                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2096                 fibManager.addFibEntryForRouterInterface(dataBroker, primaryRd, prefix,
2097                         routerInt, label, writeConfigTxn);
2098                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2099                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2100                         macAddress);
2101                 return;
2102             }
2103         }
2104         LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2105                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}", interfaceName,
2106                 primaryRd, vpnName);
2107     }
2108
2109     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
2110                                                     WriteTransaction writeConfigTxn, String vpnName) {
2111         Adjacencies adjs = vpnInterface.getAugmentation(Adjacencies.class);
2112         String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
2113         if (adjs != null) {
2114             List<Adjacency> adjsList = adjs.getAdjacency();
2115             for (Adjacency adj : adjsList) {
2116                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2117                     String primaryInterfaceIp = adj.getIpAddress();
2118                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2119                     fibManager.removeFibEntry(dataBroker, rd, prefix, writeConfigTxn);
2120                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2121                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2122                     return;
2123                 }
2124             }
2125         } else {
2126             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2127                     vpnInterface.getName(), rd);
2128         }
2129     }
2130
2131     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2132         if (!canHandleNewVpnInterface(intefaceData.identifier, intefaceData.vpnInterface, vpnName)) {
2133             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
2134                   vpnName, intefaceData.vpnInterface.getName());
2135             return;
2136         }
2137         final VpnInterfaceKey key = intefaceData.identifier
2138                .firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
2139         final String interfaceName = key.getName();
2140         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2141                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2142         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2143                   intefaceData.identifier, vpnName);
2144         return;
2145     }
2146
2147     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2148                                               VpnInterface vpnInterface, String vpnName) {
2149         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2150                .get(vpnName);
2151         if (vpnInterfaces == null) {
2152             vpnInterfaces = new ConcurrentLinkedQueue<>();
2153         }
2154         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2155         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2156         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2157                 vpnInterface.getName(), vpnName);
2158     }
2159
2160     public boolean isVpnInstanceReady(String vpnInstanceName) {
2161         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnInstanceName);
2162         if (vpnRd == null) {
2163             return false;
2164         }
2165         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
2166
2167         return vpnInstanceOpDataEntry != null;
2168     }
2169
2170     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2171         synchronized (vpnInstanceName.intern()) {
2172             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2173                     unprocessedVpnInterfaces.get(vpnInstanceName);
2174             if (vpnInterfaces != null) {
2175                 while (!vpnInterfaces.isEmpty()) {
2176                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2177                     if (hasVpnInstanceCreatedSuccessfully) {
2178                         processSavedInterface(savedInterface, vpnInstanceName);
2179                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2180                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2181                     } else {
2182                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2183                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2184                     }
2185                 }
2186             } else {
2187                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2188             }
2189         }
2190     }
2191
2192     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2193             VpnInterface vpnInterface) {
2194         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2195             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2196                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2197             if (vpnInterfaces != null) {
2198                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2199                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2200                             + "unprocessed list", vpnInterface.getName(),
2201                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2202                 }
2203             } else {
2204                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2205                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2206             }
2207         }
2208     }
2209
2210     public void vpnInstanceIsReady(String vpnInstanceName) {
2211         processSavedInterfaces(vpnInstanceName, true);
2212     }
2213
2214     public void vpnInstanceFailed(String vpnInstanceName) {
2215         processSavedInterfaces(vpnInstanceName, false);
2216     }
2217
2218     private static class UnprocessedVpnInterfaceData {
2219         InstanceIdentifier<VpnInterface> identifier;
2220         VpnInterface vpnInterface;
2221
2222         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2223             this.identifier = identifier;
2224             this.vpnInterface = vpnInterface;
2225         }
2226
2227         @Override
2228         public int hashCode() {
2229             final int prime = 31;
2230             int result = 1;
2231             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2232             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2233             return result;
2234         }
2235
2236         @Override
2237         public boolean equals(Object obj) {
2238             if (this == obj) {
2239                 return true;
2240             }
2241             if (obj == null) {
2242                 return false;
2243             }
2244             if (getClass() != obj.getClass()) {
2245                 return false;
2246             }
2247             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2248             if (identifier == null) {
2249                 if (other.identifier != null) {
2250                     return false;
2251                 }
2252             } else if (!identifier.equals(other.identifier)) {
2253                 return false;
2254             }
2255             if (vpnInterface == null) {
2256                 if (other.vpnInterface != null) {
2257                     return false;
2258                 }
2259             } else if (!vpnInterface.equals(other.vpnInterface)) {
2260                 return false;
2261             }
2262             return true;
2263         }
2264     }
2265
2266     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2267         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
2268         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
2269         if (vpnInstanceOpData == null) {
2270             return;
2271         }
2272         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2273         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2274             return;
2275         }
2276         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2277         vpnToDpnLists.forEach(vpnToDpnList -> vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2278             InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2279                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2280             Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = VpnUtil.read(dataBroker,
2281                     LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2282             if (!vpnInterfaceOptional.isPresent()) {
2283                 return;
2284             }
2285             List<Adjacency> configVpnAdjacencies = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker,
2286                     vpnInterface.getInterfaceName());
2287             if (configVpnAdjacencies == null) {
2288                 LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2289                 return;
2290             }
2291             List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2292                     .getAugmentation(AdjacenciesOp.class).getAdjacency();
2293             // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2294             // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2295             // in operational DS. These unprocessed adjacencies will be handled below.
2296             // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in operational
2297             // DS are retrieved which is used to call addNewAdjToVpnInterface method.
2298             configVpnAdjacencies.stream()
2299                     .filter(adjacency -> operationVpnAdjacencies.stream()
2300                             .noneMatch(operationalAdjacency ->
2301                                     operationalAdjacency.getIpAddress().equals(adjacency.getIpAddress())))
2302                     .forEach(adjacency -> {
2303                         LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2304                         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName() + vpnName,
2305                             () -> {
2306                                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
2307                                 WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
2308                                 if (VpnUtil.isAdjacencyEligibleToVpn(
2309                                      dataBroker, adjacency, vpnName, vpnInterface.getInterfaceName())) {
2310                                     addNewAdjToVpnInterface(existingVpnInterfaceId, primaryRd, adjacency,
2311                                             vpnInterfaceOptional.get().getDpnId(), writeConfigTxn, writeOperTxn);
2312                                     ListenableFuture<Void> operFuture = writeOperTxn.submit();
2313                                     try {
2314                                         operFuture.get();
2315                                     } catch (ExecutionException | InterruptedException e) {
2316                                         LOG.error("Exception encountered while submitting operational"
2317                                                 + " future for vpnInterface {}", vpnInterface, e);
2318                                     }
2319                                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2320                                     futures.add(writeConfigTxn.submit());
2321                                     return futures;
2322                                 }  else {
2323                                     return Collections.emptyList();
2324                                 }
2325                             });
2326                     });
2327         }));
2328     }
2329
2330     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2331         private final String interfaceName;
2332         private final boolean add;
2333         private final String txnDestination;
2334
2335         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2336             this.interfaceName = interfaceName;
2337             this.add = add;
2338             this.txnDestination = transactionDest;
2339         }
2340
2341         @Override
2342         public void onSuccess(Void voidObj) {
2343             if (add) {
2344                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2345                         interfaceName, txnDestination);
2346             } else {
2347                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2348             }
2349         }
2350
2351         @Override
2352         public void onFailure(Throwable throwable) {
2353             if (add) {
2354                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}"
2355                         + " with exception: {}", interfaceName, txnDestination, throwable);
2356             } else {
2357                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed with exception: {}", interfaceName,
2358                         throwable);
2359                 VpnUtil.unsetScheduledToRemoveForVpnInterface(dataBroker, interfaceName);
2360             }
2361         }
2362     }
2363 }