vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.collect.Iterators;
  13 import com.google.common.util.concurrent.FutureCallback;
  14 import com.google.common.util.concurrent.Futures;
  15 import com.google.common.util.concurrent.ListenableFuture;
  16 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  17 import java.math.BigInteger;
  18 import java.util.ArrayList;
  19 import java.util.Collections;
  20 import java.util.Iterator;
  21 import java.util.List;
  22 import java.util.Map;
  23 import java.util.Objects;
  24 import java.util.concurrent.BlockingQueue;
  25 import java.util.concurrent.ConcurrentHashMap;
  26 import java.util.concurrent.ConcurrentLinkedQueue;
  27 import java.util.concurrent.ExecutionException;
  28 import java.util.concurrent.Executors;
  29 import java.util.concurrent.LinkedBlockingQueue;
  30 import java.util.concurrent.ScheduledExecutorService;
  31 import java.util.concurrent.TimeUnit;
  32 import java.util.function.Consumer;
  33 import java.util.function.Predicate;
  34 import java.util.stream.Collectors;
  35 import javax.annotation.PostConstruct;
  36 import javax.annotation.PreDestroy;
  37 import javax.inject.Inject;
  38 import javax.inject.Singleton;
  39
  40 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  41 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  42 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  43 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  44 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  45 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  46 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  47 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  48 import org.opendaylight.genius.mdsalutil.MDSALUtil;
  49 import org.opendaylight.genius.mdsalutil.NWUtil;
  50 import org.opendaylight.genius.mdsalutil.NwConstants;
  51 import org.opendaylight.genius.mdsalutil.cache.DataObjectCache;
  52 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  53 import org.opendaylight.infrautils.caches.CacheProvider;
  54 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  55 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  56 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  57 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  58 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  59 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  60 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  61 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  62 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  63 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  64 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  65 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  66 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  67 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  68 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  69 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  70 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  71 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  72 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListBuilder;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfaces;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesBuilder;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.dpn.vpninterfaces.list.RouterInterfacesKey;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
 107 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
 108 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
 109 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 110 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 111 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 112 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 113 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 114 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 115 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 116 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 117 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 118 import org.slf4j.Logger;
 119 import org.slf4j.LoggerFactory;
 120
 121 @Singleton
 122 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 123
 124     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 125     private static final int VPN_INF_UPDATE_TIMER_TASK_DELAY = 1000;
 126     private static final TimeUnit TIME_UNIT = TimeUnit.MILLISECONDS;
 127     private static final short DJC_MAX_RETRIES = 3;
 128
 129     private final DataBroker dataBroker;
 130     private final ManagedNewTransactionRunner txRunner;
 131     private final IBgpManager bgpManager;
 132     private final IFibManager fibManager;
 133     private final IMdsalApiManager mdsalManager;
 134     private final IdManagerService idManager;
 135     private final OdlInterfaceRpcService ifaceMgrRpcService;
 136     private final VpnFootprintService vpnFootprintService;
 137     private final IInterfaceManager interfaceManager;
 138     private final IVpnManager vpnManager;
 139     private final ArpResponderHandler arpResponderHandler;
 140     private final JobCoordinator jobCoordinator;
 141
 142     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 143
 144     private final BlockingQueue<UpdateData> vpnInterfacesUpdateQueue = new LinkedBlockingQueue<>();
 145     private final ScheduledExecutorService vpnInfUpdateTaskExecutor = Executors.newScheduledThreadPool(1);
 146
 147     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 148             new ConcurrentHashMap<>();
 149
 150     private final DataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 151
 152     @Inject
 153     public VpnInterfaceManager(final DataBroker dataBroker,
 154                                final IBgpManager bgpManager,
 155                                final IdManagerService idManager,
 156                                final IMdsalApiManager mdsalManager,
 157                                final IFibManager fibManager,
 158                                final OdlInterfaceRpcService ifaceMgrRpcService,
 159                                final VpnFootprintService vpnFootprintService,
 160                                final IInterfaceManager interfaceManager,
 161                                final IVpnManager vpnManager,
 162                                final ArpResponderHandler arpResponderHandler,
 163                                final JobCoordinator jobCoordinator,
 164                                final CacheProvider cacheProvider) {
 165         super(VpnInterface.class, VpnInterfaceManager.class);
 166
 167         this.dataBroker = dataBroker;
 168         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 169         this.bgpManager = bgpManager;
 170         this.idManager = idManager;
 171         this.mdsalManager = mdsalManager;
 172         this.fibManager = fibManager;
 173         this.ifaceMgrRpcService = ifaceMgrRpcService;
 174         this.vpnFootprintService = vpnFootprintService;
 175         this.interfaceManager = interfaceManager;
 176         this.vpnManager = vpnManager;
 177         this.arpResponderHandler = arpResponderHandler;
 178         this.jobCoordinator = jobCoordinator;
 179         vpnInfUpdateTaskExecutor.scheduleWithFixedDelay(new VpnInterfaceUpdateTimerTask(),
 180             0, VPN_INF_UPDATE_TIMER_TASK_DELAY, TIME_UNIT);
 181
 182         vpnInstanceOpDataEntryCache = new DataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 183                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 184                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 185     }
 186
 187     public Runnable isNotifyTaskQueued(String intfName) {
 188         return vpnIntfMap.remove(intfName);
 189     }
 190
 191     @PostConstruct
 192     public void start() {
 193         LOG.info("{} start", getClass().getSimpleName());
 194         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 195     }
 196
 197     @Override
 198     @PreDestroy
 199     public void close() {
 200         super.close();
 201         vpnInfUpdateTaskExecutor.shutdown();
 202         vpnInstanceOpDataEntryCache.close();
 203     }
 204
 205     @Override
 206     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 207         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 208     }
 209
 210     @Override
 211     protected VpnInterfaceManager getDataTreeChangeListener() {
 212         return VpnInterfaceManager.this;
 213     }
 214
 215     @Override
 216     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 217         LOG.info("add: intfName {} onto vpnName {}",
 218                  vpnInterface.getName(),
 219                  VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 220         addVpnInterface(identifier, vpnInterface, null, null);
 221     }
 222
 223     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 224                           final VpnInterface vpnInterface, String vpnName) {
 225         synchronized (vpnName.intern()) {
 226             if (isVpnInstanceReady(vpnName)) {
 227                 return true;
 228             }
 229             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 230             return false;
 231         }
 232     }
 233
 234     // TODO Clean up the exception handling
 235     @SuppressWarnings("checkstyle:IllegalCatch")
 236     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 237                              final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs) {
 238         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
 239             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 240             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 241         }
 242     }
 243
 244     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 245                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 246         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 247         final String interfaceName = key.getName();
 248
 249         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 250             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 251                   vpnName, vpnInterface.getName());
 252             return;
 253         }
 254         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 255              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 256         List<Adjacency> copyOldAdjs = null;
 257         if (oldAdjs != null) {
 258             copyOldAdjs = new ArrayList<>();
 259             copyOldAdjs.addAll(oldAdjs);
 260         }
 261         List<Adjacency> copyNewAdjs = null;
 262         if (newAdjs != null) {
 263             copyNewAdjs = new ArrayList<>();
 264             copyNewAdjs.addAll(newAdjs);
 265         }
 266         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 267     }
 268
 269     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 270                     final VpnInterface vpnInterface, final List<Adjacency> oldAdjs,
 271                     final List<Adjacency> newAdjs,
 272                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 273         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 274         final String interfaceName = key.getName();
 275         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 276         if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
 277             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 278             boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
 279             if (interfaceState != null) {
 280                 try {
 281                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 282                     final int ifIndex = interfaceState.getIfIndex();
 283                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName + vpnName, () -> {
 284                         WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 285                         WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
 286                         WriteTransaction writeInvTxn = dataBroker.newWriteOnlyTransaction();
 287                         LOG.info("addVpnInterface: VPN Interface add event - intfName {} vpnName {} on dpn {}" ,
 288                                 vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 289                         processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false, writeConfigTxn,
 290                                      writeOperTxn, writeInvTxn, interfaceState, vpnName);
 291                         if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 292                             LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 293                                     + " Update for swapping VPN {} case.", interfaceName, vpnName);
 294                             if (newAdjs != null) {
 295                                 for (Adjacency adj : newAdjs) {
 296                                     if (oldAdjs.contains(adj)) {
 297                                         oldAdjs.remove(adj);
 298                                     } else {
 299                                         if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(
 300                                                                   dataBroker, adj)) {
 301                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd,
 302                                                   adj, dpnId, writeOperTxn, writeConfigTxn);
 303                                         }
 304                                     }
 305                                 }
 306                             }
 307                             for (Adjacency adj : oldAdjs) {
 308                                 if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(
 309                                                           dataBroker, adj)) {
 310                                     delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 311                                         writeOperTxn, writeConfigTxn);
 312                                 }
 313                             }
 314                         }
 315                         ListenableFuture<Void> operFuture = writeOperTxn.submit();
 316                         try {
 317                             operFuture.get();
 318                         } catch (ExecutionException e) {
 319                             LOG.error("addVpnInterface: Exception encountered while submitting operational future for"
 320                                     + " addVpnInterface {} on vpn {}: {}", vpnInterface.getName(), vpnName, e);
 321                             return null;
 322                         }
 323                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 324                         ListenableFuture<Void> configFuture = writeConfigTxn.submit();
 325                         futures.add(configFuture);
 326                         Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"));
 327                         futures.add(writeInvTxn.submit());
 328                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 329                                 + " processed successfully", interfaceName, vpnName, dpnId);
 330                         return futures;
 331                     });
 332                 } catch (NumberFormatException | IllegalStateException e) {
 333                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 334                                     + "interface {}. Interface addition on vpn {} failed", interfaceName,
 335                                     vpnName, e);
 336                     return;
 337                 }
 338             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 339                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName() + vpnName,
 340                     () -> {
 341                         WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
 342                         createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 343                                        writeConfigTxn, vpnName);
 344                         LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 345                                  vpnName, vpnInterface.getDpnId());
 346                         ListenableFuture<Void> futures = writeConfigTxn.submit();
 347                         String errorText = "addVpnInterfaceCall: Exception encountered while submitting writeConfigTxn"
 348                              + " for interface " + vpnInterface.getName() + " on vpn " + vpnName;
 349                         ListenableFutures.addErrorLogging(futures, LOG, errorText);
 350                         return Collections.singletonList(futures);
 351                     });
 352             } else {
 353                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 354                         + " is not available", interfaceName, vpnName);
 355             }
 356         } else {
 357             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 358                       + " as vpn is pending delete", interfaceName, vpnName,
 359                     vpnInterface.getDpnId());
 360         }
 361     }
 362
 363     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 364     // see comments below.
 365     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 366     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 367             final int lportTag, boolean isInterfaceUp,
 368             WriteTransaction writeConfigTxn,
 369             WriteTransaction writeOperTxn,
 370             WriteTransaction writeInvTxn,
 371             Interface interfaceState,
 372             final String vpnName) {
 373         final String interfaceName = vpnInterface.getName();
 374         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = VpnUtil.getVpnInterfaceOpDataEntry(dataBroker,
 375                                                          interfaceName, vpnName);
 376         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 377         boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
 378         if (!isInterfaceUp) {
 379             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 380                      interfaceName, dpId, vpnName);
 381             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 382             if (vpnId == VpnConstants.INVALID_ID) {
 383                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 384                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 385                         dpId);
 386                 return;
 387             }
 388
 389             boolean waitForVpnInterfaceOpRemoval = false;
 390             if (opVpnInterface != null && !opVpnInterface.isScheduledForRemove()) {
 391                 String opVpnName = opVpnInterface.getVpnInstanceName();
 392                 String primaryInterfaceIp = null;
 393                 if (opVpnName.equals(vpnName)) {
 394                     // Please check if the primary VRF Entry does not exist for VPNInterface
 395                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 396                     // back to back
 397                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 398                     List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
 399                     if (adjs == null) {
 400                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 401                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 402                                 vpnName);
 403                         return;
 404                     }
 405                     for (Adjacency adj : adjs) {
 406                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 407                             primaryInterfaceIp = adj.getIpAddress();
 408                             break;
 409                         }
 410                     }
 411                     if (primaryInterfaceIp == null) {
 412                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 413                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 414                                 dpId, vpnName);
 415                         return;
 416                     }
 417                     // Get the rd of the vpn instance
 418                     VrfEntry vrf = VpnUtil.getVrfEntry(dataBroker, primaryRd, primaryInterfaceIp);
 419                     if (vrf != null) {
 420                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 421                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 422                         return;
 423                     }
 424                     waitForVpnInterfaceOpRemoval = true;
 425                 } else {
 426                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 427                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 428                 }
 429             }
 430             if (!waitForVpnInterfaceOpRemoval) {
 431                 // Add the VPNInterface and quit
 432                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 433                         null/*ipAddressSourceValuePair*/,
 434                         true /* add */);
 435                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 436                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 437                 if (!isBgpVpnInternetVpn) {
 438                     VpnUtil.bindService(vpnName, interfaceName, dataBroker, false /*isTunnelInterface*/,
 439                             jobCoordinator);
 440                 }
 441                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 442                         dpId, vpnName);
 443                 if (interfaceManager.isExternalInterface(interfaceName)) {
 444                     processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
 445                         NwConstants.ADD_FLOW);
 446                 }
 447                 return;
 448             }
 449
 450             // FIB didn't get a chance yet to clean up this VPNInterface
 451             // Let us give it a chance here !
 452             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 453                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 454             try {
 455                 Runnable notifyTask = new VpnNotifyTask();
 456                 synchronized (notifyTask) {
 457                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 458                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 459                     // published here while holding the lock on it so this path will hit the wait before notify can be
 460                     // invoked.
 461                     vpnIntfMap.put(interfaceName, notifyTask);
 462                     try {
 463                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 464                     } catch (InterruptedException e) {
 465                         // Ignored
 466                     }
 467                 }
 468             } finally {
 469                 vpnIntfMap.remove(interfaceName);
 470             }
 471
 472             if (opVpnInterface != null) {
 473                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 474                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 475                         dpId, vpnName);
 476                 VpnUtil.unsetScheduledToRemoveForVpnInterface(dataBroker, interfaceName);
 477                 return;
 478             }
 479             // VPNInterface got removed, proceed with Add
 480             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 481                     interfaceName, dpId, vpnName);
 482             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 483                     null/*ipAddressSourceValuePair*/,
 484                     true /* add */);
 485             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 486                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 487             if (!isBgpVpnInternetVpn) {
 488                 VpnUtil.bindService(vpnName, interfaceName, dataBroker, false/*isTunnelInterface*/, jobCoordinator);
 489             }
 490             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 491                     + " FIB to clean up", interfaceName, dpId, vpnName);
 492             if (interfaceManager.isExternalInterface(interfaceName)) {
 493                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId,
 494                                lportTag, writeInvTxn, NwConstants.ADD_FLOW);
 495             }
 496         } else {
 497             // Interface is retained in the DPN, but its Link Up.
 498             // Advertise prefixes again for this interface to BGP
 499             InstanceIdentifier<VpnInterface> identifier =
 500                 VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 501             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 502                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 503             advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 504             // Perform similar operation as interface add event for extraroutes.
 505             InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 506             Optional<Adjacencies> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 507             if (!optAdjacencies.isPresent()) {
 508                 LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 509                 return;
 510             }
 511             List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
 512             for (Adjacency adjacency : adjacencies) {
 513                 if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 514                     continue;
 515                 }
 516                 // if BGPVPN Internet, filter only IPv6 Adjacencies
 517                 if (isBgpVpnInternetVpn && !VpnUtil.isAdjacencyEligibleToVpnInternet(
 518                                 dataBroker, adjacency)) {
 519                     continue;
 520                 }
 521                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 522                           dpId, writeOperTxn, writeConfigTxn);
 523             }
 524         }
 525     }
 526
 527     private void processExternalVpnInterface(String interfaceName, String vpnName, long vpnId, BigInteger dpId,
 528             int lportTag, WriteTransaction writeInvTxn, int addOrRemove) {
 529         Uuid extNetworkId;
 530         try {
 531             // vpn instance of ext-net interface is the network-id
 532             extNetworkId = new Uuid(vpnName);
 533         } catch (IllegalArgumentException e) {
 534             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 535                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 536             return;
 537         }
 538
 539         List<Uuid> routerIds = VpnUtil.getExternalNetworkRouterIds(dataBroker, extNetworkId);
 540         if (routerIds == null || routerIds.isEmpty()) {
 541             LOG.info("processExternalVpnInterface: No router is associated with {}."
 542                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 543                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 544             return;
 545         }
 546
 547         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 548                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 549         for (Uuid routerId : routerIds) {
 550             String routerName = routerId.getValue();
 551             BigInteger primarySwitch = VpnUtil.getPrimarySwitchForRouter(dataBroker, routerName);
 552             if (Objects.equals(primarySwitch, dpId)) {
 553                 Routers router = VpnUtil.getExternalRouter(dataBroker, routerName);
 554                 if (router != null) {
 555                     if (addOrRemove == NwConstants.ADD_FLOW) {
 556                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 557                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 558                                 dpId, interfaceName, lportTag, writeInvTxn);
 559                     } else {
 560                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 561                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 562                                 dpId, interfaceName, lportTag);
 563                     }
 564                 } else {
 565                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 566                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 567                             interfaceName, dpId, vpnName);
 568                 }
 569             }
 570         }
 571     }
 572
 573     // TODO Clean up the exception handling
 574     @SuppressWarnings("checkstyle:IllegalCatch")
 575     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 576                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 577                                                   String vpnName, String interfaceName) {
 578         if (rd == null) {
 579             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 580                   interfaceName, dpnId, vpnName);
 581             return;
 582         } else {
 583             if (rd.equals(vpnName)) {
 584                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 585                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 586                 return;
 587             }
 588         }
 589         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 590                 interfaceName, dpnId, vpnName, rd);
 591
 592         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 593         if (nextHopIp == null) {
 594             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 595                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 596                     rd, vpnName);
 597             return;
 598         }
 599
 600         //Read NextHops
 601         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 602         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 603         if (adjacencies.isPresent()) {
 604             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 605
 606             if (!nextHops.isEmpty()) {
 607                 LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 608                         + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 609                 VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, rd);
 610                 long l3vni = vpnInstanceOpData.getL3vni();
 611                 VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 612                         ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 613                 for (Adjacency nextHop : nextHops) {
 614                     if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 615                         continue;
 616                     }
 617                     String gatewayMac = null;
 618                     long label = 0;
 619                     if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 620                         final VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker,
 621                             vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 622                         gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName).get();
 623                     } else {
 624                         label = nextHop.getLabel();
 625                     }
 626                     try {
 627                         LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 628                                 + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 629                         bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 630                                 encapType, (int)label, l3vni, 0 /*l2vni*/,
 631                                 gatewayMac);
 632                         LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 633                                 + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 634                                 nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 635                     } catch (Exception e) {
 636                         LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {} with rd {}"
 637                                 + " for interface {} on dpn {}", nextHop.getIpAddress(),
 638                                   vpnName, rd, interfaceName, dpnId, e);
 639                     }
 640                 }
 641             }
 642         }
 643     }
 644
 645     // TODO Clean up the exception handling
 646     @SuppressWarnings("checkstyle:IllegalCatch")
 647     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 648                         String vpnName, String interfaceName, WriteTransaction writeConfigTxn) {
 649         //Read NextHops
 650         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 651         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
 652
 653         String rd = VpnUtil.getVpnRd(dataBroker, interfaceName);
 654         if (rd == null) {
 655             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 656                 interfaceName, vpnName);
 657             return;
 658         } else {
 659             if (rd.equals(vpnName)) {
 660                 LOG.info(
 661                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 662                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 663                 return;
 664             }
 665         }
 666         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 667                vpnName, rd);
 668         if (adjacencies.isPresent()) {
 669             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 670
 671             if (!nextHops.isEmpty()) {
 672                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 673                         nextHops, interfaceName, vpnName, rd);
 674                 for (Adjacency nextHop : nextHops) {
 675                     try {
 676                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 677                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 678                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 679                                     interfaceName, vpnName);
 680                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 681                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 682                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 683                                     interfaceName, vpnName);
 684                         } else {
 685                             // Perform similar operation as interface delete event for extraroutes.
 686                             String allocatedRd = nextHop.getVrfId();
 687                             for (String nh : nextHop.getNextHopIpList()) {
 688                                 deleteExtraRouteFromCurrentAndImportingVpns(
 689                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn);
 690                             }
 691                         }
 692                     } catch (Exception e) {
 693                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 694                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 695                     }
 696                 }
 697             }
 698         }
 699     }
 700
 701     @SuppressWarnings("checkstyle:IllegalCatch")
 702     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 703                                                   String primaryRd, String interfaceName, final long vpnId,
 704                                                   WriteTransaction writeConfigTxn,
 705                                                   WriteTransaction writeOperTxn,
 706                                                   final WriteTransaction writeInvTxn,
 707                                                   Interface interfaceState) {
 708         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 709         // Read NextHops
 710         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 711         Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
 712         if (!adjacencies.isPresent()) {
 713             addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 714                     null/*gwMac*/, writeOperTxn);
 715             return;
 716         }
 717
 718         // Get the rd of the vpn instance
 719         String nextHopIp = null;
 720         try {
 721             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 722         } catch (Exception e) {
 723             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 724                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 725         }
 726         List<String> nhList = new ArrayList<>();
 727         if (nextHopIp != null) {
 728             nhList.add(nextHopIp);
 729             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 730                     interfaceName, dpnId, vpnName, nhList);
 731         }
 732         Optional<String> gwMac = Optional.absent();
 733         String vpnInterfaceSubnetGwMacAddress = null;
 734         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
 735         Long l3vni = vpnInstanceOpData.getL3vni();
 736         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 737         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 738         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 739         List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 740         List<Adjacency> value = new ArrayList<>();
 741         for (Adjacency nextHop : nextHops) {
 742             String rd = primaryRd;
 743             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 744             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 745                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 746                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 747                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 748                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 749                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 750                 continue;
 751             }
 752             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 753                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 754                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 755                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 756                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 757                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 758                 writeOperTxn.merge(
 759                     LogicalDatastoreType.OPERATIONAL,
 760                     VpnUtil.getPrefixToInterfaceIdentifier(
 761                         VpnUtil.getVpnId(dataBroker, vpnName), prefix),
 762                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, nextHop.getSubnetId(),
 763                             prefixCue), true);
 764                 final Uuid subnetId = nextHop.getSubnetId();
 765
 766                 String gatewayIp = nextHop.getSubnetGatewayIp();
 767                 if (gatewayIp == null) {
 768                     Optional<String> gatewayIpOptional = VpnUtil.getVpnSubnetGatewayIp(dataBroker, subnetId);
 769                     if (gatewayIpOptional.isPresent()) {
 770                         gatewayIp = gatewayIpOptional.get();
 771                     }
 772                 }
 773
 774                 if (gatewayIp != null) {
 775                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 776                     if (gwMac.isPresent()) {
 777                         // A valid mac-address is available for this subnet-gateway-ip
 778                         // Use this for programming ARP_RESPONDER table here.  And save this
 779                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 780                         // to populate L3_GW_MAC_TABLE there.
 781                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 782                                 gatewayIp, gwMac.get());
 783                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 784                     } else {
 785                         // A valid mac-address is not available for this subnet-gateway-ip
 786                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 787                         // Save this connected-mac-address as gateway-mac-address for the
 788                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 789                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 790                         if (gwMac.isPresent()) {
 791                             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
 792                                     vpnId, writeInvTxn, NwConstants.ADD_FLOW, gwMac.get());
 793                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 794                                     gatewayIp, gwMac.get());
 795                         } else {
 796                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 797                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 798                                 + "gwIp {}",
 799                                 interfaceName, vpnName, gatewayIp);
 800                         }
 801                     }
 802                 } else {
 803                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 804                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 805                         subnetId, interfaceName, vpnName);
 806                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 807                 }
 808                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 809                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 810             } else {
 811                 //Extra route adjacency
 812                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 813                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 814                 synchronized (vpnPrefixKey.intern()) {
 815                     java.util.Optional<String> rdToAllocate = VpnUtil
 816                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(dataBroker, vpnId, null,
 817                             prefix, vpnName, nextHop.getNextHopIpList().get(0), dpnId);
 818                     if (rdToAllocate.isPresent()) {
 819                         rd = rdToAllocate.get();
 820                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 821                             rd, prefix);
 822                     } else {
 823                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 824                         continue;
 825                     }
 826                 }
 827                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 828                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 829                         interfaceName, dpnId);
 830             }
 831             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 832             // can be different from the gateway-mac-address within the VRFEntry as the
 833             // gateway-mac-address is a superset.
 834             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 835                     : RouteOrigin.STATIC;
 836             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 837                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 838                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 839             Adjacency operationalAdjacency = null;
 840             try {
 841                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 842             } catch (NullPointerException e) {
 843                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 844                     input, e.getMessage());
 845                 return;
 846             }
 847             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 848                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 849                         vpnName, l3vni, origin,
 850                         interfaceName, operationalAdjacency, encapType, writeConfigTxn);
 851             }
 852             value.add(operationalAdjacency);
 853         }
 854
 855         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 856         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 857                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 858
 859         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 860                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 861                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 862
 863         for (Adjacency nextHop : aug.getAdjacency()) {
 864             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 865             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 866                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 867                         : RouteOrigin.STATIC;
 868                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 869                 registeredPopulator.populateFib(input, writeConfigTxn);
 870             }
 871         }
 872     }
 873
 874     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 875                                               long lportTag, String gwMac, WriteTransaction writeOperTxn) {
 876         VpnInterfaceOpDataEntry opInterface =
 877               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, Boolean.FALSE, lportTag, gwMac);
 878         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 879             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 880         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 881                 WriteTransaction.CREATE_MISSING_PARENTS);
 882         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 883                 interfaceName, dpnId, vpnName);
 884     }
 885
 886     // TODO Clean up the exception handling
 887     @SuppressWarnings("checkstyle:IllegalCatch")
 888     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 889                                            StateTunnelList stateTunnelList,
 890                                            WriteTransaction writeConfigTxn,
 891                                            WriteTransaction writeOperTxn) {
 892
 893         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
 894         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 895         AdjacenciesOp adjacencies = vpnInterface.getAugmentation(AdjacenciesOp.class);
 896         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 897         if (adjList.isEmpty()) {
 898             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 899                     vpnInterface, srcDpnId);
 900             return;
 901         }
 902         String prefix = null;
 903         long label = 0;
 904         List<Adjacency> value = new ArrayList<>();
 905         boolean isNextHopAddReqd = false;
 906         String vpnName = vpnInterface.getVpnInstanceName();
 907         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
 908         String primaryRd = VpnUtil.getPrimaryRd(dataBroker, vpnName);
 909         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 910                 vpnInterface.getName(), vpnInterface.getDpnId(),
 911                 vpnInterface.getVpnInstanceName(), adjList);
 912         for (Adjacency adj : adjList) {
 913             String rd = adj.getVrfId();
 914             rd = rd != null ? rd : vpnName;
 915             prefix = adj.getIpAddress();
 916             label = adj.getLabel();
 917             List<String> nhList = Collections.singletonList(srcTepIp);
 918             List<String> nextHopList = adj.getNextHopIpList();
 919             // If TEP is added , update the nexthop of primary adjacency.
 920             // Secondary adj nexthop is already pointing to primary adj IP address.
 921             if (nextHopList != null && !nextHopList.isEmpty() && nextHopList.get(0).equalsIgnoreCase(srcTepIp)) {
 922                 /* everything right already */
 923             } else {
 924                 isNextHopAddReqd = true;
 925             }
 926
 927             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 928                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 929             } else {
 930                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 931                 if (!vrfEntryOptional.isPresent()) {
 932                     continue;
 933                 }
 934                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 935                 if (!nhList.contains(srcTepIp)) {
 936                     nhList.add(srcTepIp);
 937                     isNextHopAddReqd = true;
 938                 }
 939                 value.add(adj);
 940             }
 941
 942             if (isNextHopAddReqd) {
 943                 updateLabelMapper(label, nhList);
 944                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 945                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 946                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 947                 // Update the VRF entry with nextHop
 948                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 949                         label, true, writeConfigTxn);
 950
 951                 //Get the list of VPN's importing this route(prefix) .
 952                 // Then update the VRF entry with nhList
 953                 List<VpnInstanceOpDataEntry> vpnsToImportRoute =
 954                            VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
 955                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 956                     String vpnRd = vpn.getVrfId();
 957                     if (vpnRd != null) {
 958                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
 959                                 srcTepIp, label, true, writeConfigTxn);
 960                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 961                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
 962                             vpnInterface.getName(), srcDpnId, vpnName,
 963                             vpn.getVpnInstanceName(), vpnRd);
 964                     }
 965                 }
 966                 // Advertise the prefix to BGP only for external vpn
 967                 // since there is a nexthop change.
 968                 try {
 969                     if (!rd.equalsIgnoreCase(vpnName)) {
 970                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
 971                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
 972                                 null /*gatewayMacAddress*/);
 973                     }
 974                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
 975                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
 976                             srcDpnId, vpnName);
 977                 } catch (Exception ex) {
 978                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
 979                             + " on rd {} for interface {} on dpn {} vpn {} as {}", prefix, nhList, label, rd,
 980                             vpnInterface.getName(), srcDpnId,
 981                             vpnName, ex);
 982                 }
 983             }
 984         }
 985         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 986         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
 987                 .setKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
 988                 .addAugmentation(AdjacenciesOp.class, aug).build();
 989         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
 990                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
 991         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 992                 WriteTransaction.CREATE_MISSING_PARENTS);
 993         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
 994                 vpnInterface.getName(), srcDpnId, vpnName);
 995
 996     }
 997
 998     // TODO Clean up the exception handling
 999     @SuppressWarnings("checkstyle:IllegalCatch")
1000     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1001                                               StateTunnelList stateTunnelList,
1002                                               WriteTransaction writeConfigTxn,
1003                                               WriteTransaction writeOperTxn) {
1004
1005         AdjacenciesOp adjacencies = vpnInterface.getAugmentation(AdjacenciesOp.class);
1006         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1007         String prefix = null;
1008         long label = 0;
1009         boolean isNextHopRemoveReqd = false;
1010         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
1011         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
1012         String vpnName = vpnInterface.getVpnInstanceName();
1013         long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1014         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1015         if (adjList != null) {
1016             List<Adjacency> value = new ArrayList<>();
1017             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1018                     vpnInterface.getName(), vpnInterface.getDpnId(),
1019                     vpnInterface.getVpnInstanceName(), adjList);
1020             for (Adjacency adj : adjList) {
1021                 List<String> nhList = new ArrayList<>();
1022                 String rd = adj.getVrfId();
1023                 rd = rd != null ? rd : vpnName;
1024                 prefix = adj.getIpAddress();
1025                 List<String> nextHopList = adj.getNextHopIpList();
1026                 label = adj.getLabel();
1027                 if (nextHopList != null && !nextHopList.isEmpty()) {
1028                     isNextHopRemoveReqd = true;
1029                 }
1030                 // If TEP is deleted , remove the nexthop from primary adjacency.
1031                 // Secondary adj nexthop will continue to point to primary adj IP address.
1032                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1033                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1034                 } else {
1035                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1036                     if (!vrfEntryOptional.isPresent()) {
1037                         continue;
1038                     }
1039                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1040                     if (nhList.contains(srcTepIp)) {
1041                         nhList.remove(srcTepIp);
1042                         isNextHopRemoveReqd = true;
1043                     }
1044                     value.add(adj);
1045                 }
1046
1047                 if (isNextHopRemoveReqd) {
1048                     updateLabelMapper(label, nhList);
1049                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1050                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1051                             prefix, nhList, vpnName,
1052                             vpnId, rd, vpnInterface.getName());
1053                     // Update the VRF entry with removed nextHop
1054                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1055                             label, false, writeConfigTxn);
1056
1057                     //Get the list of VPN's importing this route(prefix) .
1058                     // Then update the VRF entry with nhList
1059                     List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1060                           VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1061                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1062                         String vpnRd = vpn.getVrfId();
1063                         if (vpnRd != null) {
1064                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1065                                     srcTepIp, label, false, writeConfigTxn);
1066                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1067                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1068                                     nhList, label, vpnInterface.getName(), srcDpnId,
1069                                     vpnName,
1070                                     vpn.getVpnInstanceName(), vpnRd);
1071                         }
1072                     }
1073
1074                     // Withdraw prefix from BGP only for external vpn.
1075                     try {
1076                         if (!rd.equalsIgnoreCase(vpnName)) {
1077                             bgpManager.withdrawPrefix(rd, prefix);
1078                         }
1079                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1080                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1081                                 vpnInterface.getName(), srcDpnId,
1082                                 vpnName);
1083                     } catch (Exception ex) {
1084                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1085                                 + " on rd {} for interface {} on dpn {} vpn {} as {}", prefix, nhList, label, rd,
1086                                 vpnInterface.getName(), srcDpnId,
1087                                 vpnName, ex);
1088                     }
1089                 }
1090             }
1091             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1092             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1093                     .setKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1094                     .addAugmentation(AdjacenciesOp.class, aug).build();
1095             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1096                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1097             writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
1098                     WriteTransaction.CREATE_MISSING_PARENTS);
1099             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1100                          vpnInterface.getName(), srcDpnId, vpnName);
1101         }
1102     }
1103
1104     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1105         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1106
1107         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1108         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
1109         if (vpnInstanceOpDataEntry == null) {
1110             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1111                     + " to check for vpns exporting the routes", vpnName);
1112             return vpnsToExportRoute;
1113         }
1114
1115         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1116             if (input.getVpnInstanceName() == null) {
1117                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1118                         input.getVrfId());
1119                 return false;
1120             }
1121             return !input.getVpnInstanceName().equals(vpnName);
1122         };
1123
1124         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1125             Iterable<String> commonRTs =
1126                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1127                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1128             return Iterators.size(commonRTs.iterator()) > 0;
1129         };
1130
1131         vpnsToExportRoute =
1132                 VpnUtil.getAllVpnInstanceOpData(dataBroker).stream().filter(excludeVpn).filter(matchRTs).collect(
1133                         Collectors.toList());
1134         return vpnsToExportRoute;
1135     }
1136
1137     // TODO Clean up the exception handling
1138     @SuppressWarnings("checkstyle:IllegalCatch")
1139     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1140         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1141         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1142             List<VrfEntry> vrfEntries = VpnUtil.getAllVrfEntries(dataBroker, vpn.getVrfId());
1143             WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1144             if (vrfEntries != null) {
1145                 for (VrfEntry vrfEntry : vrfEntries) {
1146                     try {
1147                         if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1148                                 RouteOrigin.value(vrfEntry.getOrigin()))) {
1149                             LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1150                                     + " is not a controller managed non intervpn link route. Ignoring.",
1151                                     vpn.getVrfId(), vrfEntry.getDestPrefix());
1152                             continue;
1153                         }
1154                         String prefix = vrfEntry.getDestPrefix();
1155                         String gwMac = vrfEntry.getGatewayMacAddress();
1156                         vrfEntry.getRoutePaths().forEach(routePath -> {
1157                             String nh = routePath.getNexthopAddress();
1158                             int label = routePath.getLabel().intValue();
1159                             if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1160                                     vrfEntry.getOrigin()))) {
1161                                 LOG.info("handleVpnsExportingRoutesImporting: Importing fib entry rd {} prefix {}"
1162                                         + " nexthop {} label {} to vpn {} vpnRd {}", vpn.getVrfId(), prefix, nh, label,
1163                                         vpnName, vpnRd);
1164                                 fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1165                                         Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1166                                         0 /*l3vni*/, gwMac,  null /*parentVpnRd*/, RouteOrigin.SELF_IMPORTED,
1167                                         writeConfigTxn);
1168                             } else {
1169                                 LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry rd {} prefix {}"
1170                                         + " nexthop {} label {} to vpn {} vpnRd {}", vpn.getVrfId(), prefix, nh, label,
1171                                         vpnName, vpnRd);
1172                                 SubnetRoute route = vrfEntry.getAugmentation(SubnetRoute.class);
1173                                 importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, writeConfigTxn);
1174                             }
1175                         });
1176                     } catch (RuntimeException e) {
1177                         LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1178                                 + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1179                                 vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1180                     }
1181                 }
1182                 writeConfigTxn.submit();
1183             } else {
1184                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1185                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1186             }
1187         }
1188     }
1189
1190     @SuppressWarnings("checkstyle:IllegalCatch")
1191     @Override
1192     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1193         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1194         final String interfaceName = key.getName();
1195         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
1196             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1197             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1198         }
1199     }
1200
1201     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1202                                 final VpnInterface vpnInterface, final String vpnName,
1203                                 final String interfaceName) {
1204         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1205             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName() + vpnName, () -> {
1206                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1207                 deleteFibEntryForRouterInterface(vpnInterface, writeConfigTxn, vpnName);
1208                 LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1209                 ListenableFuture<Void> futures = writeConfigTxn.submit();
1210                 String errorText = "removeVpnInterfaceCall: Exception encountered while submitting writeConfigTxn"
1211                         + " for interface " + vpnInterface.getName() + " on vpn " + vpnName;
1212                 ListenableFutures.addErrorLogging(futures, LOG, errorText);
1213                 return Collections.singletonList(futures);
1214             }, DJC_MAX_RETRIES);
1215         } else {
1216             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1217             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName, interfaceState);
1218         }
1219     }
1220
1221     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1222     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1223                                 final VpnInterface vpnInterface, final String vpnName,
1224                                 final String interfaceName, final Interface interfaceState) {
1225         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1226                 vpnName, vpnInterface.getDpnId());
1227         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1228         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName + vpnName,
1229             () -> {
1230                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1231                 ListenableFuture<Void> configFuture = txRunner
1232                         .callWithNewWriteOnlyTransactionAndSubmit(writeConfigTxn -> {
1233                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeOperTxn -> {
1234                                 futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeInvTxn -> {
1235                                     LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1236                                             interfaceName, vpnName);
1237                                     BigInteger dpId = BigInteger.ZERO;
1238                                     int ifIndex = 0;
1239                                     String gwMacAddress = null;
1240                                     InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1241                                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1242                                     final Optional<VpnInterfaceOpDataEntry> optVpnInterface =
1243                                             VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, interfaceId);
1244                                     if (interfaceState != null) {
1245                                         try {
1246                                             dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1247                                         } catch (NumberFormatException | IllegalStateException e) {
1248                                             LOG.error("remove: Unable to retrieve dpnId from interface operational"
1249                                                     + " data store for interface {} on dpn {} for vpn {} Fetching"
1250                                                             + " from vpn interface op data store. ", interfaceName,
1251                                                     vpnInterface.getDpnId(), vpnName, e);
1252                                             dpId = BigInteger.ZERO;
1253                                         }
1254                                         ifIndex = interfaceState.getIfIndex();
1255                                         gwMacAddress = interfaceState.getPhysAddress().getValue();
1256                                     } else {
1257                                         LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1258                                                 + " from vpn interface op.", interfaceName);
1259                                         if (optVpnInterface.isPresent()) {
1260                                             VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1261                                             dpId = vpnOpInterface.getDpnId();
1262                                             ifIndex = vpnOpInterface.getLportTag().intValue();
1263                                             gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1264                                         } else {
1265                                             LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1266                                                             + " as interfaceState and vpn interface op is not"
1267                                                     + " available", interfaceName, vpnName);
1268                                             return;
1269                                         }
1270                                     }
1271                                     processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1272                                             optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1273                                             writeConfigTxn, writeOperTxn, writeInvTxn);
1274                                     LOG.info(
1275                                             "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1276                                                     + "successfully",
1277                                             interfaceName, vpnInterface.getDpnId(), vpnName);
1278                                 }));
1279                             }));
1280                         });
1281                 futures.add(configFuture);
1282                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1283                 return futures;
1284             }, DJC_MAX_RETRIES);
1285     }
1286
1287     protected void processVpnInterfaceDown(BigInteger dpId,
1288                                            String interfaceName,
1289                                            int lportTag,
1290                                            String gwMac,
1291                                            VpnInterfaceOpDataEntry vpnOpInterface,
1292                                            boolean isInterfaceStateDown,
1293                                            WriteTransaction writeConfigTxn,
1294                                            WriteTransaction writeOperTxn,
1295                                            WriteTransaction writeInvTxn) {
1296         if (vpnOpInterface == null) {
1297             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1298                     + " as it is not available in operational data store", interfaceName, dpId);
1299             return;
1300         }
1301         final String vpnName = vpnOpInterface.getVpnInstanceName();
1302         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1303                                                     interfaceName, vpnName);
1304         if (!isInterfaceStateDown) {
1305             final long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1306             VpnUtil.scheduleVpnInterfaceForRemoval(dataBroker, interfaceName, dpId, vpnName, Boolean.TRUE,
1307                     null);
1308             final boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, vpnName);
1309             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1310                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1311             if (interfaceManager.isExternalInterface(interfaceName)) {
1312                 processExternalVpnInterface(interfaceName, vpnName, vpnId, dpId, lportTag, writeInvTxn,
1313                         NwConstants.DEL_FLOW);
1314             }
1315             if (!isBgpVpnInternetVpn) {
1316                 VpnUtil.unbindService(dataBroker, interfaceName, isInterfaceStateDown, jobCoordinator);
1317             }
1318             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1319                     + " successful", interfaceName, dpId, vpnName);
1320         } else {
1321             // Interface is retained in the DPN, but its Link Down.
1322             // Only withdraw the prefixes for this interface from BGP
1323             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn);
1324         }
1325     }
1326
1327     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1328                                           final String vpnName, final long vpnId, String gwMac,
1329                                           WriteTransaction writeConfigTxn, final WriteTransaction writeOperTxn,
1330                                           final WriteTransaction writeInvTxn) {
1331         //Read NextHops
1332         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1333                 .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1334         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1335         Optional<AdjacenciesOp> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1336
1337         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
1338         LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1339                 interfaceName, dpnId, vpnName, primaryRd);
1340         if (adjacencies.isPresent() && !adjacencies.get().getAdjacency().isEmpty()) {
1341             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1342             LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1343                     interfaceName, dpnId, vpnName, nextHops);
1344             for (Adjacency nextHop : nextHops) {
1345                 if (nextHop.isPhysNetworkFunc()) {
1346                     LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1347                             nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1348                     fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1349                             null/*writeCfgTxn*/);
1350                 } else {
1351                     String rd = nextHop.getVrfId();
1352                     List<String> nhList;
1353                     if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1354                         nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1355                     } else {
1356                         // This is a primary adjacency
1357                         nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1358                                 : Collections.emptyList();
1359                         removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1360                                 interfaceName, writeInvTxn);
1361                     }
1362                     if (!nhList.isEmpty()) {
1363                         if (rd.equals(vpnName)) {
1364                             //this is an internal vpn - the rd is assigned to the vpn instance name;
1365                             //remove from FIB directly
1366                             nhList.forEach((nh) -> removeAdjacencyFromInternalVpn(nextHop, vpnName,
1367                                     interfaceName, dpnId, writeConfigTxn));
1368                         } else {
1369                             removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd, interfaceName,
1370                                     writeConfigTxn);
1371                         }
1372                     } else {
1373                         LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1374                                         + " interface {}", nextHop.getIpAddress(), rd,
1375                                 nextHop.getAdjacencyType().toString(), interfaceName);
1376                         bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
1377                         fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1378                     }
1379                 }
1380                 String ip = nextHop.getIpAddress().split("/")[0];
1381                 LearntVpnVipToPort vpnVipToPort = VpnUtil.getLearntVpnVipToPort(dataBroker, vpnName, ip);
1382                 if (vpnVipToPort != null) {
1383                     VpnUtil.removeLearntVpnVipToPort(dataBroker, vpnName, ip);
1384                     LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed adjacency for Interface {}"
1385                                     + " ip {} on dpn {} for vpn {} from VpnPortData Entry", vpnVipToPort.getPortName(),
1386                             ip, dpnId, vpnName);
1387                 }
1388             }
1389         } else {
1390             // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1391             LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1392                     + " Removing it.", interfaceName, vpnName, dpnId);
1393             writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, identifier);
1394         }
1395     }
1396
1397     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName,
1398                                                                    String interfaceName, BigInteger dpnId,
1399                                                                    WriteTransaction writeConfigTxn) {
1400         return (nh) -> {
1401             fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1402                     writeConfigTxn);
1403             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1404                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1405                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1406         };
1407     }
1408
1409     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1410                                            BigInteger dpnId, String rd, String interfaceName,
1411                                            WriteTransaction writeConfigTxn) {
1412         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1413                 VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1414         nhList.forEach((nh) -> {
1415             //IRT: remove routes from other vpns importing it
1416             vpnManager.removePrefixFromBGP(primaryRd, rd, vpnName, nextHop.getIpAddress(),
1417                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn);
1418             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1419                 String vpnRd = vpn.getVrfId();
1420                 if (vpnRd != null) {
1421                     fibManager.removeOrUpdateFibEntry(vpnRd,
1422                             nextHop.getIpAddress(), nh, writeConfigTxn);
1423                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1424                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1425                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1426                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1427                 }
1428             }
1429         });
1430     }
1431
1432     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1433                                                  int lportTag, String gwMac, String interfaceName,
1434                                                  WriteTransaction writeInvTxn) {
1435         final Uuid subnetId = nextHop.getSubnetId();
1436         if (nextHop.getSubnetGatewayMacAddress() == null) {
1437             // A valid mac-address was not available for this subnet-gateway-ip
1438             // So a connected-mac-address was used for this subnet and we need
1439             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1440             VpnUtil.setupGwMacIfExternalVpn(dataBroker, mdsalManager, dpnId, interfaceName,
1441                     vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1442         }
1443         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1444                 subnetId);
1445     }
1446
1447     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1448                                                   String interfaceName) {
1449         // This is either an extra-route (or) a learned IP via subnet-route
1450         List<String> nhList = null;
1451         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1452         if (nextHopIp == null || nextHopIp.isEmpty()) {
1453             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1454                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1455                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1456                     vpnName);
1457             nhList = Collections.emptyList();
1458         } else {
1459             nhList = Collections.singletonList(nextHopIp);
1460         }
1461         return nhList;
1462     }
1463
1464     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1465         VpnPortipToPort gwPort = VpnUtil.getNeutronPortFromVpnPortFixedIp(dataBroker, vpnName, ipAddress);
1466         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1467         // else use connected interface
1468         if (gwPort != null && gwPort.isSubnetIp()) {
1469             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1470                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1471             return Optional.of(gwPort.getMacAddress());
1472         }
1473         return Optional.absent();
1474     }
1475
1476     @Override
1477     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1478         final VpnInterface update) {
1479         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}" ,update.getName(),
1480                 update.getDpnId(), original.getVpnInstanceNames(),
1481                 update.getVpnInstanceNames());
1482         final String vpnInterfaceName = update.getName();
1483         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1484         final Adjacencies origAdjs = original.getAugmentation(Adjacencies.class);
1485         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1486             != null ? origAdjs.getAdjacency() : new ArrayList<>();
1487         final Adjacencies updateAdjs = update.getAugmentation(Adjacencies.class);
1488         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1489             != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1490
1491         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1492         //handles switching between <internal VPN - external VPN>
1493         for (VpnInstanceNames vpnInterfaceVpnInstance : original.getVpnInstanceNames()) {
1494             String oldVpnName = vpnInterfaceVpnInstance.getVpnName();
1495             if (oldVpnName != null && (update.getVpnInstanceNames() == null
1496                 || !VpnHelper.doesVpnInterfaceBelongToVpnInstance(oldVpnName, update.getVpnInstanceNames()))) {
1497                 UpdateData updateData = new UpdateData(identifier, original, update);
1498                 vpnInterfacesUpdateQueue.add(updateData);
1499                 LOG.info("update: UpdateData on VPNInterface {} on dpn {} update upon VPN swap from oldVpn(s) {}"
1500                         + "to newVpn(s) {} added to update queue",
1501                         updateData.getOriginal().getName(), dpnId,
1502                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1503                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1504                 return;
1505             }
1506         }
1507         for (VpnInstanceNames vpnInterfaceVpnInstance : update.getVpnInstanceNames()) {
1508             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1509             String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName);
1510             if (newVpnName != null && (original.getVpnInstanceNames() == null
1511                 || !VpnHelper.doesVpnInterfaceBelongToVpnInstance(newVpnName,
1512                                 original.getVpnInstanceNames()))) {
1513                 if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
1514                     InstanceIdentifier<VpnInterfaceOpDataEntry> opIdentifier = VpnUtil
1515                         .getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1516                     if (canHandleNewVpnInterface(identifier, update, newVpnName)) {
1517                         List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1518                         List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1519                         addVpnInterfaceToVpn(opIdentifier, update, copyOldAdjs, copyNewAdjs, identifier, newVpnName);
1520                     }
1521                 } else {
1522                     UpdateData updateData = new UpdateData(identifier, original, update);
1523                     vpnInterfacesUpdateQueue.add(updateData);
1524                     LOG.info("update: UpdateData on VPNInterface {} on dpn {} update upon VPN swap from oldVpn(s) {}"
1525                             + "to newVpn(s) {} added to update queue",
1526                             updateData.getOriginal().getName(), dpnId,
1527                             VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1528                             VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1529                     return;
1530                 }
1531             }
1532         }
1533         for (VpnInstanceNames vpnInterfaceVpnInstance : update.getVpnInstanceNames()) {
1534             String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1535             List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1536             List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1537             String primaryRd = VpnUtil.getPrimaryRd(dataBroker, newVpnName);
1538             if (!VpnUtil.isVpnPendingDelete(dataBroker, primaryRd)) {
1539                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName + newVpnName, () -> {
1540                     WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
1541                     WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
1542                     InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1543                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1544                     LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven",
1545                             update.getName(), newVpnName);
1546                     //handle both addition and removal of adjacencies
1547                     //currently, new adjacency may be an extra route
1548                     boolean isBgpVpnInternetVpn = VpnUtil.isBgpVpnInternet(dataBroker, newVpnName);
1549                     if (!oldAdjs.equals(newAdjs)) {
1550                         for (Adjacency adj : copyNewAdjs) {
1551                             if (copyOldAdjs.contains(adj)) {
1552                                 copyOldAdjs.remove(adj);
1553                             } else {
1554                                 // add new adjacency - right now only extra route will hit this path
1555                                 if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker, adj)) {
1556                                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1557                                              dpnId, writeOperTxn, writeConfigTxn);
1558                                 }
1559                                 LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to vpn "
1560                                                 + "interface {} on vpn {} dpnId {}",
1561                                         adj.getIpAddress(), adj.getNextHopIpList(),
1562                                         adj.getLabel(), adj.getSubnetId(), update.getName(),
1563                                         newVpnName, dpnId);
1564                             }
1565                         }
1566                         for (Adjacency adj : copyOldAdjs) {
1567                             if (!isBgpVpnInternetVpn || VpnUtil.isAdjacencyEligibleToVpnInternet(dataBroker, adj)) {
1568                                 delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1569                                        writeOperTxn, writeConfigTxn);
1570                             }
1571                             LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1572                                 + " vpn interface {} on vpn {}", adj.getIpAddress(), adj.getNextHopIpList(),
1573                                 adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1574                         }
1575                     }
1576                     ListenableFuture<Void> operFuture = writeOperTxn.submit();
1577                     try {
1578                         operFuture.get();
1579                     } catch (ExecutionException e) {
1580                         LOG.error("Exception encountered while submitting operational future for update"
1581                                 + " VpnInterface {} on vpn {}: {}", vpnInterfaceName, newVpnName, e);
1582                         return null;
1583                     }
1584                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1585                     futures.add(writeConfigTxn.submit());
1586                     LOG.info("update: vpn interface updated for interface {} oldVpn(s) {} newVpn {}"
1587                         + "processed successfully", update.getName(),
1588                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()), newVpnName);
1589                     return futures;
1590                 });
1591             } else {
1592                 LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1593                         + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1594             }
1595         }
1596     }
1597
1598     @SuppressWarnings("checkstyle:IllegalCatch")
1599     private class VpnInterfaceUpdateTimerTask implements Runnable {
1600         private final Logger log = LoggerFactory.getLogger(VpnInterfaceUpdateTimerTask.class);
1601
1602         @Override
1603         public void run() {
1604             try {
1605                 doRun();
1606             } catch (ReadFailedException | RuntimeException e) {
1607                 LOG.error("VpnInterfaceUpdateTimerTask failed", e);
1608             }
1609         }
1610
1611         private void doRun() throws ReadFailedException {
1612             List<UpdateData> processQueue = new ArrayList<>();
1613             List<UpdateData> updateDataList = new ArrayList<>();
1614             vpnInterfacesUpdateQueue.drainTo(processQueue);
1615             int maxInterfaceList = 0;
1616
1617             for (UpdateData updData : processQueue) {
1618                 final VpnInterfaceKey key = updData.getIdentifier()
1619                     .firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1620                 final String interfaceName = key.getName();
1621                 Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
1622                 for (VpnInstanceNames vpnInterfaceVpnInstance : updData.getOriginal().getVpnInstanceNames()) {
1623                     String oldVpnName = vpnInterfaceVpnInstance.getVpnName();
1624                     if (updData.getUpdate().getVpnInstanceNames() != null
1625                         && VpnHelper.doesVpnInterfaceBelongToVpnInstance(oldVpnName,
1626                                     updData.getUpdate().getVpnInstanceNames())) {
1627                         continue;
1628                     }
1629                     log.info("run: VPN Interface update event - intfName {} remove vpnName {} running"
1630                         + " config-driven swap removal", updData.getOriginal().getName(), oldVpnName);
1631                     maxInterfaceList ++;
1632                     removeVpnInterfaceCall(updData.getIdentifier(), updData.getOriginal(),
1633                                        oldVpnName, interfaceName);
1634                     log.info("run: Processed Remove for update on VPNInterface {} upon VPN swap from old vpn {}"
1635                             + " to newVpn(s) {}", updData.getOriginal().getName(),
1636                             oldVpnName, VpnHelper.getVpnInterfaceVpnInstanceNamesString(updData
1637                                        .getUpdate().getVpnInstanceNames()));
1638                 }
1639                 updateDataList.add(updData);
1640             }
1641             /* Decide the max-wait time based on number of VpnInterfaces.
1642             *  max-wait-time is num-of-interface * 4seconds (random choice).
1643             *  Every 2sec poll VpnToDpnList. If VpnInterface is removed ,
1644             *  remove it from vpnInterfaceList.
1645             */
1646             int maxWaitTime =
1647                 maxInterfaceList * (int) (VpnConstants.PER_INTERFACE_MAX_WAIT_TIME_IN_MILLISECONDS / 1000);
1648             int waitTime = 2;
1649             Iterator<UpdateData> updateDataIterator = updateDataList.iterator();
1650             UpdateData updateDataSet;
1651             while (waitTime < maxWaitTime) {
1652                 try {
1653                     Thread.sleep(2000); // sleep for 2sec
1654                 } catch (InterruptedException e) {
1655                     // Ignored
1656                 }
1657
1658                 while (updateDataIterator.hasNext()) {
1659                     boolean interfaceIsRemoved = true;
1660                     updateDataSet = updateDataIterator.next();
1661                     for (VpnInstanceNames vpnInterfaceVpnInstance : updateDataSet.getOriginal().getVpnInstanceNames()) {
1662                         String oldVpnName = vpnInterfaceVpnInstance.getVpnName();
1663                         if (oldVpnName != null && updateDataSet.getUpdate().getVpnInstanceNames() != null
1664                             && VpnHelper.doesVpnInterfaceBelongToVpnInstance(oldVpnName,
1665                                     updateDataSet.getUpdate().getVpnInstanceNames())) {
1666                             continue;
1667                         }
1668
1669                         boolean isPresent = isVpnIntfPresentInVpnToDpnList(updateDataSet.getOriginal(), oldVpnName);
1670                         if (isPresent) {
1671                             interfaceIsRemoved = false;
1672                         }
1673                     }
1674                     if (interfaceIsRemoved) {
1675                         updateDataIterator.remove();
1676                     }
1677                 }
1678                 if (updateDataList.isEmpty()) {
1679                     log.info("run: All VpnInterfaces are successfully removed from OLD VPN after time {}", waitTime);
1680                     break;
1681                 }
1682                 waitTime += 2; //Increment linearly by 2sec.
1683             }
1684
1685             if (updateDataList.size() > 0) {
1686                 log.error("run: VpnInterfacesList not removed from old Vpn even after waiting {}", waitTime);
1687             }
1688             for (UpdateData updData : processQueue) {
1689                 if (updateDataList.contains(updData)) {
1690                     log.error("run: Failed to swap VpnInterface {} from oldVpn {} to target VPN {}"
1691                              + "as it has not been cleaned up from the oldVpn", updData.getOriginal().getName(),
1692                              VpnHelper.getVpnInterfaceVpnInstanceNamesString(updData.getOriginal()
1693                                             .getVpnInstanceNames()),
1694                              VpnHelper.getVpnInterfaceVpnInstanceNamesString(updData.getUpdate()
1695                                             .getVpnInstanceNames()));
1696                     continue;
1697                 }
1698                 for (VpnInstanceNames vpnInterfaceVpnInstance : updData.getUpdate().getVpnInstanceNames()) {
1699                     String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1700                     if (updData.getOriginal().getVpnInstanceNames() != null
1701                         && VpnHelper.doesVpnInterfaceBelongToVpnInstance(newVpnName,
1702                                               updData.getOriginal().getVpnInstanceNames())) {
1703                         continue;
1704                     }
1705                     log.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven"
1706                            + " swap addition", updData.getUpdate().getName(), newVpnName);
1707                     final Adjacencies origAdjs = updData.getOriginal().getAugmentation(Adjacencies.class);
1708                     final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency() != null
1709                         ? origAdjs.getAdjacency() : new ArrayList<>();
1710                     final Adjacencies updateAdjs = updData.getUpdate().getAugmentation(Adjacencies.class);
1711                     final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency() != null
1712                         ? updateAdjs.getAdjacency() : new ArrayList<>();
1713
1714                     addVpnInterfaceCall(updData.getIdentifier(), updData.getUpdate(),
1715                                       oldAdjs, newAdjs, newVpnName);
1716                     log.info("run: Processed Add for update on VPNInterface {} from oldVpn(s) {} to newVpn {}"
1717                             + " upon VPN swap", updData.getUpdate().getName(),
1718                             VpnHelper.getVpnInterfaceVpnInstanceNamesString(updData.getOriginal()
1719                                        .getVpnInstanceNames()), newVpnName);
1720                 }
1721             }
1722         }
1723
1724         private boolean isVpnIntfPresentInVpnToDpnList(VpnInterface vpnInterface, String vpnName)
1725                 throws ReadFailedException {
1726             BigInteger dpnId = vpnInterface.getDpnId();
1727             String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
1728             log.trace("isVpnIntfPresentInVpnToDpnList: GOT rd {} for VpnInterface {}  VpnInstance {} ", rd ,
1729                     vpnInterface.getName(), vpnName);
1730
1731             if (rd == null) {
1732                 return false;
1733             }
1734
1735             Optional<VpnInstanceOpDataEntry> maybeVpnInstanceOpData = vpnInstanceOpDataEntryCache.get(
1736                     VpnUtil.getVpnInstanceOpDataIdentifier(rd));
1737             if (maybeVpnInstanceOpData.isPresent()) {
1738                 log.trace("isVpnIntfPresentInVpnToDpnList: GOT VpnInstanceOp {} for rd {} ",
1739                         maybeVpnInstanceOpData.get(), rd);
1740                 List<VpnToDpnList> dpnToVpns = maybeVpnInstanceOpData.get().getVpnToDpnList();
1741                 if (dpnToVpns != null) {
1742                     for (VpnToDpnList dpn : dpnToVpns) {
1743                         if (dpn.getDpnId().equals(dpnId)) {
1744                             return dpn.getVpnInterfaces().stream().anyMatch(
1745                                 vpnInterfaces -> vpnInterface.getName().equals(vpnInterfaces.getInterfaceName()));
1746                         }
1747                         log.info("isVpnIntfPresentInVpnToDpnList: VpnInterface {} not present in DpnId {} vpn {}",
1748                                 vpnInterface.getName(), dpn.getDpnId(), vpnName);
1749                     }
1750                 }
1751             }
1752
1753             return false;
1754         }
1755     }
1756
1757     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1758         Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1759         synchronized (label.toString().intern()) {
1760             InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1761                     .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1762             Optional<LabelRouteInfo> opResult = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid);
1763             if (opResult.isPresent()) {
1764                 LabelRouteInfo labelRouteInfo =
1765                     new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1766                 MDSALUtil.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid, labelRouteInfo);
1767             }
1768         }
1769         LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label, nextHopIpList);
1770     }
1771
1772     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1773         SubnetRoute route, WriteTransaction writeConfigTxn) {
1774
1775         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1776         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, null /* parentVpnRd */)
1777                 .addAugmentation(SubnetRoute.class, route).build();
1778         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1779         InstanceIdentifierBuilder<VrfTables> idBuilder =
1780             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1781         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1782         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1783         if (writeConfigTxn != null) {
1784             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1785         } else {
1786             VpnUtil.syncUpdate(dataBroker, LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1787         }
1788         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1789                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1790     }
1791
1792     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1793                                            Adjacency adj, BigInteger dpnId, WriteTransaction writeOperTxn,
1794                                            WriteTransaction writeConfigTxn) {
1795
1796         Optional<VpnInterfaceOpDataEntry> optVpnInterface = VpnUtil.read(dataBroker,
1797                                                 LogicalDatastoreType.OPERATIONAL, identifier);
1798
1799         if (optVpnInterface.isPresent()) {
1800             VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1801             String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1802             String vpnName = currVpnIntf.getVpnInstanceName();
1803             VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
1804             InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1805             Optional<AdjacenciesOp> optAdjacencies = VpnUtil.read(dataBroker,
1806                                             LogicalDatastoreType.OPERATIONAL, adjPath);
1807             boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1808             VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1809             long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1810             VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1811             List<Adjacency> adjacencies;
1812             if (optAdjacencies.isPresent()) {
1813                 adjacencies = optAdjacencies.get().getAdjacency();
1814             } else {
1815                 // This code will be hit in case of first PNF adjacency
1816                 adjacencies = new ArrayList<>();
1817             }
1818             long vpnId = VpnUtil.getVpnId(dataBroker, vpnName);
1819             L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1820                     .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1821             Adjacency operationalAdjacency = null;
1822             if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()) {
1823                 RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
1824                         : RouteOrigin.STATIC;
1825                 String nh = adj.getNextHopIpList().get(0);
1826                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1827                 synchronized (vpnPrefixKey.intern()) {
1828                     java.util.Optional<String> rdToAllocate = VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1829                             dataBroker, vpnId, null, prefix, vpnName, nh, dpnId);
1830                     if (rdToAllocate.isPresent()) {
1831                         input.setRd(rdToAllocate.get());
1832                         operationalAdjacency = populator.createOperationalAdjacency(input);
1833                         int label = operationalAdjacency.getLabel().intValue();
1834                         vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1835                                 currVpnIntf.getVpnInstanceName(), l3vni, origin,
1836                                 currVpnIntf.getName(), operationalAdjacency, encapType, writeConfigTxn);
1837                         LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1838                                 + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1839                                 vpnName, label, currVpnIntf.getName(), dpnId);
1840                     } else {
1841                         LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}", vpnName,
1842                                 prefix);
1843                         return;
1844                     }
1845                     // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1846                     // Keeping the MPLS check for now.
1847                     if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1848                         final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1849                         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1850                                 VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1851                         vpnsToImportRoute.forEach(vpn -> {
1852                             if (vpn.getVrfId() != null) {
1853                                 VpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1854                                         dataBroker, vpn.getVpnId(), vpnId, prefix,
1855                                         VpnUtil.getVpnName(dataBroker, vpn.getVpnId()), nh, dpnId)
1856                                         .ifPresent(
1857                                             rds -> vpnManager.addExtraRoute(
1858                                                     VpnUtil.getVpnName(dataBroker, vpn.getVpnId()),
1859                                                     adj.getIpAddress(), nh, rds,
1860                                                     currVpnIntf.getVpnInstanceName(),
1861                                                     l3vni, RouteOrigin.SELF_IMPORTED,
1862                                                     currVpnIntf.getName(), opAdjacency, encapType, writeConfigTxn));
1863                             }
1864                         });
1865                     }
1866                 }
1867             } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1868                 LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1869                         currVpnIntf.getName(), vpnName);
1870
1871                 String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1872
1873                 writeOperTxn.merge(
1874                         LogicalDatastoreType.OPERATIONAL,
1875                         VpnUtil.getPrefixToInterfaceIdentifier(VpnUtil.getVpnId(dataBroker,
1876                                 adj.getSubnetId().getValue()), prefix),
1877                         VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1878                                 adj.getSubnetId(), Prefixes.PrefixCue.PhysNetFunc), true);
1879
1880                 fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1881                         adj.getIpAddress(), Collections.emptyList(), null /* EncapType */, 0 /* label */, 0 /*l3vni*/,
1882                       null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1883
1884                 input.setRd(adj.getVrfId());
1885             }
1886             if (operationalAdjacency == null) {
1887                 operationalAdjacency = populator.createOperationalAdjacency(input);
1888             }
1889             adjacencies.add(operationalAdjacency);
1890             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1891             VpnInterfaceOpDataEntry newVpnIntf =
1892                 VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
1893                     aug, dpnId, currVpnIntf.isScheduledForRemove(), currVpnIntf.getLportTag(),
1894                         currVpnIntf.getGatewayMacAddress());
1895
1896             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1897         }
1898     }
1899
1900     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
1901         Subnets subnets = VpnUtil.getExternalSubnet(dataBroker, adj.getSubnetId());
1902         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
1903     }
1904
1905     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
1906             BigInteger dpnId, WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) {
1907         Optional<VpnInterfaceOpDataEntry> optVpnInterface = VpnUtil.read(dataBroker,
1908                                           LogicalDatastoreType.OPERATIONAL, identifier);
1909
1910         if (optVpnInterface.isPresent()) {
1911             VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1912
1913             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1914             Optional<AdjacenciesOp> optAdjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.OPERATIONAL, path);
1915             if (optAdjacencies.isPresent()) {
1916                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1917
1918                 if (!adjacencies.isEmpty()) {
1919                     LOG.trace("delAdjFromVpnInterface: Adjacencies are " + adjacencies);
1920                     Iterator<Adjacency> adjIt = adjacencies.iterator();
1921                     while (adjIt.hasNext()) {
1922                         Adjacency adjElem = adjIt.next();
1923                         if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1924                             String rd = adjElem.getVrfId();
1925                             adjIt.remove();
1926
1927                             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1928                             VpnInterfaceOpDataEntry newVpnIntf = VpnUtil
1929                                     .getVpnInterfaceOpDataEntry(currVpnIntf.getName(),
1930                                     currVpnIntf.getVpnInstanceName(), aug, dpnId, currVpnIntf.isScheduledForRemove(),
1931                                             currVpnIntf.getLportTag(), currVpnIntf.getGatewayMacAddress());
1932
1933                             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1934                             if (adj.getNextHopIpList() != null) {
1935                                 for (String nh : adj.getNextHopIpList()) {
1936                                     deleteExtraRouteFromCurrentAndImportingVpns(
1937                                         currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
1938                                         currVpnIntf.getName(), writeConfigTxn);
1939                                 }
1940                             } else if (adj.isPhysNetworkFunc()) {
1941                                 LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet [}",
1942                                         adj.getIpAddress(), adj.getSubnetId());
1943                                 fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
1944                                         writeConfigTxn);
1945                             }
1946                             break;
1947                         }
1948
1949                     }
1950                 }
1951                 LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
1952                         dpnId, adj.getVrfId());
1953             } else {
1954                 LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
1955                         + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
1956             }
1957         }
1958     }
1959
1960     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
1961         String rd, String intfName, WriteTransaction writeConfigTxn) {
1962         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn);
1963         List<VpnInstanceOpDataEntry> vpnsToImportRoute = VpnUtil.getVpnsImportingMyRoute(dataBroker, vpnName);
1964         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1965             String vpnRd = vpn.getVrfId();
1966             if (vpnRd != null) {
1967                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn);
1968             }
1969         }
1970     }
1971
1972     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1973         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1974             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1975             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1976     }
1977
1978     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1979         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1980             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1981     }
1982
1983     protected void addToNeutronRouterDpnsMap(String routerName, String vpnInterfaceName,
1984         WriteTransaction writeOperTxn) {
1985         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1986         if (dpId.equals(BigInteger.ZERO)) {
1987             LOG.error("addToNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
1988                     + " association model", vpnInterfaceName, routerName);
1989
1990             return;
1991         }
1992         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
1993
1994         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
1995             .OPERATIONAL, routerDpnListIdentifier);
1996         RouterInterfaces routerInterface =
1997             new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
1998                 vpnInterfaceName).build();
1999         if (optionalRouterDpnList.isPresent()) {
2000             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
2001                 RouterInterfaces.class, new RouterInterfacesKey(vpnInterfaceName)), routerInterface, true);
2002         } else {
2003             RouterDpnListBuilder builder = new RouterDpnListBuilder();
2004             builder.setRouterId(routerName);
2005             DpnVpninterfacesListBuilder dpnVpnList = new DpnVpninterfacesListBuilder().setDpnId(dpId);
2006             builder.setDpnVpninterfacesList(Collections.singletonList(dpnVpnList.build()));
2007             writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL,
2008                 getRouterId(routerName),
2009                 builder.build(), true);
2010         }
2011     }
2012
2013     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName,
2014         WriteTransaction writeOperTxn) {
2015         BigInteger dpId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
2016         if (dpId.equals(BigInteger.ZERO)) {
2017             LOG.error("removeFromNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
2018                     + " dissociation model", vpnInterfaceName, routerName);
2019
2020             return;
2021         }
2022         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
2023         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
2024             .OPERATIONAL, routerDpnListIdentifier);
2025         if (optionalRouterDpnList.isPresent()) {
2026             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
2027             RouterInterfaces routerInterface =
2028                 new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
2029                     vpnInterfaceName).build();
2030
2031             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
2032                 if (routerInterfaces.isEmpty()) {
2033                     if (writeOperTxn != null) {
2034                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
2035                     } else {
2036                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
2037                     }
2038                 } else {
2039                     if (writeOperTxn != null) {
2040                         writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
2041                             RouterInterfaces.class,
2042                             new RouterInterfacesKey(vpnInterfaceName)));
2043                     } else {
2044                         MDSALUtil.syncDelete(dataBroker, LogicalDatastoreType.OPERATIONAL,
2045                             routerDpnListIdentifier.child(
2046                                 RouterInterfaces.class,
2047                                 new RouterInterfacesKey(vpnInterfaceName)));
2048                     }
2049                 }
2050             }
2051         }
2052     }
2053
2054     protected void removeFromNeutronRouterDpnsMap(String routerName, String vpnInterfaceName, BigInteger dpId,
2055         WriteTransaction writeOperTxn) {
2056         if (dpId.equals(BigInteger.ZERO)) {
2057             LOG.error("removeFromNeutronRouterDpnsMap: Could not retrieve dp id for interface {} to handle router {}"
2058                     + " dissociation model", vpnInterfaceName, routerName);
2059
2060             return;
2061         }
2062         InstanceIdentifier<DpnVpninterfacesList> routerDpnListIdentifier = getRouterDpnId(routerName, dpId);
2063         Optional<DpnVpninterfacesList> optionalRouterDpnList = VpnUtil.read(dataBroker, LogicalDatastoreType
2064             .OPERATIONAL, routerDpnListIdentifier);
2065         if (optionalRouterDpnList.isPresent()) {
2066             List<RouterInterfaces> routerInterfaces = optionalRouterDpnList.get().getRouterInterfaces();
2067             RouterInterfaces routerInterface =
2068                 new RouterInterfacesBuilder().setKey(new RouterInterfacesKey(vpnInterfaceName)).setInterface(
2069                     vpnInterfaceName).build();
2070             if (routerInterfaces != null && routerInterfaces.remove(routerInterface)) {
2071                 if (routerInterfaces.isEmpty()) {
2072                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier);
2073                 } else {
2074                     writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, routerDpnListIdentifier.child(
2075                         RouterInterfaces.class,
2076                         new RouterInterfacesKey(vpnInterfaceName)));
2077                 }
2078             }
2079         }
2080     }
2081
2082     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
2083                                                     WriteTransaction writeConfigTxn, String vpnName) {
2084         if (vpnInterface == null) {
2085             return;
2086         }
2087         List<Adjacency> adjs = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker, interfaceName);
2088         if (adjs == null) {
2089             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
2090                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
2091             return;
2092         }
2093         for (Adjacency adj : adjs) {
2094             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2095                 String primaryInterfaceIp = adj.getIpAddress();
2096                 String macAddress = adj.getMacAddress();
2097                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2098
2099                 long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME,
2100                     VpnUtil.getNextHopLabelKey(primaryRd, prefix));
2101
2102                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
2103                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
2104                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
2105                         routerInt, label, writeConfigTxn);
2106                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
2107                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
2108                         macAddress);
2109                 return;
2110             }
2111         }
2112         LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
2113                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}", interfaceName,
2114                 primaryRd, vpnName);
2115     }
2116
2117     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
2118                                                     WriteTransaction writeConfigTxn, String vpnName) {
2119         Adjacencies adjs = vpnInterface.getAugmentation(Adjacencies.class);
2120         String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
2121         if (adjs != null) {
2122             List<Adjacency> adjsList = adjs.getAdjacency();
2123             for (Adjacency adj : adjsList) {
2124                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
2125                     String primaryInterfaceIp = adj.getIpAddress();
2126                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
2127                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
2128                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
2129                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
2130                     return;
2131                 }
2132             }
2133         } else {
2134             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
2135                     vpnInterface.getName(), rd);
2136         }
2137     }
2138
2139     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2140         if (!canHandleNewVpnInterface(intefaceData.identifier, intefaceData.vpnInterface, vpnName)) {
2141             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
2142                   vpnName, intefaceData.vpnInterface.getName());
2143             return;
2144         }
2145         final VpnInterfaceKey key = intefaceData.identifier
2146                .firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
2147         final String interfaceName = key.getName();
2148         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2149                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2150         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2151                   intefaceData.identifier, vpnName);
2152         return;
2153     }
2154
2155     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2156                                               VpnInterface vpnInterface, String vpnName) {
2157         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2158                .get(vpnName);
2159         if (vpnInterfaces == null) {
2160             vpnInterfaces = new ConcurrentLinkedQueue<>();
2161         }
2162         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2163         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2164         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2165                 vpnInterface.getName(), vpnName);
2166     }
2167
2168     public boolean isVpnInstanceReady(String vpnInstanceName) {
2169         String vpnRd = VpnUtil.getVpnRd(dataBroker, vpnInstanceName);
2170         if (vpnRd == null) {
2171             return false;
2172         }
2173         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = VpnUtil.getVpnInstanceOpData(dataBroker, vpnRd);
2174
2175         return vpnInstanceOpDataEntry != null;
2176     }
2177
2178     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2179         synchronized (vpnInstanceName.intern()) {
2180             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2181                     unprocessedVpnInterfaces.get(vpnInstanceName);
2182             if (vpnInterfaces != null) {
2183                 while (!vpnInterfaces.isEmpty()) {
2184                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2185                     if (hasVpnInstanceCreatedSuccessfully) {
2186                         processSavedInterface(savedInterface, vpnInstanceName);
2187                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2188                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2189                     } else {
2190                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2191                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2192                     }
2193                 }
2194             } else {
2195                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2196             }
2197         }
2198     }
2199
2200     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2201             VpnInterface vpnInterface) {
2202         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2203             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2204                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2205             if (vpnInterfaces != null) {
2206                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2207                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2208                             + "unprocessed list", vpnInterface.getName(),
2209                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2210                 }
2211             } else {
2212                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2213                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2214             }
2215         }
2216     }
2217
2218     public void vpnInstanceIsReady(String vpnInstanceName) {
2219         processSavedInterfaces(vpnInstanceName, true);
2220     }
2221
2222     public void vpnInstanceFailed(String vpnInstanceName) {
2223         processSavedInterfaces(vpnInstanceName, false);
2224     }
2225
2226     private static class UnprocessedVpnInterfaceData {
2227         InstanceIdentifier<VpnInterface> identifier;
2228         VpnInterface vpnInterface;
2229
2230         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2231             this.identifier = identifier;
2232             this.vpnInterface = vpnInterface;
2233         }
2234
2235         @Override
2236         public int hashCode() {
2237             final int prime = 31;
2238             int result = 1;
2239             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2240             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2241             return result;
2242         }
2243
2244         @Override
2245         public boolean equals(Object obj) {
2246             if (this == obj) {
2247                 return true;
2248             }
2249             if (obj == null) {
2250                 return false;
2251             }
2252             if (getClass() != obj.getClass()) {
2253                 return false;
2254             }
2255             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2256             if (identifier == null) {
2257                 if (other.identifier != null) {
2258                     return false;
2259                 }
2260             } else if (!identifier.equals(other.identifier)) {
2261                 return false;
2262             }
2263             if (vpnInterface == null) {
2264                 if (other.vpnInterface != null) {
2265                     return false;
2266                 }
2267             } else if (!vpnInterface.equals(other.vpnInterface)) {
2268                 return false;
2269             }
2270             return true;
2271         }
2272     }
2273
2274     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2275         String primaryRd = VpnUtil.getVpnRd(dataBroker, vpnName);
2276         VpnInstanceOpDataEntry vpnInstanceOpData = VpnUtil.getVpnInstanceOpData(dataBroker, primaryRd);
2277         if (vpnInstanceOpData == null) {
2278             return;
2279         }
2280         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2281         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2282             return;
2283         }
2284         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2285         vpnToDpnLists.forEach(vpnToDpnList -> vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2286             InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2287                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2288             Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = VpnUtil.read(dataBroker,
2289                     LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2290             if (!vpnInterfaceOptional.isPresent()) {
2291                 return;
2292             }
2293             List<Adjacency> configVpnAdjacencies = VpnUtil.getAdjacenciesForVpnInterfaceFromConfig(dataBroker,
2294                     vpnInterface.getInterfaceName());
2295             if (configVpnAdjacencies == null) {
2296                 LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2297                 return;
2298             }
2299             List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2300                     .getAugmentation(AdjacenciesOp.class).getAdjacency();
2301             // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2302             // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2303             // in operational DS. These unprocessed adjacencies will be handled below.
2304             // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in operational
2305             // DS are retrieved which is used to call addNewAdjToVpnInterface method.
2306             configVpnAdjacencies.stream()
2307                     .filter(adjacency -> operationVpnAdjacencies.stream()
2308                             .noneMatch(operationalAdjacency ->
2309                                     operationalAdjacency.getIpAddress().equals(adjacency.getIpAddress())))
2310                     .forEach(adjacency -> {
2311                         LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2312                         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName() + vpnName,
2313                             () -> {
2314                                 WriteTransaction writeConfigTxn = dataBroker.newWriteOnlyTransaction();
2315                                 WriteTransaction writeOperTxn = dataBroker.newWriteOnlyTransaction();
2316                                 if (VpnUtil.isAdjacencyEligibleToVpn(dataBroker, adjacency, vpnName)) {
2317                                     addNewAdjToVpnInterface(existingVpnInterfaceId, primaryRd, adjacency,
2318                                             vpnInterfaceOptional.get().getDpnId(), writeConfigTxn, writeOperTxn);
2319                                     ListenableFuture<Void> operFuture = writeOperTxn.submit();
2320                                     try {
2321                                         operFuture.get();
2322                                     } catch (ExecutionException | InterruptedException e) {
2323                                         LOG.error("Exception encountered while submitting operational"
2324                                                 + " future for vpnInterface {}", vpnInterface, e);
2325                                     }
2326                                     List<ListenableFuture<Void>> futures = new ArrayList<>();
2327                                     futures.add(writeConfigTxn.submit());
2328                                     return futures;
2329                                 }  else {
2330                                     return Collections.emptyList();
2331                                 }
2332                             });
2333                     });
2334         }));
2335     }
2336
2337     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2338         private final String interfaceName;
2339         private final boolean add;
2340         private final String txnDestination;
2341
2342         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2343             this.interfaceName = interfaceName;
2344             this.add = add;
2345             this.txnDestination = transactionDest;
2346         }
2347
2348         @Override
2349         public void onSuccess(Void voidObj) {
2350             if (add) {
2351                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2352                         interfaceName, txnDestination);
2353             } else {
2354                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2355             }
2356         }
2357
2358         @Override
2359         public void onFailure(Throwable throwable) {
2360             if (add) {
2361                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}"
2362                         + " with exception: {}", interfaceName, txnDestination, throwable);
2363             } else {
2364                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed with exception: {}", interfaceName,
2365                         throwable);
2366                 VpnUtil.unsetScheduledToRemoveForVpnInterface(dataBroker, interfaceName);
2367             }
2368         }
2369     }
2370 }