vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.collect.Iterators;
  13 import com.google.common.util.concurrent.FutureCallback;
  14 import com.google.common.util.concurrent.Futures;
  15 import com.google.common.util.concurrent.ListenableFuture;
  16 import com.google.common.util.concurrent.MoreExecutors;
  17 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  18 import java.math.BigInteger;
  19 import java.util.ArrayList;
  20 import java.util.Collections;
  21 import java.util.Iterator;
  22 import java.util.List;
  23 import java.util.Map;
  24 import java.util.Objects;
  25 import java.util.concurrent.ConcurrentHashMap;
  26 import java.util.concurrent.ConcurrentLinkedQueue;
  27 import java.util.function.Consumer;
  28 import java.util.function.Predicate;
  29 import java.util.stream.Collectors;
  30 import javax.annotation.PostConstruct;
  31 import javax.annotation.PreDestroy;
  32 import javax.inject.Inject;
  33 import javax.inject.Singleton;
  34 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  35 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  36 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  37 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  38 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  39 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  40 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  41 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  42 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  43 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  44 import org.opendaylight.genius.mdsalutil.NWUtil;
  45 import org.opendaylight.genius.mdsalutil.NwConstants;
  46 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  47 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  48 import org.opendaylight.infrautils.caches.CacheProvider;
  49 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  50 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  51 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  52 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  53 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  54 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  55 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  56 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  57 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  58 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  59 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  60 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  61 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  62 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  63 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  64 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  65 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  66 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  67 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 107 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 108 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 109 import org.slf4j.Logger;
 110 import org.slf4j.LoggerFactory;
 111
 112 @Singleton
 113 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 114
 115     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 116     private static final short DJC_MAX_RETRIES = 3;
 117
 118     private final DataBroker dataBroker;
 119     private final ManagedNewTransactionRunner txRunner;
 120     private final IBgpManager bgpManager;
 121     private final IFibManager fibManager;
 122     private final IMdsalApiManager mdsalManager;
 123     private final IdManagerService idManager;
 124     private final OdlInterfaceRpcService ifaceMgrRpcService;
 125     private final VpnFootprintService vpnFootprintService;
 126     private final IInterfaceManager interfaceManager;
 127     private final IVpnManager vpnManager;
 128     private final ArpResponderHandler arpResponderHandler;
 129     private final JobCoordinator jobCoordinator;
 130     private final VpnUtil vpnUtil;
 131
 132     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 133
 134     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 135             new ConcurrentHashMap<>();
 136
 137     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 138
 139     @Inject
 140     public VpnInterfaceManager(final DataBroker dataBroker,
 141                                final IBgpManager bgpManager,
 142                                final IdManagerService idManager,
 143                                final IMdsalApiManager mdsalManager,
 144                                final IFibManager fibManager,
 145                                final OdlInterfaceRpcService ifaceMgrRpcService,
 146                                final VpnFootprintService vpnFootprintService,
 147                                final IInterfaceManager interfaceManager,
 148                                final IVpnManager vpnManager,
 149                                final ArpResponderHandler arpResponderHandler,
 150                                final JobCoordinator jobCoordinator,
 151                                final CacheProvider cacheProvider,
 152                                final VpnUtil vpnUtil) {
 153         super(VpnInterface.class, VpnInterfaceManager.class);
 154
 155         this.dataBroker = dataBroker;
 156         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 157         this.bgpManager = bgpManager;
 158         this.idManager = idManager;
 159         this.mdsalManager = mdsalManager;
 160         this.fibManager = fibManager;
 161         this.ifaceMgrRpcService = ifaceMgrRpcService;
 162         this.vpnFootprintService = vpnFootprintService;
 163         this.interfaceManager = interfaceManager;
 164         this.vpnManager = vpnManager;
 165         this.arpResponderHandler = arpResponderHandler;
 166         this.jobCoordinator = jobCoordinator;
 167         this.vpnUtil = vpnUtil;
 168
 169         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 170                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 171                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 172     }
 173
 174     public Runnable isNotifyTaskQueued(String intfName) {
 175         return vpnIntfMap.remove(intfName);
 176     }
 177
 178     @PostConstruct
 179     public void start() {
 180         LOG.info("{} start", getClass().getSimpleName());
 181         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 182     }
 183
 184     @Override
 185     @PreDestroy
 186     public void close() {
 187         super.close();
 188         vpnInstanceOpDataEntryCache.close();
 189     }
 190
 191     @Override
 192     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 193         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 194     }
 195
 196     @Override
 197     protected VpnInterfaceManager getDataTreeChangeListener() {
 198         return VpnInterfaceManager.this;
 199     }
 200
 201     @Override
 202     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 203         LOG.info("add: intfName {} onto vpnName {}",
 204                  vpnInterface.getName(),
 205                  VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 206         addVpnInterface(identifier, vpnInterface, null, null);
 207     }
 208
 209     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 210                           final VpnInterface vpnInterface, String vpnName) {
 211         synchronized (vpnName.intern()) {
 212             if (isVpnInstanceReady(vpnName)) {
 213                 return true;
 214             }
 215             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 216             return false;
 217         }
 218     }
 219
 220     // TODO Clean up the exception handling
 221     @SuppressWarnings("checkstyle:IllegalCatch")
 222     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 223                              final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs) {
 224         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
 225             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 226             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 227         }
 228     }
 229
 230     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 231                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 232         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 233         final String interfaceName = key.getName();
 234
 235         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 236             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 237                   vpnName, vpnInterface.getName());
 238             return;
 239         }
 240         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 241              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 242         List<Adjacency> copyOldAdjs = null;
 243         if (oldAdjs != null) {
 244             copyOldAdjs = new ArrayList<>();
 245             copyOldAdjs.addAll(oldAdjs);
 246         }
 247         List<Adjacency> copyNewAdjs = null;
 248         if (newAdjs != null) {
 249             copyNewAdjs = new ArrayList<>();
 250             copyNewAdjs.addAll(newAdjs);
 251         }
 252         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 253     }
 254
 255     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 256                     final VpnInterface vpnInterface, final List<Adjacency> oldAdjs,
 257                     final List<Adjacency> newAdjs,
 258                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 259         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 260         final String interfaceName = key.getName();
 261         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 262         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 263             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 264             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 265             if (interfaceState != null) {
 266                 try {
 267                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 268                     final int ifIndex = interfaceState.getIfIndex();
 269                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 270                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 271                         // (the inventory tx goes in last)
 272                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 273                         ListenableFuture<Void> confFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 274                             confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 275                                 operTx -> futures.add(
 276                                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(invTx -> {
 277                                             LOG.info(
 278                                                     "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 279                                                     + " on dpn {}",
 280                                                     vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 281                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 282                                                     confTx, operTx, invTx, interfaceState, vpnName);
 283                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 284                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 285                                                         + " Update for swapping VPN {} case.", interfaceName, vpnName);
 286                                                 if (newAdjs != null) {
 287                                                     for (Adjacency adj : newAdjs) {
 288                                                         if (oldAdjs.contains(adj)) {
 289                                                             oldAdjs.remove(adj);
 290                                                         } else {
 291                                                             if (!isBgpVpnInternetVpn
 292                                                                     || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 293                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 294                                                                         primaryRd, adj, dpnId, operTx, confTx);
 295                                                             }
 296                                                         }
 297                                                     }
 298                                                 }
 299                                                 for (Adjacency adj : oldAdjs) {
 300                                                     if (!isBgpVpnInternetVpn
 301                                                             || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 302                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 303                                                                 operTx, confTx);
 304                                                     }
 305                                                 }
 306                                             }
 307                                         })))));
 308                         futures.add(confFuture);
 309                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 310                                 MoreExecutors.directExecutor());
 311                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 312                                 + " processed successfully", interfaceName, vpnName, dpnId);
 313                         return futures;
 314                     });
 315                 } catch (NumberFormatException | IllegalStateException e) {
 316                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 317                                     + "interface {}. Interface addition on vpn {} failed", interfaceName,
 318                                     vpnName, e);
 319                     return;
 320                 }
 321             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 322                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 323                     () -> {
 324                         ListenableFuture<Void> future = txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
 325                             createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 326                                     confTx, vpnName);
 327                             LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 328                                     vpnName, vpnInterface.getDpnId());
 329                         });
 330                         ListenableFutures.addErrorLogging(future, LOG,
 331                                 "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 332                         return Collections.singletonList(future);
 333                     });
 334             } else {
 335                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 336                         + " is not available", interfaceName, vpnName);
 337             }
 338         } else {
 339             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 340                       + " as vpn is pending delete", interfaceName, vpnName,
 341                     vpnInterface.getDpnId());
 342         }
 343     }
 344
 345     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 346     // see comments below.
 347     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 348     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 349             final int lportTag, boolean isInterfaceUp,
 350             WriteTransaction writeConfigTxn,
 351             WriteTransaction writeOperTxn,
 352             WriteTransaction writeInvTxn,
 353             Interface interfaceState,
 354             final String vpnName) {
 355         final String interfaceName = vpnInterface.getName();
 356         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 357                 vpnName);
 358         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 359         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 360         if (!isInterfaceUp) {
 361             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 362                      interfaceName, dpId, vpnName);
 363             long vpnId = vpnUtil.getVpnId(vpnName);
 364             if (vpnId == VpnConstants.INVALID_ID) {
 365                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 366                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 367                         dpId);
 368                 return;
 369             }
 370
 371             boolean waitForVpnInterfaceOpRemoval = false;
 372             if (opVpnInterface != null) {
 373                 String opVpnName = opVpnInterface.getVpnInstanceName();
 374                 String primaryInterfaceIp = null;
 375                 if (opVpnName.equals(vpnName)) {
 376                     // Please check if the primary VRF Entry does not exist for VPNInterface
 377                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 378                     // back to back
 379                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 380                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 381                     if (adjs == null) {
 382                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 383                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 384                                 vpnName);
 385                         return;
 386                     }
 387                     for (Adjacency adj : adjs) {
 388                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 389                             primaryInterfaceIp = adj.getIpAddress();
 390                             break;
 391                         }
 392                     }
 393                     if (primaryInterfaceIp == null) {
 394                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 395                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 396                                 dpId, vpnName);
 397                         return;
 398                     }
 399                     // Get the rd of the vpn instance
 400                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 401                     if (vrf != null) {
 402                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 403                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 404                         return;
 405                     }
 406                     waitForVpnInterfaceOpRemoval = true;
 407                 } else {
 408                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 409                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 410                 }
 411             }
 412             if (!waitForVpnInterfaceOpRemoval) {
 413                 // Add the VPNInterface and quit
 414                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 415                         null/*ipAddressSourceValuePair*/,
 416                         true /* add */);
 417                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 418                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 419                 if (!isBgpVpnInternetVpn) {
 420                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 421                 }
 422                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 423                         dpId, vpnName);
 424                 if (interfaceManager.isExternalInterface(interfaceName)) {
 425                     processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
 426                         NwConstants.ADD_FLOW);
 427                 }
 428                 return;
 429             }
 430
 431             // FIB didn't get a chance yet to clean up this VPNInterface
 432             // Let us give it a chance here !
 433             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 434                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 435             try {
 436                 Runnable notifyTask = new VpnNotifyTask();
 437                 synchronized (notifyTask) {
 438                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 439                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 440                     // published here while holding the lock on it so this path will hit the wait before notify can be
 441                     // invoked.
 442                     vpnIntfMap.put(interfaceName, notifyTask);
 443                     try {
 444                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 445                     } catch (InterruptedException e) {
 446                         // Ignored
 447                     }
 448                 }
 449             } finally {
 450                 vpnIntfMap.remove(interfaceName);
 451             }
 452
 453             if (opVpnInterface != null) {
 454                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 455                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 456                         dpId, vpnName);
 457                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 458                 return;
 459             }
 460             // VPNInterface got removed, proceed with Add
 461             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 462                     interfaceName, dpId, vpnName);
 463             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 464                     null/*ipAddressSourceValuePair*/,
 465                     true /* add */);
 466             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 467                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 468             if (!isBgpVpnInternetVpn) {
 469                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 470             }
 471             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 472                     + " FIB to clean up", interfaceName, dpId, vpnName);
 473             if (interfaceManager.isExternalInterface(interfaceName)) {
 474                 processExternalVpnInterface(interfaceName, vpnName, dpId,
 475                                lportTag, NwConstants.ADD_FLOW);
 476             }
 477         } else {
 478             try {
 479                 // Interface is retained in the DPN, but its Link Up.
 480                 // Advertise prefixes again for this interface to BGP
 481                 InstanceIdentifier<VpnInterface> identifier =
 482                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 483                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 484                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 485                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 486                 // Perform similar operation as interface add event for extraroutes.
 487                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 488                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 489                         LogicalDatastoreType.CONFIGURATION, path);
 490                 if (!optAdjacencies.isPresent()) {
 491                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 492                     return;
 493                 }
 494                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
 495                 for (Adjacency adjacency : adjacencies) {
 496                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 497                         continue;
 498                     }
 499                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 500                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 501                         continue;
 502                     }
 503                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 504                             dpId, writeOperTxn, writeConfigTxn);
 505                 }
 506             } catch (ReadFailedException e) {
 507                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 508                         interfaceName, vpnName, primaryRd, dpId);
 509             }
 510         }
 511     }
 512
 513     private void processExternalVpnInterface(String interfaceName, String vpnName, BigInteger dpId,
 514         int lportTag, int addOrRemove) {
 515         Uuid extNetworkId;
 516         try {
 517             // vpn instance of ext-net interface is the network-id
 518             extNetworkId = new Uuid(vpnName);
 519         } catch (IllegalArgumentException e) {
 520             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 521                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 522             return;
 523         }
 524
 525         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 526         if (routerIds == null || routerIds.isEmpty()) {
 527             LOG.info("processExternalVpnInterface: No router is associated with {}."
 528                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 529                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 530             return;
 531         }
 532
 533         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 534                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 535         for (Uuid routerId : routerIds) {
 536             String routerName = routerId.getValue();
 537             BigInteger primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 538             if (Objects.equals(primarySwitch, dpId)) {
 539                 Routers router = vpnUtil.getExternalRouter(routerName);
 540                 if (router != null) {
 541                     if (addOrRemove == NwConstants.ADD_FLOW) {
 542                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 543                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 544                                 dpId, interfaceName, lportTag);
 545                     } else {
 546                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 547                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 548                                 dpId, interfaceName, lportTag);
 549                     }
 550                 } else {
 551                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 552                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 553                             interfaceName, dpId, vpnName);
 554                 }
 555             }
 556         }
 557     }
 558
 559     // TODO Clean up the exception handling
 560     @SuppressWarnings("checkstyle:IllegalCatch")
 561     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 562                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 563                                                   String vpnName, String interfaceName) {
 564         if (rd == null) {
 565             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 566                   interfaceName, dpnId, vpnName);
 567             return;
 568         } else {
 569             if (rd.equals(vpnName)) {
 570                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 571                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 572                 return;
 573             }
 574         }
 575         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 576                 interfaceName, dpnId, vpnName, rd);
 577
 578         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 579         if (nextHopIp == null) {
 580             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 581                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 582                     rd, vpnName);
 583             return;
 584         }
 585
 586         try {
 587             //Read NextHops
 588             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 589             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 590                     LogicalDatastoreType.OPERATIONAL, path);
 591             if (adjacencies.isPresent()) {
 592                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 593                 if (!nextHops.isEmpty()) {
 594                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 595                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 596                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 597                     long l3vni = vpnInstanceOpData.getL3vni();
 598                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 599                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 600                     for (Adjacency nextHop : nextHops) {
 601                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 602                             continue;
 603                         }
 604                         String gatewayMac = null;
 605                         long label = 0;
 606                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 607                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 608                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 609                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 610                                     .get();
 611                         } else {
 612                             label = nextHop.getLabel();
 613                         }
 614                         try {
 615                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 616                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 617                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 618                                     encapType, (int)label, l3vni, 0 /*l2vni*/,
 619                                     gatewayMac);
 620                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 621                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 622                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 623                         } catch (Exception e) {
 624                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 625                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 626                                     interfaceName, dpnId, e);
 627                         }
 628                     }
 629                 }
 630             }
 631         } catch (ReadFailedException e) {
 632             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 633                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 634         }
 635     }
 636
 637     // TODO Clean up the exception handling
 638     @SuppressWarnings("checkstyle:IllegalCatch")
 639     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 640                         String vpnName, String interfaceName, WriteTransaction writeConfigTxn,
 641                         WriteTransaction writeOperTx) {
 642         //Read NextHops
 643         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 644         String rd = vpnUtil.getVpnRd(interfaceName);
 645         if (rd == null) {
 646             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 647                 interfaceName, vpnName);
 648             return;
 649         } else {
 650             if (rd.equals(vpnName)) {
 651                 LOG.info(
 652                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 653                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 654                 return;
 655             }
 656         }
 657         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 658                vpnName, rd);
 659         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 660         try {
 661             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 662                     path);
 663         } catch (ReadFailedException e) {
 664             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 665                     interfaceName, vpnName);
 666         }
 667         if (adjacencies.isPresent()) {
 668             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 669
 670             if (!nextHops.isEmpty()) {
 671                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 672                         nextHops, interfaceName, vpnName, rd);
 673                 for (Adjacency nextHop : nextHops) {
 674                     try {
 675                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 676                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 677                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 678                                     interfaceName, vpnName);
 679                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 680                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 681                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 682                                     interfaceName, vpnName);
 683                         } else {
 684                             // Perform similar operation as interface delete event for extraroutes.
 685                             String allocatedRd = nextHop.getVrfId();
 686                             for (String nh : nextHop.getNextHopIpList()) {
 687                                 deleteExtraRouteFromCurrentAndImportingVpns(
 688                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
 689                                     writeOperTx);
 690                             }
 691                         }
 692                     } catch (Exception e) {
 693                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 694                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 695                     }
 696                 }
 697             }
 698         }
 699     }
 700
 701     @SuppressWarnings("checkstyle:IllegalCatch")
 702     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 703                                                   String primaryRd, String interfaceName, final long vpnId,
 704                                                   WriteTransaction writeConfigTxn,
 705                                                   WriteTransaction writeOperTxn,
 706                                                   final WriteTransaction writeInvTxn,
 707                                                   Interface interfaceState) {
 708         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 709         // Read NextHops
 710         Optional<VpnInterface> vpnInteface = Optional.absent();
 711         try {
 712             vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 713             LogicalDatastoreType.CONFIGURATION, identifier);
 714         } catch (ReadFailedException e) {
 715             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 716                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 717         }
 718         Uuid intfnetworkUuid = null;
 719         NetworkType networkType = null;
 720         Long segmentationId = Long.valueOf(-1);
 721         Adjacencies adjacencies = null;
 722         if (vpnInteface.isPresent()) {
 723             intfnetworkUuid = vpnInteface.get().getNetworkId();
 724             networkType = vpnInteface.get().getNetworkType();
 725             segmentationId = vpnInteface.get().getSegmentationId();
 726             adjacencies = vpnInteface.get().augmentation(Adjacencies.class);
 727             if (adjacencies == null) {
 728                 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 729                         null/*gwMac*/, writeOperTxn);
 730                 return;
 731             }
 732         }
 733         // Get the rd of the vpn instance
 734         String nextHopIp = null;
 735         try {
 736             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 737         } catch (Exception e) {
 738             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 739                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 740         }
 741         List<String> nhList = new ArrayList<>();
 742         if (nextHopIp != null) {
 743             nhList.add(nextHopIp);
 744             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 745                     interfaceName, dpnId, vpnName, nhList);
 746         }
 747         Optional<String> gwMac = Optional.absent();
 748         String vpnInterfaceSubnetGwMacAddress = null;
 749         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 750         Long l3vni = vpnInstanceOpData.getL3vni();
 751         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 752         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 753         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 754         List<Adjacency> nextHops = (adjacencies != null) ? adjacencies.getAdjacency() : Collections.emptyList();
 755         List<Adjacency> value = new ArrayList<>();
 756         for (Adjacency nextHop : nextHops) {
 757             String rd = primaryRd;
 758             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 759             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 760                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 761                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 762                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 763                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 764                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 765                 continue;
 766             }
 767             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 768                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 769                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 770                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 771                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 772                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 773
 774                 Prefixes prefixes = (intfnetworkUuid != null)
 775                     ? VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, intfnetworkUuid ,networkType,
 776                             segmentationId, prefixCue) :
 777                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, prefixCue);
 778                 writeOperTxn.merge(
 779                     LogicalDatastoreType.OPERATIONAL,
 780                     VpnUtil.getPrefixToInterfaceIdentifier(
 781                         vpnUtil.getVpnId(vpnName), prefix), prefixes, true);
 782                 final Uuid subnetId = nextHop.getSubnetId();
 783
 784                 String gatewayIp = nextHop.getSubnetGatewayIp();
 785                 if (gatewayIp == null) {
 786                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 787                     if (gatewayIpOptional.isPresent()) {
 788                         gatewayIp = gatewayIpOptional.get();
 789                     }
 790                 }
 791
 792                 if (gatewayIp != null) {
 793                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 794                     if (gwMac.isPresent()) {
 795                         // A valid mac-address is available for this subnet-gateway-ip
 796                         // Use this for programming ARP_RESPONDER table here.  And save this
 797                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 798                         // to populate L3_GW_MAC_TABLE there.
 799                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 800                                 gatewayIp, gwMac.get());
 801                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 802                     } else {
 803                         // A valid mac-address is not available for this subnet-gateway-ip
 804                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 805                         // Save this connected-mac-address as gateway-mac-address for the
 806                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 807                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 808                         if (gwMac.isPresent()) {
 809                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 810                                     NwConstants.ADD_FLOW, gwMac.get());
 811                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 812                                     gatewayIp, gwMac.get());
 813                         } else {
 814                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 815                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 816                                 + "gwIp {}",
 817                                 subnetId, interfaceName, vpnName, gatewayIp);
 818                         }
 819                     }
 820                 } else {
 821                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 822                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 823                         subnetId, interfaceName, vpnName);
 824                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 825                 }
 826                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 827                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 828             } else {
 829                 //Extra route adjacency
 830                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 831                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 832                 synchronized (vpnPrefixKey.intern()) {
 833                     java.util.Optional<String> rdToAllocate = vpnUtil
 834                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 835                                     nextHop.getNextHopIpList().get(0), dpnId);
 836                     if (rdToAllocate.isPresent()) {
 837                         rd = rdToAllocate.get();
 838                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 839                             rd, prefix);
 840                     } else {
 841                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 842                         continue;
 843                     }
 844                 }
 845                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 846                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 847                         interfaceName, dpnId);
 848             }
 849             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 850             // can be different from the gateway-mac-address within the VRFEntry as the
 851             // gateway-mac-address is a superset.
 852             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 853                     : RouteOrigin.STATIC;
 854             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 855                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 856                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 857             Adjacency operationalAdjacency = null;
 858             try {
 859                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 860             } catch (NullPointerException e) {
 861                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 862                     input, e.getMessage());
 863                 return;
 864             }
 865             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 866                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 867                         vpnName, l3vni, origin,
 868                         interfaceName, operationalAdjacency, encapType, writeConfigTxn);
 869             }
 870             value.add(operationalAdjacency);
 871         }
 872
 873         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 874         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 875                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 876
 877         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 878                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 879                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 880
 881         for (Adjacency nextHop : aug.getAdjacency()) {
 882             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 883             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 884                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 885                         : RouteOrigin.STATIC;
 886                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 887                 registeredPopulator.populateFib(input, writeConfigTxn);
 888             }
 889         }
 890     }
 891
 892     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 893                                               long lportTag, String gwMac, WriteTransaction writeOperTxn) {
 894         VpnInterfaceOpDataEntry opInterface =
 895               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac);
 896         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 897             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 898         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 899                 WriteTransaction.CREATE_MISSING_PARENTS);
 900         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 901                 interfaceName, dpnId, vpnName);
 902     }
 903
 904     // TODO Clean up the exception handling
 905     @SuppressWarnings("checkstyle:IllegalCatch")
 906     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 907                                            StateTunnelList stateTunnelList,
 908                                            WriteTransaction writeConfigTxn,
 909                                            WriteTransaction writeOperTxn) {
 910
 911         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
 912         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 913         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 914         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 915         if (adjList.isEmpty()) {
 916             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 917                     vpnInterface, srcDpnId);
 918             return;
 919         }
 920         String prefix = null;
 921         long label = 0;
 922         List<Adjacency> value = new ArrayList<>();
 923         boolean isNextHopAddReqd = false;
 924         String vpnName = vpnInterface.getVpnInstanceName();
 925         long vpnId = vpnUtil.getVpnId(vpnName);
 926         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 927         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 928                 vpnInterface.getName(), vpnInterface.getDpnId(),
 929                 vpnInterface.getVpnInstanceName(), adjList);
 930         for (Adjacency adj : adjList) {
 931             String rd = adj.getVrfId();
 932             rd = rd != null ? rd : vpnName;
 933             prefix = adj.getIpAddress();
 934             label = adj.getLabel();
 935             List<String> nhList = Collections.singletonList(srcTepIp);
 936             List<String> nextHopList = adj.getNextHopIpList();
 937             // If TEP is added , update the nexthop of primary adjacency.
 938             // Secondary adj nexthop is already pointing to primary adj IP address.
 939             if (nextHopList != null && !nextHopList.isEmpty()) {
 940                 /* everything right already */
 941             } else {
 942                 isNextHopAddReqd = true;
 943             }
 944
 945             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 946                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 947             } else {
 948                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 949                 if (!vrfEntryOptional.isPresent()) {
 950                     continue;
 951                 }
 952                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 953                 if (!nhList.contains(srcTepIp)) {
 954                     nhList.add(srcTepIp);
 955                     isNextHopAddReqd = true;
 956                 }
 957                 value.add(adj);
 958             }
 959
 960             if (isNextHopAddReqd) {
 961                 updateLabelMapper(label, nhList);
 962                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 963                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 964                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 965                 // Update the VRF entry with nextHop
 966                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 967                         label, true, writeConfigTxn);
 968
 969                 //Get the list of VPN's importing this route(prefix) .
 970                 // Then update the VRF entry with nhList
 971                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
 972                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 973                     String vpnRd = vpn.getVrfId();
 974                     if (vpnRd != null) {
 975                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
 976                                 srcTepIp, label, true, writeConfigTxn);
 977                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 978                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
 979                             vpnInterface.getName(), srcDpnId, vpnName,
 980                             vpn.getVpnInstanceName(), vpnRd);
 981                     }
 982                 }
 983                 // Advertise the prefix to BGP only for external vpn
 984                 // since there is a nexthop change.
 985                 try {
 986                     if (!rd.equalsIgnoreCase(vpnName)) {
 987                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
 988                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
 989                                 null /*gatewayMacAddress*/);
 990                     }
 991                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
 992                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
 993                             srcDpnId, vpnName);
 994                 } catch (Exception ex) {
 995                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
 996                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
 997                             vpnInterface.getName(), srcDpnId, vpnName, ex);
 998                 }
 999             }
1000         }
1001         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1002         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1003                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1004                 .addAugmentation(AdjacenciesOp.class, aug).build();
1005         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1006                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1007         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
1008                 WriteTransaction.CREATE_MISSING_PARENTS);
1009         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
1010                 vpnInterface.getName(), srcDpnId, vpnName);
1011
1012     }
1013
1014     // TODO Clean up the exception handling
1015     @SuppressWarnings("checkstyle:IllegalCatch")
1016     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1017                                               StateTunnelList stateTunnelList,
1018                                               WriteTransaction writeConfigTxn,
1019                                               WriteTransaction writeOperTxn) {
1020
1021         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1022         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1023         String prefix = null;
1024         long label = 0;
1025         boolean isNextHopRemoveReqd = false;
1026         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
1027         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
1028         String vpnName = vpnInterface.getVpnInstanceName();
1029         long vpnId = vpnUtil.getVpnId(vpnName);
1030         String primaryRd = vpnUtil.getVpnRd(vpnName);
1031         if (adjList != null) {
1032             List<Adjacency> value = new ArrayList<>();
1033             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1034                     vpnInterface.getName(), vpnInterface.getDpnId(),
1035                     vpnInterface.getVpnInstanceName(), adjList);
1036             for (Adjacency adj : adjList) {
1037                 List<String> nhList = new ArrayList<>();
1038                 String rd = adj.getVrfId();
1039                 rd = rd != null ? rd : vpnName;
1040                 prefix = adj.getIpAddress();
1041                 List<String> nextHopList = adj.getNextHopIpList();
1042                 label = adj.getLabel();
1043                 if (nextHopList != null && !nextHopList.isEmpty()) {
1044                     isNextHopRemoveReqd = true;
1045                 }
1046                 // If TEP is deleted , remove the nexthop from primary adjacency.
1047                 // Secondary adj nexthop will continue to point to primary adj IP address.
1048                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1049                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1050                 } else {
1051                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1052                     if (!vrfEntryOptional.isPresent()) {
1053                         continue;
1054                     }
1055                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1056                     if (nhList.contains(srcTepIp)) {
1057                         nhList.remove(srcTepIp);
1058                         isNextHopRemoveReqd = true;
1059                     }
1060                     value.add(adj);
1061                 }
1062
1063                 if (isNextHopRemoveReqd) {
1064                     updateLabelMapper(label, nhList);
1065                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1066                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1067                             prefix, nhList, vpnName,
1068                             vpnId, rd, vpnInterface.getName());
1069                     // Update the VRF entry with removed nextHop
1070                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1071                             label, false, writeConfigTxn);
1072
1073                     //Get the list of VPN's importing this route(prefix) .
1074                     // Then update the VRF entry with nhList
1075                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1076                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1077                         String vpnRd = vpn.getVrfId();
1078                         if (vpnRd != null) {
1079                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1080                                     srcTepIp, label, false, writeConfigTxn);
1081                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1082                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1083                                     nhList, label, vpnInterface.getName(), srcDpnId,
1084                                     vpnName,
1085                                     vpn.getVpnInstanceName(), vpnRd);
1086                         }
1087                     }
1088
1089                     // Withdraw prefix from BGP only for external vpn.
1090                     try {
1091                         if (!rd.equalsIgnoreCase(vpnName)) {
1092                             bgpManager.withdrawPrefix(rd, prefix);
1093                         }
1094                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1095                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1096                                 vpnInterface.getName(), srcDpnId,
1097                                 vpnName);
1098                     } catch (Exception ex) {
1099                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1100                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1101                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1102                     }
1103                 }
1104             }
1105             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1106             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1107                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1108                     .addAugmentation(AdjacenciesOp.class, aug).build();
1109             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1110                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1111             writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
1112                     WriteTransaction.CREATE_MISSING_PARENTS);
1113             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1114                          vpnInterface.getName(), srcDpnId, vpnName);
1115         }
1116     }
1117
1118     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1119         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1120
1121         String vpnRd = vpnUtil.getVpnRd(vpnName);
1122         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1123         if (vpnInstanceOpDataEntry == null) {
1124             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1125                     + " to check for vpns exporting the routes", vpnName);
1126             return vpnsToExportRoute;
1127         }
1128
1129         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1130             if (input.getVpnInstanceName() == null) {
1131                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1132                         input.getVrfId());
1133                 return false;
1134             }
1135             return !input.getVpnInstanceName().equals(vpnName);
1136         };
1137
1138         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1139             Iterable<String> commonRTs =
1140                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1141                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1142             return Iterators.size(commonRTs.iterator()) > 0;
1143         };
1144
1145         vpnsToExportRoute =
1146                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1147                         Collectors.toList());
1148         return vpnsToExportRoute;
1149     }
1150
1151     // TODO Clean up the exception handling
1152     @SuppressWarnings("checkstyle:IllegalCatch")
1153     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1154         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1155         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1156             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1157             if (vrfEntries != null) {
1158                 ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1159                     for (VrfEntry vrfEntry : vrfEntries) {
1160                         try {
1161                             if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1162                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1163                                 LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1164                                                 + " is not a controller managed non intervpn link route. Ignoring.",
1165                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1166                                 continue;
1167                             }
1168                             String prefix = vrfEntry.getDestPrefix();
1169                             String gwMac = vrfEntry.getGatewayMacAddress();
1170                             vrfEntry.getRoutePaths().forEach(routePath -> {
1171                                 String nh = routePath.getNexthopAddress();
1172                                 int label = routePath.getLabel().intValue();
1173                                 if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1174                                         vrfEntry.getOrigin()))) {
1175                                     LOG.info("handleVpnsExportingRoutesImporting: Importing fib entry rd {} prefix {}"
1176                                                     + " nexthop {} label {} to vpn {} vpnRd {}",
1177                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1178                                     fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1179                                             Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1180                                             0 /*l3vni*/, gwMac,  vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1181                                             confTx);
1182                                 } else {
1183                                     LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry rd {} "
1184                                                     + "prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1185                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1186                                     SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1187                                     importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1188                                             confTx);
1189                                 }
1190                             });
1191                         } catch (RuntimeException e) {
1192                             LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1193                                             + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1194                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1195                         }
1196                     }
1197                 }), LOG, "Error handing VPN exporting routes");
1198             } else {
1199                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1200                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1201             }
1202         }
1203     }
1204
1205     @Override
1206     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1207         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1208         final String interfaceName = key.getName();
1209         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
1210             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1211             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1212         }
1213     }
1214
1215     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1216                                 final VpnInterface vpnInterface, final String vpnName,
1217                                 final String interfaceName) {
1218         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1219             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1220                 ListenableFuture<Void> future = txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1221                     deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1222                     LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1223                 });
1224                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1225                         vpnInterface.getName(), vpnName);
1226                 return Collections.singletonList(future);
1227             }, DJC_MAX_RETRIES);
1228         } else {
1229             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName);
1230         }
1231     }
1232
1233     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1234     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1235                                 final VpnInterface vpnInterface, final String vpnName,
1236                                 final String interfaceName) {
1237         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1238                 vpnName, vpnInterface.getDpnId());
1239         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1240         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1241             () -> {
1242                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1243                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1244                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1245                         writeOperTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeInvTxn -> {
1246                             LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1247                                     interfaceName, vpnName);
1248                             BigInteger dpId = BigInteger.ZERO;
1249                             int ifIndex = 0;
1250                             String gwMacAddress = null;
1251                             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1252                                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1253                             Optional<VpnInterfaceOpDataEntry> optVpnInterface = Optional.absent();
1254                             try {
1255                                 optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1256                                         LogicalDatastoreType.OPERATIONAL, interfaceId);
1257                             } catch (ReadFailedException e) {
1258                                 LOG.error("remove: Failed to read data store for interface {} vpn {}", interfaceName,
1259                                         vpnName);
1260                                 return;
1261                             }
1262                             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1263                                     interfaceName);
1264                             if (interfaceState != null) {
1265                                 try {
1266                                     dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1267                                 } catch (NumberFormatException | IllegalStateException e) {
1268                                     LOG.error("remove: Unable to retrieve dpnId from interface operational"
1269                                             + " data store for interface {} on dpn {} for vpn {} Fetching"
1270                                                     + " from vpn interface op data store. ", interfaceName,
1271                                             vpnInterface.getDpnId(), vpnName, e);
1272                                     dpId = BigInteger.ZERO;
1273                                 }
1274                                 ifIndex = interfaceState.getIfIndex();
1275                                 gwMacAddress = interfaceState.getPhysAddress().getValue();
1276                             } else {
1277                                 LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1278                                         + " from vpn interface op.", interfaceName);
1279                                 if (optVpnInterface.isPresent()) {
1280                                     VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1281                                     dpId = vpnOpInterface.getDpnId();
1282                                     ifIndex = vpnOpInterface.getLportTag().intValue();
1283                                     gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1284                                 } else {
1285                                     LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1286                                                     + " as interfaceState and vpn interface op is not"
1287                                             + " available", interfaceName, vpnName);
1288                                     return;
1289                                 }
1290                             }
1291                             processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1292                                     optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1293                                     writeConfigTxn, writeOperTxn, writeInvTxn);
1294                             LOG.info(
1295                                     "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1296                                             + "successfully",
1297                                     interfaceName, vpnInterface.getDpnId(), vpnName);
1298                         })))));
1299                 futures.add(configFuture);
1300                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1301                 return futures;
1302             }, DJC_MAX_RETRIES);
1303     }
1304
1305     protected void processVpnInterfaceDown(BigInteger dpId,
1306                                            String interfaceName,
1307                                            int lportTag,
1308                                            String gwMac,
1309                                            VpnInterfaceOpDataEntry vpnOpInterface,
1310                                            boolean isInterfaceStateDown,
1311                                            WriteTransaction writeConfigTxn,
1312                                            WriteTransaction writeOperTxn,
1313                                            WriteTransaction writeInvTxn) {
1314         if (vpnOpInterface == null) {
1315             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1316                     + " as it is not available in operational data store", interfaceName, dpId);
1317             return;
1318         }
1319         final String vpnName = vpnOpInterface.getVpnInstanceName();
1320         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1321                                                     interfaceName, vpnName);
1322         if (!isInterfaceStateDown) {
1323             final long vpnId = vpnUtil.getVpnId(vpnName);
1324             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName,  null);
1325             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1326             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1327                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1328             if (interfaceManager.isExternalInterface(interfaceName)) {
1329                 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1330                     NwConstants.DEL_FLOW);
1331             }
1332             if (!isBgpVpnInternetVpn) {
1333                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1334             }
1335             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1336                     + " successful", interfaceName, dpId, vpnName);
1337         } else {
1338             // Interface is retained in the DPN, but its Link Down.
1339             // Only withdraw the prefixes for this interface from BGP
1340             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1341         }
1342     }
1343
1344     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1345                                           final String vpnName, final long vpnId, String gwMac,
1346                                           WriteTransaction writeConfigTxn, final WriteTransaction writeOperTxn,
1347                                           final WriteTransaction writeInvTxn) {
1348         //Read NextHops
1349         try {
1350             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1351                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1352             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1353             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1354                     LogicalDatastoreType.OPERATIONAL, path);
1355             String primaryRd = vpnUtil.getVpnRd(vpnName);
1356             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1357                     interfaceName, dpnId, vpnName, primaryRd);
1358             if (adjacencies.isPresent() && !adjacencies.get().getAdjacency().isEmpty()) {
1359                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1360                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1361                         interfaceName, dpnId, vpnName, nextHops);
1362                 for (Adjacency nextHop : nextHops) {
1363                     if (nextHop.isPhysNetworkFunc()) {
1364                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1365                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1366                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1367                                 null/*writeCfgTxn*/);
1368                     } else {
1369                         String rd = nextHop.getVrfId();
1370                         List<String> nhList;
1371                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1372                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1373                         } else {
1374                             // This is a primary adjacency
1375                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1376                                     : Collections.emptyList();
1377                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1378                                     interfaceName, writeInvTxn);
1379                         }
1380                         if (!nhList.isEmpty()) {
1381                             if (rd.equals(vpnName)) {
1382                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1383                                 //remove from FIB directly
1384                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1385                                         interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1386                             } else {
1387                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1388                                         interfaceName, writeConfigTxn, writeOperTxn);
1389                             }
1390                         } else {
1391                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1392                                             + " interface {}", nextHop.getIpAddress(), rd,
1393                                     nextHop.getAdjacencyType().toString(), interfaceName);
1394                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1395                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1396                         }
1397                     }
1398                     String ip = nextHop.getIpAddress().split("/")[0];
1399                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1400                     if (vpnVipToPort != null) {
1401                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1402                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1403                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1404                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1405                     }
1406                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1407                     if (vpnPortipToPort != null) {
1408                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1409                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1410                                  + "Interface {} ip {} on dpn {} for vpn {}",
1411                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1412                     }
1413                 }
1414             } else {
1415                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1416                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1417                         + " Removing it.", interfaceName, vpnName, dpnId);
1418                 writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, identifier);
1419             }
1420         } catch (ReadFailedException e) {
1421             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1422                     interfaceName, dpnId, vpnName);
1423         }
1424     }
1425
1426     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName, String interfaceName,
1427                                                             BigInteger dpnId, WriteTransaction writeConfigTxn,
1428                                                             WriteTransaction writeOperTx) {
1429         return (nh) -> {
1430             String primaryRd = vpnUtil.getVpnRd(vpnName);
1431             String prefix = nextHop.getIpAddress();
1432             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1433             LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1434                     nextHop, vpnName, interfaceName, dpnId);
1435             synchronized (vpnNamePrefixKey.intern()) {
1436                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1437                         prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1438                     //If extra-route is present behind at least one VM, then do not remove or update
1439                     //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1440                     //prefixtointerface DS
1441                     return;
1442                 }
1443                 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1444                         writeConfigTxn);
1445             }
1446             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1447                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1448                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1449         };
1450     }
1451
1452     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1453                                            BigInteger dpnId, String rd, String interfaceName,
1454                                            WriteTransaction writeConfigTxn, WriteTransaction writeOperTx) {
1455         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1456                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1457         nhList.forEach((nh) -> {
1458             //IRT: remove routes from other vpns importing it
1459             vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1460                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1461             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1462                 String vpnRd = vpn.getVrfId();
1463                 if (vpnRd != null) {
1464                     fibManager.removeOrUpdateFibEntry(vpnRd,
1465                             nextHop.getIpAddress(), nh, writeConfigTxn);
1466                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1467                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1468                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1469                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1470                 }
1471             }
1472         });
1473     }
1474
1475     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1476                                                  int lportTag, String gwMac, String interfaceName,
1477                                                  WriteTransaction writeInvTxn) {
1478         final Uuid subnetId = nextHop.getSubnetId();
1479         if (nextHop.getSubnetGatewayMacAddress() == null) {
1480             // A valid mac-address was not available for this subnet-gateway-ip
1481             // So a connected-mac-address was used for this subnet and we need
1482             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1483             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1484         }
1485         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1486                 subnetId);
1487     }
1488
1489     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1490                                                   String interfaceName) {
1491         // This is either an extra-route (or) a learned IP via subnet-route
1492         List<String> nhList = null;
1493         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1494         if (nextHopIp == null || nextHopIp.isEmpty()) {
1495             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1496                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1497                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1498                     vpnName);
1499             nhList = Collections.emptyList();
1500         } else {
1501             nhList = Collections.singletonList(nextHopIp);
1502         }
1503         return nhList;
1504     }
1505
1506     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1507         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1508         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1509         // else use connected interface
1510         if (gwPort != null && gwPort.isSubnetIp()) {
1511             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1512                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1513             return Optional.of(gwPort.getMacAddress());
1514         }
1515         return Optional.absent();
1516     }
1517
1518     @Override
1519     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1520         final VpnInterface update) {
1521         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}" ,update.getName(),
1522                 update.getDpnId(), original.getVpnInstanceNames(),
1523                 update.getVpnInstanceNames());
1524         final String vpnInterfaceName = update.getName();
1525         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1526         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1527         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1528             != null ? origAdjs.getAdjacency() : new ArrayList<>();
1529         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1530         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1531             != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1532
1533         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1534         //handles switching between <internal VPN - external VPN>
1535         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1536             if (handleVpnSwapForVpnInterface(identifier, original, update)) {
1537                 LOG.info("update: handled VPNInterface {} on dpn {} update"
1538                                 + "upon VPN swap from oldVpn(s) {} to newVpn(s) {}",
1539                         original.getName(), dpnId,
1540                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1541                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1542                 return Collections.emptyList();
1543             }
1544             for (VpnInstanceNames vpnInterfaceVpnInstance : update.getVpnInstanceNames()) {
1545                 String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1546                 List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1547                 List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1548                 String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1549                 if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1550                     // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1551                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1552                     futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1553                         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(operTx -> {
1554                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1555                                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1556                             LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven",
1557                                     update.getName(), newVpnName);
1558                             //handle both addition and removal of adjacencies
1559                             //currently, new adjacency may be an extra route
1560                             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1561                             if (!oldAdjs.equals(newAdjs)) {
1562                                 for (Adjacency adj : copyNewAdjs) {
1563                                     if (copyOldAdjs.contains(adj)) {
1564                                         copyOldAdjs.remove(adj);
1565                                     } else {
1566                                         // add new adjacency - right now only extra route will hit this path
1567                                         if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1568                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1569                                                     dpnId, operTx, confTx);
1570                                         }
1571                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to"
1572                                                 + " vpn interface {} on vpn {} dpnId {}",
1573                                                 adj.getIpAddress(), adj.getNextHopIpList(),
1574                                                 adj.getLabel(), adj.getSubnetId(), update.getName(),
1575                                                 newVpnName, dpnId);
1576                                     }
1577                                 }
1578                                 for (Adjacency adj : copyOldAdjs) {
1579                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1580                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1581                                                 && !adj.isPhysNetworkFunc()) {
1582                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1583                                                     operTx, confTx);
1584                                             Optional<VpnInterfaceOpDataEntry> optVpnInterface = operTx.read(
1585                                                     LogicalDatastoreType.OPERATIONAL, vpnInterfaceOpIdentifier).get();
1586                                             if (optVpnInterface.isPresent()) {
1587                                                 VpnInterfaceOpDataEntry vpnInterfaceOpDataEntry = optVpnInterface.get();
1588                                                 long vpnId = vpnUtil.getVpnId(newVpnName);
1589                                                 vpnUtil.removePrefixToInterfaceAdj(adj, vpnId,
1590                                                         vpnInterfaceOpDataEntry, operTx);
1591                                             } else {
1592                                                 LOG.info("update: Vpninterface {} not present in Operational",
1593                                                         vpnInterfaceName);
1594                                             }
1595                                             //remove FIB entry
1596                                             String vpnRd = vpnUtil.getVpnRd(newVpnName);
1597                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1598                                                     + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1599                                             //remove BGP entry
1600                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1601                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1602                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1603                                             }
1604                                         } else {
1605                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1606                                                     operTx, confTx);
1607                                         }
1608                                     }
1609                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1610                                                     + " vpn interface {} on vpn {}", adj.getIpAddress(), adj
1611                                                     .getNextHopIpList(),
1612                                             adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1613                                 }
1614                             }
1615                         }));
1616                     }));
1617                     for (ListenableFuture<Void> future : futures) {
1618                         ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1619                                 update.getName(), update.getVpnInstanceNames());
1620                     }
1621                     return futures;
1622                 } else {
1623                     LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1624                             + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1625                 }
1626             }
1627             return Collections.emptyList();
1628         });
1629     }
1630
1631     private boolean handleVpnSwapForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1632                                                  VpnInterface original, VpnInterface update) {
1633         boolean isSwap = Boolean.FALSE;
1634         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1635         final String interfaceName = key.getName();
1636         List<String> oldVpnList = original.getVpnInstanceNames().stream()
1637             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1638         List<String> oldVpnListCopy = new ArrayList<>();
1639         oldVpnListCopy.addAll(oldVpnList);
1640         List<String> newVpnList = update.getVpnInstanceNames().stream()
1641             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1642         oldVpnList.removeAll(newVpnList);
1643         newVpnList.removeAll(oldVpnListCopy);
1644         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1645             for (String oldVpnName: oldVpnList) {
1646                 isSwap = Boolean.TRUE;
1647                 LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} remove vpnName {}"
1648                         + " running config-driven swap removal", interfaceName, oldVpnName);
1649                 removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1650                 LOG.info("handleVpnSwapForVpnInterface: Processed Remove for update on VPNInterface {} upon VPN swap"
1651                         + "from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName, newVpnList);
1652             }
1653             //Wait for previous interface bindings to be removed
1654             try {
1655                 Thread.sleep(2000);
1656             } catch (InterruptedException e) {
1657                 //Ignore
1658             }
1659
1660             final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1661             final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1662                     ? origAdjs.getAdjacency() : new ArrayList<>();
1663             final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1664             final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1665                     ? updateAdjs.getAdjacency() : new ArrayList<>();
1666
1667             for (String newVpnName: newVpnList) {
1668                 String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1669                 isSwap = Boolean.TRUE;
1670                 if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1671                     LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} onto vpnName {}"
1672                             + "running config-driven swap addition", interfaceName, newVpnName);
1673                     addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1674                     LOG.info("handleVpnSwapForVpnInterface: Processed Add for update on VPNInterface {}"
1675                                     + "from oldVpn(s) {} to newVpn {} upon VPN swap",
1676                             interfaceName, oldVpnListCopy, newVpnName);
1677                 }
1678             }
1679         }
1680         return isSwap;
1681     }
1682
1683     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1684         try {
1685             Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1686             synchronized (label.toString().intern()) {
1687                 InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1688                         .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1689                 Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1690                         LogicalDatastoreType.OPERATIONAL, lriIid);
1691                 if (opResult.isPresent()) {
1692                     LabelRouteInfo labelRouteInfo =
1693                             new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1694                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1695                             labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1696                 }
1697             }
1698             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1699                     nextHopIpList);
1700         } catch (ReadFailedException e) {
1701             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1702                     nextHopIpList);
1703         } catch (TransactionCommitFailedException e) {
1704             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1705                     nextHopIpList);
1706         }
1707     }
1708
1709     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1710         SubnetRoute route, String parentVpnRd, WriteTransaction writeConfigTxn) {
1711
1712         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1713         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1714                 .addAugmentation(SubnetRoute.class, route).build();
1715         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1716         InstanceIdentifierBuilder<VrfTables> idBuilder =
1717             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1718         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1719         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1720         if (writeConfigTxn != null) {
1721             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1722         } else {
1723             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1724         }
1725         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1726                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1727     }
1728
1729     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1730                                            Adjacency adj, BigInteger dpnId, WriteTransaction writeOperTxn,
1731                                            WriteTransaction writeConfigTxn) {
1732         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1733         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1734         try {
1735             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1736                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1737             if (optVpnInterface.isPresent()) {
1738                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1739                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1740                 String vpnName = currVpnIntf.getVpnInstanceName();
1741                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1742                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1743                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1744                         LogicalDatastoreType.OPERATIONAL, adjPath);
1745                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1746                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1747                 long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1748                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1749                 List<Adjacency> adjacencies;
1750                 if (optAdjacencies.isPresent()) {
1751                     adjacencies = optAdjacencies.get().getAdjacency();
1752                 } else {
1753                     // This code will be hit in case of first PNF adjacency
1754                     adjacencies = new ArrayList<>();
1755                 }
1756                 long vpnId = vpnUtil.getVpnId(vpnName);
1757                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1758                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1759                 Adjacency operationalAdjacency = null;
1760                 //Handling dual stack neutron port primary adjacency
1761                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1762                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1763                             currVpnIntf.getName(), vpnName);
1764                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1765                             currVpnIntf.getName());
1766                     if (interfaceState != null) {
1767                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1768                                 currVpnIntf.getName(),
1769                                 vpnId, writeConfigTxn, writeOperTxn, null, interfaceState);
1770                     }
1771                 }
1772                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1773                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1774                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
1775                             : RouteOrigin.STATIC;
1776                     String nh = adj.getNextHopIpList().get(0);
1777                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1778                     synchronized (vpnPrefixKey.intern()) {
1779                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1780                                 vpnId, null, prefix, vpnName, nh, dpnId);
1781                         if (rdToAllocate.isPresent()) {
1782                             input.setRd(rdToAllocate.get());
1783                             operationalAdjacency = populator.createOperationalAdjacency(input);
1784                             int label = operationalAdjacency.getLabel().intValue();
1785                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1786                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
1787                                     currVpnIntf.getName(), operationalAdjacency, encapType, writeConfigTxn);
1788                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1789                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1790                                     vpnName, label, currVpnIntf.getName(), dpnId);
1791                         } else {
1792                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
1793                                     vpnName, prefix);
1794                             return;
1795                         }
1796                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1797                         // Keeping the MPLS check for now.
1798                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1799                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1800                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1801                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
1802                             vpnsToImportRoute.forEach(vpn -> {
1803                                 if (vpn.getVrfId() != null) {
1804                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(), vpnId, prefix,
1805                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
1806                                             .ifPresent(
1807                                                 rds -> vpnManager.addExtraRoute(
1808                                                         vpnUtil.getVpnName(vpn.getVpnId()),
1809                                                         adj.getIpAddress(), nh, rds,
1810                                                         currVpnIntf.getVpnInstanceName(), l3vni,
1811                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(),
1812                                                         opAdjacency, encapType, writeConfigTxn));
1813                                 }
1814                             });
1815                         }
1816                     }
1817                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1818                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1819                             currVpnIntf.getName(), vpnName);
1820
1821                     InstanceIdentifier<VpnInterface> vpnIfaceConfigidentifier = VpnUtil
1822                             .getVpnInterfaceIdentifier(currVpnIntf.getName());
1823                     Optional<VpnInterface> vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1824                             LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier);
1825                     Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1826                             Prefixes.PrefixCue.PhysNetFunc);
1827                     if (vpnIntefaceConfig.isPresent()) {
1828                         pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1829                                 vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(),
1830                                 vpnIntefaceConfig.get().getSegmentationId(), Prefixes.PrefixCue.PhysNetFunc);
1831                     }
1832
1833                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1834
1835                     writeOperTxn.merge(
1836                             LogicalDatastoreType.OPERATIONAL,
1837                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
1838                                     prefix), pnfPrefix, true);
1839
1840                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1841                             adj.getIpAddress(), Collections.emptyList(), null /* EncapType */, 0 /* label */,
1842                             0 /*l3vni*/, null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1843
1844                     input.setRd(adj.getVrfId());
1845                 }
1846                 if (operationalAdjacency == null) {
1847                     operationalAdjacency = populator.createOperationalAdjacency(input);
1848                 }
1849                 adjacencies.add(operationalAdjacency);
1850                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1851                 VpnInterfaceOpDataEntry newVpnIntf =
1852                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
1853                                 aug, dpnId, currVpnIntf.getLportTag(),
1854                                 currVpnIntf.getGatewayMacAddress());
1855
1856                 writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1857             }
1858         } catch (ReadFailedException e) {
1859             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
1860                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
1861         }
1862     }
1863
1864     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
1865         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
1866         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
1867     }
1868
1869     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
1870             BigInteger dpnId, WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) {
1871         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1872         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1873         try {
1874             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
1875                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1876             if (optVpnInterface.isPresent()) {
1877                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1878                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1879                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1880                         LogicalDatastoreType.OPERATIONAL, path);
1881                 if (optAdjacencies.isPresent()) {
1882                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1883
1884                     if (!adjacencies.isEmpty()) {
1885                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
1886                         Iterator<Adjacency> adjIt = adjacencies.iterator();
1887                         while (adjIt.hasNext()) {
1888                             Adjacency adjElem = adjIt.next();
1889                             if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1890                                 String rd = adjElem.getVrfId();
1891                                 if (adj.getNextHopIpList() != null) {
1892                                     for (String nh : adj.getNextHopIpList()) {
1893                                         deleteExtraRouteFromCurrentAndImportingVpns(
1894                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
1895                                                 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
1896                                     }
1897                                 } else if (adj.isPhysNetworkFunc()) {
1898                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
1899                                             adj.getIpAddress(), adj.getSubnetId());
1900                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
1901                                             writeConfigTxn);
1902                                 }
1903                                 break;
1904                             }
1905
1906                         }
1907                     }
1908                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
1909                             dpnId, adj.getVrfId());
1910                 } else {
1911                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
1912                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
1913                 }
1914             }
1915         } catch (ReadFailedException e) {
1916             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
1917                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
1918         }
1919     }
1920
1921     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
1922         String rd, String intfName, WriteTransaction writeConfigTxn, WriteTransaction writeOperTx) {
1923         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
1924         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1925         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1926             String vpnRd = vpn.getVrfId();
1927             if (vpnRd != null) {
1928                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
1929                         writeOperTx);
1930             }
1931         }
1932     }
1933
1934     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1935         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1936             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1937             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1938     }
1939
1940     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1941         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1942             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1943     }
1944
1945     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
1946                                                     WriteTransaction writeConfigTxn, String vpnName) {
1947         if (vpnInterface == null) {
1948             return;
1949         }
1950         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
1951         if (adjs == null) {
1952             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
1953                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
1954             return;
1955         }
1956         for (Adjacency adj : adjs) {
1957             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1958                 String primaryInterfaceIp = adj.getIpAddress();
1959                 String macAddress = adj.getMacAddress();
1960                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1961
1962                 long label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
1963                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
1964
1965                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
1966                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
1967                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
1968                         routerInt, label, writeConfigTxn);
1969                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
1970                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
1971                         macAddress);
1972                 return;
1973             }
1974         }
1975         LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
1976                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}", interfaceName,
1977                 primaryRd, vpnName);
1978     }
1979
1980     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
1981                                                     WriteTransaction writeConfigTxn, String vpnName) {
1982         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
1983         String rd = vpnUtil.getVpnRd(vpnName);
1984         if (adjs != null) {
1985             List<Adjacency> adjsList = adjs.getAdjacency();
1986             for (Adjacency adj : adjsList) {
1987                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1988                     String primaryInterfaceIp = adj.getIpAddress();
1989                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1990                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
1991                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
1992                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
1993                     return;
1994                 }
1995             }
1996         } else {
1997             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
1998                     vpnInterface.getName(), rd);
1999         }
2000     }
2001
2002     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2003         if (!canHandleNewVpnInterface(intefaceData.identifier, intefaceData.vpnInterface, vpnName)) {
2004             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
2005                   vpnName, intefaceData.vpnInterface.getName());
2006             return;
2007         }
2008         final VpnInterfaceKey key = intefaceData.identifier
2009                .firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
2010         final String interfaceName = key.getName();
2011         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2012                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2013         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2014                   intefaceData.identifier, vpnName);
2015         return;
2016     }
2017
2018     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2019                                               VpnInterface vpnInterface, String vpnName) {
2020         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2021                .get(vpnName);
2022         if (vpnInterfaces == null) {
2023             vpnInterfaces = new ConcurrentLinkedQueue<>();
2024         }
2025         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2026         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2027         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2028                 vpnInterface.getName(), vpnName);
2029     }
2030
2031     public boolean isVpnInstanceReady(String vpnInstanceName) {
2032         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2033         if (vpnRd == null) {
2034             return false;
2035         }
2036         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2037
2038         return vpnInstanceOpDataEntry != null;
2039     }
2040
2041     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2042         synchronized (vpnInstanceName.intern()) {
2043             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2044                     unprocessedVpnInterfaces.get(vpnInstanceName);
2045             if (vpnInterfaces != null) {
2046                 while (!vpnInterfaces.isEmpty()) {
2047                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2048                     if (hasVpnInstanceCreatedSuccessfully) {
2049                         processSavedInterface(savedInterface, vpnInstanceName);
2050                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2051                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2052                     } else {
2053                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2054                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2055                     }
2056                 }
2057             } else {
2058                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2059             }
2060         }
2061     }
2062
2063     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2064             VpnInterface vpnInterface) {
2065         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2066             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2067                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2068             if (vpnInterfaces != null) {
2069                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2070                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2071                             + "unprocessed list", vpnInterface.getName(),
2072                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2073                 }
2074             } else {
2075                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2076                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2077             }
2078         }
2079     }
2080
2081     public void vpnInstanceIsReady(String vpnInstanceName) {
2082         processSavedInterfaces(vpnInstanceName, true);
2083     }
2084
2085     public void vpnInstanceFailed(String vpnInstanceName) {
2086         processSavedInterfaces(vpnInstanceName, false);
2087     }
2088
2089     private static class UnprocessedVpnInterfaceData {
2090         InstanceIdentifier<VpnInterface> identifier;
2091         VpnInterface vpnInterface;
2092
2093         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2094             this.identifier = identifier;
2095             this.vpnInterface = vpnInterface;
2096         }
2097
2098         @Override
2099         public int hashCode() {
2100             final int prime = 31;
2101             int result = 1;
2102             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2103             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2104             return result;
2105         }
2106
2107         @Override
2108         public boolean equals(Object obj) {
2109             if (this == obj) {
2110                 return true;
2111             }
2112             if (obj == null) {
2113                 return false;
2114             }
2115             if (getClass() != obj.getClass()) {
2116                 return false;
2117             }
2118             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2119             if (identifier == null) {
2120                 if (other.identifier != null) {
2121                     return false;
2122                 }
2123             } else if (!identifier.equals(other.identifier)) {
2124                 return false;
2125             }
2126             if (vpnInterface == null) {
2127                 if (other.vpnInterface != null) {
2128                     return false;
2129                 }
2130             } else if (!vpnInterface.equals(other.vpnInterface)) {
2131                 return false;
2132             }
2133             return true;
2134         }
2135     }
2136
2137     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2138         String primaryRd = vpnUtil.getVpnRd(vpnName);
2139         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2140         if (vpnInstanceOpData == null) {
2141             return;
2142         }
2143         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2144         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2145             return;
2146         }
2147         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2148         vpnToDpnLists.forEach(vpnToDpnList -> {
2149             if (vpnToDpnList.getVpnInterfaces() == null) {
2150                 return;
2151             }
2152             vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2153                 try {
2154                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2155                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2156                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2157                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2158                     if (!vpnInterfaceOptional.isPresent()) {
2159                         return;
2160                     }
2161                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2162                             vpnInterface.getInterfaceName());
2163                     if (configVpnAdjacencies == null) {
2164                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2165                         return;
2166                     }
2167                     List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2168                             .augmentation(AdjacenciesOp.class).getAdjacency();
2169                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2170                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2171                     // in operational DS. These unprocessed adjacencies will be handled below.
2172                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2173                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2174                     configVpnAdjacencies.stream()
2175                         .filter(adjacency -> operationVpnAdjacencies.stream()
2176                                 .noneMatch(operationalAdjacency ->
2177                                         operationalAdjacency.getIpAddress().equals(adjacency.getIpAddress())))
2178                         .forEach(adjacency -> {
2179                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2180                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2181                                 () -> {
2182                                     // TODO Deal with sequencing — the config tx must only submitted
2183                                     // if the oper tx goes in
2184                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2185                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2186                                         futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(operTx ->
2187                                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
2188                                                 confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2189                                                     primaryRd, adjacency, vpnInterfaceOptional.get()
2190                                                         .getDpnId(), confTx, operTx)))));
2191                                         return futures;
2192                                     } else {
2193                                         return Collections.emptyList();
2194                                     }
2195                                 });
2196                         });
2197                 } catch (ReadFailedException e) {
2198                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2199                             vpnName, primaryRd);
2200                 }
2201             });
2202         });
2203     }
2204
2205     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2206         private final String interfaceName;
2207         private final boolean add;
2208         private final String txnDestination;
2209
2210         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2211             this.interfaceName = interfaceName;
2212             this.add = add;
2213             this.txnDestination = transactionDest;
2214         }
2215
2216         @Override
2217         public void onSuccess(Void voidObj) {
2218             if (add) {
2219                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2220                         interfaceName, txnDestination);
2221             } else {
2222                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2223             }
2224         }
2225
2226         @Override
2227         public void onFailure(Throwable throwable) {
2228             if (add) {
2229                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2230                         interfaceName, txnDestination, throwable);
2231             } else {
2232                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2233                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2234             }
2235         }
2236     }
2237 }