vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.collect.Iterators;
  13 import com.google.common.util.concurrent.FutureCallback;
  14 import com.google.common.util.concurrent.Futures;
  15 import com.google.common.util.concurrent.ListenableFuture;
  16 import com.google.common.util.concurrent.MoreExecutors;
  17 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  18 import java.math.BigInteger;
  19 import java.util.ArrayList;
  20 import java.util.Collections;
  21 import java.util.Iterator;
  22 import java.util.List;
  23 import java.util.Map;
  24 import java.util.Objects;
  25 import java.util.concurrent.ConcurrentHashMap;
  26 import java.util.concurrent.ConcurrentLinkedQueue;
  27 import java.util.function.Consumer;
  28 import java.util.function.Predicate;
  29 import java.util.stream.Collectors;
  30 import javax.annotation.PostConstruct;
  31 import javax.annotation.PreDestroy;
  32 import javax.inject.Inject;
  33 import javax.inject.Singleton;
  34 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  35 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  36 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  37 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  38 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  39 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  40 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  41 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  42 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  43 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  44 import org.opendaylight.genius.mdsalutil.NWUtil;
  45 import org.opendaylight.genius.mdsalutil.NwConstants;
  46 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  47 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  48 import org.opendaylight.infrautils.caches.CacheProvider;
  49 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  50 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  51 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  52 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  53 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  54 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  55 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  56 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  57 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  58 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  59 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  60 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  61 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  62 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  63 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  64 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  65 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  66 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  67 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.NetworkAttributes.NetworkType;
 106 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 107 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 108 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 109 import org.slf4j.Logger;
 110 import org.slf4j.LoggerFactory;
 111
 112 @Singleton
 113 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 114
 115     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 116     private static final short DJC_MAX_RETRIES = 3;
 117
 118     private final DataBroker dataBroker;
 119     private final ManagedNewTransactionRunner txRunner;
 120     private final IBgpManager bgpManager;
 121     private final IFibManager fibManager;
 122     private final IMdsalApiManager mdsalManager;
 123     private final IdManagerService idManager;
 124     private final OdlInterfaceRpcService ifaceMgrRpcService;
 125     private final VpnFootprintService vpnFootprintService;
 126     private final IInterfaceManager interfaceManager;
 127     private final IVpnManager vpnManager;
 128     private final ArpResponderHandler arpResponderHandler;
 129     private final JobCoordinator jobCoordinator;
 130     private final VpnUtil vpnUtil;
 131
 132     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 133
 134     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 135             new ConcurrentHashMap<>();
 136
 137     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 138
 139     @Inject
 140     public VpnInterfaceManager(final DataBroker dataBroker,
 141                                final IBgpManager bgpManager,
 142                                final IdManagerService idManager,
 143                                final IMdsalApiManager mdsalManager,
 144                                final IFibManager fibManager,
 145                                final OdlInterfaceRpcService ifaceMgrRpcService,
 146                                final VpnFootprintService vpnFootprintService,
 147                                final IInterfaceManager interfaceManager,
 148                                final IVpnManager vpnManager,
 149                                final ArpResponderHandler arpResponderHandler,
 150                                final JobCoordinator jobCoordinator,
 151                                final CacheProvider cacheProvider,
 152                                final VpnUtil vpnUtil) {
 153         super(VpnInterface.class, VpnInterfaceManager.class);
 154
 155         this.dataBroker = dataBroker;
 156         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 157         this.bgpManager = bgpManager;
 158         this.idManager = idManager;
 159         this.mdsalManager = mdsalManager;
 160         this.fibManager = fibManager;
 161         this.ifaceMgrRpcService = ifaceMgrRpcService;
 162         this.vpnFootprintService = vpnFootprintService;
 163         this.interfaceManager = interfaceManager;
 164         this.vpnManager = vpnManager;
 165         this.arpResponderHandler = arpResponderHandler;
 166         this.jobCoordinator = jobCoordinator;
 167         this.vpnUtil = vpnUtil;
 168
 169         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 170                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 171                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 172     }
 173
 174     public Runnable isNotifyTaskQueued(String intfName) {
 175         return vpnIntfMap.remove(intfName);
 176     }
 177
 178     @PostConstruct
 179     public void start() {
 180         LOG.info("{} start", getClass().getSimpleName());
 181         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 182     }
 183
 184     @Override
 185     @PreDestroy
 186     public void close() {
 187         super.close();
 188         vpnInstanceOpDataEntryCache.close();
 189     }
 190
 191     @Override
 192     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 193         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 194     }
 195
 196     @Override
 197     protected VpnInterfaceManager getDataTreeChangeListener() {
 198         return VpnInterfaceManager.this;
 199     }
 200
 201     @Override
 202     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 203         LOG.info("add: intfName {} onto vpnName {}",
 204                  vpnInterface.getName(),
 205                  VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 206         addVpnInterface(identifier, vpnInterface, null, null);
 207     }
 208
 209     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 210                           final VpnInterface vpnInterface, String vpnName) {
 211         synchronized (vpnName.intern()) {
 212             if (isVpnInstanceReady(vpnName)) {
 213                 return true;
 214             }
 215             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 216             return false;
 217         }
 218     }
 219
 220     // TODO Clean up the exception handling
 221     @SuppressWarnings("checkstyle:IllegalCatch")
 222     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 223                              final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs) {
 224         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
 225             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 226             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 227         }
 228     }
 229
 230     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 231                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 232         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 233         final String interfaceName = key.getName();
 234
 235         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 236             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 237                   vpnName, vpnInterface.getName());
 238             return;
 239         }
 240         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 241              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 242         List<Adjacency> copyOldAdjs = null;
 243         if (oldAdjs != null) {
 244             copyOldAdjs = new ArrayList<>();
 245             copyOldAdjs.addAll(oldAdjs);
 246         }
 247         List<Adjacency> copyNewAdjs = null;
 248         if (newAdjs != null) {
 249             copyNewAdjs = new ArrayList<>();
 250             copyNewAdjs.addAll(newAdjs);
 251         }
 252         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 253     }
 254
 255     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 256                     final VpnInterface vpnInterface, final List<Adjacency> oldAdjs,
 257                     final List<Adjacency> newAdjs,
 258                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 259         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 260         final String interfaceName = key.getName();
 261         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 262         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 263             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 264             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 265             if (interfaceState != null) {
 266                 try {
 267                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 268                     final int ifIndex = interfaceState.getIfIndex();
 269                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 270                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 271                         // (the inventory tx goes in last)
 272                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 273                         ListenableFuture<Void> confFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 274                             confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 275                                 operTx -> futures.add(
 276                                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(invTx -> {
 277                                             LOG.info(
 278                                                     "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 279                                                     + " on dpn {}",
 280                                                     vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 281                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 282                                                     confTx, operTx, invTx, interfaceState, vpnName);
 283                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 284                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 285                                                         + " Update for swapping VPN {} case.", interfaceName, vpnName);
 286                                                 if (newAdjs != null) {
 287                                                     for (Adjacency adj : newAdjs) {
 288                                                         if (oldAdjs.contains(adj)) {
 289                                                             oldAdjs.remove(adj);
 290                                                         } else {
 291                                                             if (!isBgpVpnInternetVpn
 292                                                                     || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 293                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 294                                                                         primaryRd, adj, dpnId, operTx, confTx);
 295                                                             }
 296                                                         }
 297                                                     }
 298                                                 }
 299                                                 for (Adjacency adj : oldAdjs) {
 300                                                     if (!isBgpVpnInternetVpn
 301                                                             || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 302                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 303                                                                 operTx, confTx);
 304                                                     }
 305                                                 }
 306                                             }
 307                                         })))));
 308                         futures.add(confFuture);
 309                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 310                                 MoreExecutors.directExecutor());
 311                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 312                                 + " processed successfully", interfaceName, vpnName, dpnId);
 313                         return futures;
 314                     });
 315                 } catch (NumberFormatException | IllegalStateException e) {
 316                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 317                                     + "interface {}. Interface addition on vpn {} failed", interfaceName,
 318                                     vpnName, e);
 319                     return;
 320                 }
 321             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 322                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 323                     () -> {
 324                         ListenableFuture<Void> future = txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
 325                             createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 326                                     confTx, vpnName);
 327                             LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 328                                     vpnName, vpnInterface.getDpnId());
 329                         });
 330                         ListenableFutures.addErrorLogging(future, LOG,
 331                                 "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 332                         return Collections.singletonList(future);
 333                     });
 334             } else {
 335                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 336                         + " is not available", interfaceName, vpnName);
 337             }
 338         } else {
 339             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 340                       + " as vpn is pending delete", interfaceName, vpnName,
 341                     vpnInterface.getDpnId());
 342         }
 343     }
 344
 345     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 346     // see comments below.
 347     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 348     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 349             final int lportTag, boolean isInterfaceUp,
 350             WriteTransaction writeConfigTxn,
 351             WriteTransaction writeOperTxn,
 352             WriteTransaction writeInvTxn,
 353             Interface interfaceState,
 354             final String vpnName) {
 355         final String interfaceName = vpnInterface.getName();
 356         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 357                 vpnName);
 358         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 359         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 360         if (!isInterfaceUp) {
 361             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 362                      interfaceName, dpId, vpnName);
 363             long vpnId = vpnUtil.getVpnId(vpnName);
 364             if (vpnId == VpnConstants.INVALID_ID) {
 365                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 366                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 367                         dpId);
 368                 return;
 369             }
 370
 371             boolean waitForVpnInterfaceOpRemoval = false;
 372             if (opVpnInterface != null) {
 373                 String opVpnName = opVpnInterface.getVpnInstanceName();
 374                 String primaryInterfaceIp = null;
 375                 if (opVpnName.equals(vpnName)) {
 376                     // Please check if the primary VRF Entry does not exist for VPNInterface
 377                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 378                     // back to back
 379                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 380                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 381                     if (adjs == null) {
 382                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 383                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 384                                 vpnName);
 385                         return;
 386                     }
 387                     for (Adjacency adj : adjs) {
 388                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 389                             primaryInterfaceIp = adj.getIpAddress();
 390                             break;
 391                         }
 392                     }
 393                     if (primaryInterfaceIp == null) {
 394                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 395                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 396                                 dpId, vpnName);
 397                         return;
 398                     }
 399                     // Get the rd of the vpn instance
 400                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 401                     if (vrf != null) {
 402                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 403                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 404                         return;
 405                     }
 406                     waitForVpnInterfaceOpRemoval = true;
 407                 } else {
 408                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 409                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 410                 }
 411             }
 412             if (!waitForVpnInterfaceOpRemoval) {
 413                 // Add the VPNInterface and quit
 414                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 415                         null/*ipAddressSourceValuePair*/,
 416                         true /* add */);
 417                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 418                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 419                 if (!isBgpVpnInternetVpn) {
 420                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 421                 }
 422                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 423                         dpId, vpnName);
 424                 if (interfaceManager.isExternalInterface(interfaceName)) {
 425                     processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
 426                         NwConstants.ADD_FLOW);
 427                 }
 428                 return;
 429             }
 430
 431             // FIB didn't get a chance yet to clean up this VPNInterface
 432             // Let us give it a chance here !
 433             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 434                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 435             try {
 436                 Runnable notifyTask = new VpnNotifyTask();
 437                 synchronized (notifyTask) {
 438                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 439                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 440                     // published here while holding the lock on it so this path will hit the wait before notify can be
 441                     // invoked.
 442                     vpnIntfMap.put(interfaceName, notifyTask);
 443                     try {
 444                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 445                     } catch (InterruptedException e) {
 446                         // Ignored
 447                     }
 448                 }
 449             } finally {
 450                 vpnIntfMap.remove(interfaceName);
 451             }
 452
 453             if (opVpnInterface != null) {
 454                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 455                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 456                         dpId, vpnName);
 457                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 458                 return;
 459             }
 460             // VPNInterface got removed, proceed with Add
 461             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 462                     interfaceName, dpId, vpnName);
 463             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 464                     null/*ipAddressSourceValuePair*/,
 465                     true /* add */);
 466             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 467                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 468             if (!isBgpVpnInternetVpn) {
 469                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 470             }
 471             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 472                     + " FIB to clean up", interfaceName, dpId, vpnName);
 473             if (interfaceManager.isExternalInterface(interfaceName)) {
 474                 processExternalVpnInterface(interfaceName, vpnName, dpId,
 475                                lportTag, NwConstants.ADD_FLOW);
 476             }
 477         } else {
 478             try {
 479                 // Interface is retained in the DPN, but its Link Up.
 480                 // Advertise prefixes again for this interface to BGP
 481                 InstanceIdentifier<VpnInterface> identifier =
 482                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 483                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 484                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 485                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 486                 // Perform similar operation as interface add event for extraroutes.
 487                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 488                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 489                         LogicalDatastoreType.CONFIGURATION, path);
 490                 if (!optAdjacencies.isPresent()) {
 491                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 492                     return;
 493                 }
 494                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
 495                 for (Adjacency adjacency : adjacencies) {
 496                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 497                         continue;
 498                     }
 499                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 500                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 501                         continue;
 502                     }
 503                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 504                             dpId, writeOperTxn, writeConfigTxn);
 505                 }
 506             } catch (ReadFailedException e) {
 507                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 508                         interfaceName, vpnName, primaryRd, dpId);
 509             }
 510         }
 511     }
 512
 513     private void processExternalVpnInterface(String interfaceName, String vpnName, BigInteger dpId,
 514         int lportTag, int addOrRemove) {
 515         Uuid extNetworkId;
 516         try {
 517             // vpn instance of ext-net interface is the network-id
 518             extNetworkId = new Uuid(vpnName);
 519         } catch (IllegalArgumentException e) {
 520             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 521                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 522             return;
 523         }
 524
 525         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 526         if (routerIds == null || routerIds.isEmpty()) {
 527             LOG.info("processExternalVpnInterface: No router is associated with {}."
 528                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 529                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 530             return;
 531         }
 532
 533         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 534                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 535         for (Uuid routerId : routerIds) {
 536             String routerName = routerId.getValue();
 537             BigInteger primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 538             if (Objects.equals(primarySwitch, dpId)) {
 539                 Routers router = vpnUtil.getExternalRouter(routerName);
 540                 if (router != null) {
 541                     if (addOrRemove == NwConstants.ADD_FLOW) {
 542                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 543                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 544                                 dpId, interfaceName, lportTag);
 545                     } else {
 546                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 547                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 548                                 dpId, interfaceName, lportTag);
 549                     }
 550                 } else {
 551                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 552                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 553                             interfaceName, dpId, vpnName);
 554                 }
 555             }
 556         }
 557     }
 558
 559     // TODO Clean up the exception handling
 560     @SuppressWarnings("checkstyle:IllegalCatch")
 561     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 562                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 563                                                   String vpnName, String interfaceName) {
 564         if (rd == null) {
 565             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 566                   interfaceName, dpnId, vpnName);
 567             return;
 568         } else {
 569             if (rd.equals(vpnName)) {
 570                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 571                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 572                 return;
 573             }
 574         }
 575         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 576                 interfaceName, dpnId, vpnName, rd);
 577
 578         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 579         if (nextHopIp == null) {
 580             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 581                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 582                     rd, vpnName);
 583             return;
 584         }
 585
 586         try {
 587             //Read NextHops
 588             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 589             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 590                     LogicalDatastoreType.OPERATIONAL, path);
 591             if (adjacencies.isPresent()) {
 592                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 593                 if (!nextHops.isEmpty()) {
 594                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 595                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 596                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 597                     long l3vni = vpnInstanceOpData.getL3vni();
 598                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 599                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 600                     for (Adjacency nextHop : nextHops) {
 601                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 602                             continue;
 603                         }
 604                         String gatewayMac = null;
 605                         long label = 0;
 606                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 607                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 608                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 609                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 610                                     .get();
 611                         } else {
 612                             label = nextHop.getLabel();
 613                         }
 614                         try {
 615                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 616                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 617                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 618                                     encapType, (int)label, l3vni, 0 /*l2vni*/,
 619                                     gatewayMac);
 620                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 621                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 622                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 623                         } catch (Exception e) {
 624                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 625                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 626                                     interfaceName, dpnId, e);
 627                         }
 628                     }
 629                 }
 630             }
 631         } catch (ReadFailedException e) {
 632             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 633                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 634         }
 635     }
 636
 637     // TODO Clean up the exception handling
 638     @SuppressWarnings("checkstyle:IllegalCatch")
 639     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 640                         String vpnName, String interfaceName, WriteTransaction writeConfigTxn,
 641                         WriteTransaction writeOperTx) {
 642         //Read NextHops
 643         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 644         String rd = vpnUtil.getVpnRd(interfaceName);
 645         if (rd == null) {
 646             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 647                 interfaceName, vpnName);
 648             return;
 649         } else {
 650             if (rd.equals(vpnName)) {
 651                 LOG.info(
 652                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 653                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 654                 return;
 655             }
 656         }
 657         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 658                vpnName, rd);
 659         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 660         try {
 661             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 662                     path);
 663         } catch (ReadFailedException e) {
 664             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 665                     interfaceName, vpnName);
 666         }
 667         if (adjacencies.isPresent()) {
 668             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 669
 670             if (!nextHops.isEmpty()) {
 671                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 672                         nextHops, interfaceName, vpnName, rd);
 673                 for (Adjacency nextHop : nextHops) {
 674                     try {
 675                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 676                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 677                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 678                                     interfaceName, vpnName);
 679                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 680                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 681                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 682                                     interfaceName, vpnName);
 683                         } else {
 684                             // Perform similar operation as interface delete event for extraroutes.
 685                             String allocatedRd = nextHop.getVrfId();
 686                             for (String nh : nextHop.getNextHopIpList()) {
 687                                 deleteExtraRouteFromCurrentAndImportingVpns(
 688                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
 689                                     writeOperTx);
 690                             }
 691                         }
 692                     } catch (Exception e) {
 693                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 694                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 695                     }
 696                 }
 697             }
 698         }
 699     }
 700
 701     @SuppressWarnings("checkstyle:IllegalCatch")
 702     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 703                                                   String primaryRd, String interfaceName, final long vpnId,
 704                                                   WriteTransaction writeConfigTxn,
 705                                                   WriteTransaction writeOperTxn,
 706                                                   final WriteTransaction writeInvTxn,
 707                                                   Interface interfaceState) {
 708         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 709         // Read NextHops
 710         Optional<VpnInterface> vpnInteface = Optional.absent();
 711         try {
 712             vpnInteface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 713             LogicalDatastoreType.CONFIGURATION, identifier);
 714         } catch (ReadFailedException e) {
 715             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 716                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 717         }
 718         Uuid intfnetworkUuid = null;
 719         NetworkType networkType = null;
 720         Long segmentationId = Long.valueOf(-1);
 721         Adjacencies adjacencies = null;
 722         if (vpnInteface.isPresent()) {
 723             intfnetworkUuid = vpnInteface.get().getNetworkId();
 724             networkType = vpnInteface.get().getNetworkType();
 725             segmentationId = vpnInteface.get().getSegmentationId();
 726             adjacencies = vpnInteface.get().augmentation(Adjacencies.class);
 727             if (adjacencies == null) {
 728                 addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 729                         null/*gwMac*/, writeOperTxn);
 730                 return;
 731             }
 732         }
 733         // Get the rd of the vpn instance
 734         String nextHopIp = null;
 735         try {
 736             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 737         } catch (Exception e) {
 738             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 739                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 740         }
 741         List<String> nhList = new ArrayList<>();
 742         if (nextHopIp != null) {
 743             nhList.add(nextHopIp);
 744             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 745                     interfaceName, dpnId, vpnName, nhList);
 746         }
 747         Optional<String> gwMac = Optional.absent();
 748         String vpnInterfaceSubnetGwMacAddress = null;
 749         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 750         Long l3vni = vpnInstanceOpData.getL3vni();
 751         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 752         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 753         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 754         List<Adjacency> nextHops = (adjacencies != null) ? adjacencies.getAdjacency() : Collections.emptyList();
 755         List<Adjacency> value = new ArrayList<>();
 756         for (Adjacency nextHop : nextHops) {
 757             String rd = primaryRd;
 758             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 759             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 760                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 761                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 762                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 763                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 764                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 765                 continue;
 766             }
 767             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 768                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 769                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 770                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 771                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 772                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 773
 774                 Prefixes prefixes = (intfnetworkUuid != null)
 775                     ? VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, intfnetworkUuid ,networkType,
 776                             segmentationId, prefixCue) :
 777                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, prefixCue);
 778                 writeOperTxn.merge(
 779                     LogicalDatastoreType.OPERATIONAL,
 780                     VpnUtil.getPrefixToInterfaceIdentifier(
 781                         vpnUtil.getVpnId(vpnName), prefix), prefixes, true);
 782                 final Uuid subnetId = nextHop.getSubnetId();
 783
 784                 String gatewayIp = nextHop.getSubnetGatewayIp();
 785                 if (gatewayIp == null) {
 786                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 787                     if (gatewayIpOptional.isPresent()) {
 788                         gatewayIp = gatewayIpOptional.get();
 789                     }
 790                 }
 791
 792                 if (gatewayIp != null) {
 793                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 794                     if (gwMac.isPresent()) {
 795                         // A valid mac-address is available for this subnet-gateway-ip
 796                         // Use this for programming ARP_RESPONDER table here.  And save this
 797                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 798                         // to populate L3_GW_MAC_TABLE there.
 799                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 800                                 gatewayIp, gwMac.get());
 801                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 802                     } else {
 803                         // A valid mac-address is not available for this subnet-gateway-ip
 804                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 805                         // Save this connected-mac-address as gateway-mac-address for the
 806                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 807                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 808                         if (gwMac.isPresent()) {
 809                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 810                                     NwConstants.ADD_FLOW, gwMac.get());
 811                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 812                                     gatewayIp, gwMac.get());
 813                         } else {
 814                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 815                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 816                                 + "gwIp {}",
 817                                 subnetId, interfaceName, vpnName, gatewayIp);
 818                         }
 819                     }
 820                 } else {
 821                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 822                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 823                         subnetId, interfaceName, vpnName);
 824                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 825                 }
 826                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 827                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 828             } else {
 829                 //Extra route adjacency
 830                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 831                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 832                 synchronized (vpnPrefixKey.intern()) {
 833                     java.util.Optional<String> rdToAllocate = vpnUtil
 834                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 835                                     nextHop.getNextHopIpList().get(0), dpnId);
 836                     if (rdToAllocate.isPresent()) {
 837                         rd = rdToAllocate.get();
 838                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 839                             rd, prefix);
 840                     } else {
 841                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 842                         continue;
 843                     }
 844                 }
 845                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 846                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 847                         interfaceName, dpnId);
 848             }
 849             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 850             // can be different from the gateway-mac-address within the VRFEntry as the
 851             // gateway-mac-address is a superset.
 852             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 853                     : RouteOrigin.STATIC;
 854             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 855                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 856                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 857             Adjacency operationalAdjacency = null;
 858             try {
 859                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 860             } catch (NullPointerException e) {
 861                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 862                     input, e.getMessage());
 863                 return;
 864             }
 865             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 866                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 867                         vpnName, l3vni, origin,
 868                         interfaceName, operationalAdjacency, encapType, writeConfigTxn);
 869             }
 870             value.add(operationalAdjacency);
 871         }
 872
 873         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 874         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 875                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 876
 877         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 878                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 879                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 880
 881         for (Adjacency nextHop : aug.getAdjacency()) {
 882             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 883             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 884                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 885                         : RouteOrigin.STATIC;
 886                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 887                 registeredPopulator.populateFib(input, writeConfigTxn);
 888             }
 889         }
 890     }
 891
 892     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 893                                               long lportTag, String gwMac, WriteTransaction writeOperTxn) {
 894         VpnInterfaceOpDataEntry opInterface =
 895               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac);
 896         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 897             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 898         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 899                 WriteTransaction.CREATE_MISSING_PARENTS);
 900         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 901                 interfaceName, dpnId, vpnName);
 902     }
 903
 904     // TODO Clean up the exception handling
 905     @SuppressWarnings("checkstyle:IllegalCatch")
 906     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 907                                            StateTunnelList stateTunnelList,
 908                                            WriteTransaction writeConfigTxn,
 909                                            WriteTransaction writeOperTxn) {
 910
 911         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
 912         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 913         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 914         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 915         if (adjList.isEmpty()) {
 916             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 917                     vpnInterface, srcDpnId);
 918             return;
 919         }
 920         String prefix = null;
 921         long label = 0;
 922         List<Adjacency> value = new ArrayList<>();
 923         boolean isNextHopAddReqd = false;
 924         String vpnName = vpnInterface.getVpnInstanceName();
 925         long vpnId = vpnUtil.getVpnId(vpnName);
 926         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 927         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 928                 vpnInterface.getName(), vpnInterface.getDpnId(),
 929                 vpnInterface.getVpnInstanceName(), adjList);
 930         for (Adjacency adj : adjList) {
 931             String rd = adj.getVrfId();
 932             rd = rd != null ? rd : vpnName;
 933             prefix = adj.getIpAddress();
 934             label = adj.getLabel();
 935             List<String> nhList = Collections.singletonList(srcTepIp);
 936             List<String> nextHopList = adj.getNextHopIpList();
 937             // If TEP is added , update the nexthop of primary adjacency.
 938             // Secondary adj nexthop is already pointing to primary adj IP address.
 939             if (nextHopList != null && !nextHopList.isEmpty()) {
 940                 /* everything right already */
 941             } else {
 942                 isNextHopAddReqd = true;
 943             }
 944
 945             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 946                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 947             } else {
 948                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 949                 if (!vrfEntryOptional.isPresent()) {
 950                     continue;
 951                 }
 952                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 953                 if (!nhList.contains(srcTepIp)) {
 954                     nhList.add(srcTepIp);
 955                     isNextHopAddReqd = true;
 956                 }
 957                 value.add(adj);
 958             }
 959
 960             if (isNextHopAddReqd) {
 961                 updateLabelMapper(label, nhList);
 962                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 963                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 964                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 965                 // Update the VRF entry with nextHop
 966                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 967                         label, true, writeConfigTxn);
 968
 969                 //Get the list of VPN's importing this route(prefix) .
 970                 // Then update the VRF entry with nhList
 971                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
 972                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 973                     String vpnRd = vpn.getVrfId();
 974                     if (vpnRd != null) {
 975                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
 976                                 srcTepIp, label, true, writeConfigTxn);
 977                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 978                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
 979                             vpnInterface.getName(), srcDpnId, vpnName,
 980                             vpn.getVpnInstanceName(), vpnRd);
 981                     }
 982                 }
 983                 // Advertise the prefix to BGP only for external vpn
 984                 // since there is a nexthop change.
 985                 try {
 986                     if (!rd.equalsIgnoreCase(vpnName)) {
 987                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
 988                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
 989                                 null /*gatewayMacAddress*/);
 990                     }
 991                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
 992                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
 993                             srcDpnId, vpnName);
 994                 } catch (Exception ex) {
 995                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
 996                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
 997                             vpnInterface.getName(), srcDpnId, vpnName, ex);
 998                 }
 999             }
1000         }
1001         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1002         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1003                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1004                 .addAugmentation(AdjacenciesOp.class, aug).build();
1005         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1006                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1007         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
1008                 WriteTransaction.CREATE_MISSING_PARENTS);
1009         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
1010                 vpnInterface.getName(), srcDpnId, vpnName);
1011
1012     }
1013
1014     // TODO Clean up the exception handling
1015     @SuppressWarnings("checkstyle:IllegalCatch")
1016     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1017                                               StateTunnelList stateTunnelList,
1018                                               WriteTransaction writeConfigTxn,
1019                                               WriteTransaction writeOperTxn) {
1020
1021         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1022         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1023         String prefix = null;
1024         long label = 0;
1025         boolean isNextHopRemoveReqd = false;
1026         String srcTepIp = stateTunnelList.getSrcInfo().getTepIp().stringValue();
1027         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
1028         String vpnName = vpnInterface.getVpnInstanceName();
1029         long vpnId = vpnUtil.getVpnId(vpnName);
1030         String primaryRd = vpnUtil.getVpnRd(vpnName);
1031         if (adjList != null) {
1032             List<Adjacency> value = new ArrayList<>();
1033             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1034                     vpnInterface.getName(), vpnInterface.getDpnId(),
1035                     vpnInterface.getVpnInstanceName(), adjList);
1036             for (Adjacency adj : adjList) {
1037                 List<String> nhList = new ArrayList<>();
1038                 String rd = adj.getVrfId();
1039                 rd = rd != null ? rd : vpnName;
1040                 prefix = adj.getIpAddress();
1041                 List<String> nextHopList = adj.getNextHopIpList();
1042                 label = adj.getLabel();
1043                 if (nextHopList != null && !nextHopList.isEmpty()) {
1044                     isNextHopRemoveReqd = true;
1045                 }
1046                 // If TEP is deleted , remove the nexthop from primary adjacency.
1047                 // Secondary adj nexthop will continue to point to primary adj IP address.
1048                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1049                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1050                 } else {
1051                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1052                     if (!vrfEntryOptional.isPresent()) {
1053                         continue;
1054                     }
1055                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1056                     if (nhList.contains(srcTepIp)) {
1057                         nhList.remove(srcTepIp);
1058                         isNextHopRemoveReqd = true;
1059                     }
1060                     value.add(adj);
1061                 }
1062
1063                 if (isNextHopRemoveReqd) {
1064                     updateLabelMapper(label, nhList);
1065                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1066                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1067                             prefix, nhList, vpnName,
1068                             vpnId, rd, vpnInterface.getName());
1069                     // Update the VRF entry with removed nextHop
1070                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1071                             label, false, writeConfigTxn);
1072
1073                     //Get the list of VPN's importing this route(prefix) .
1074                     // Then update the VRF entry with nhList
1075                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1076                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1077                         String vpnRd = vpn.getVrfId();
1078                         if (vpnRd != null) {
1079                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1080                                     srcTepIp, label, false, writeConfigTxn);
1081                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1082                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1083                                     nhList, label, vpnInterface.getName(), srcDpnId,
1084                                     vpnName,
1085                                     vpn.getVpnInstanceName(), vpnRd);
1086                         }
1087                     }
1088
1089                     // Withdraw prefix from BGP only for external vpn.
1090                     try {
1091                         if (!rd.equalsIgnoreCase(vpnName)) {
1092                             bgpManager.withdrawPrefix(rd, prefix);
1093                         }
1094                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1095                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1096                                 vpnInterface.getName(), srcDpnId,
1097                                 vpnName);
1098                     } catch (Exception ex) {
1099                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1100                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1101                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1102                     }
1103                 }
1104             }
1105             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1106             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1107                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1108                     .addAugmentation(AdjacenciesOp.class, aug).build();
1109             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1110                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1111             writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
1112                     WriteTransaction.CREATE_MISSING_PARENTS);
1113             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1114                          vpnInterface.getName(), srcDpnId, vpnName);
1115         }
1116     }
1117
1118     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1119         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1120
1121         String vpnRd = vpnUtil.getVpnRd(vpnName);
1122         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1123         if (vpnInstanceOpDataEntry == null) {
1124             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1125                     + " to check for vpns exporting the routes", vpnName);
1126             return vpnsToExportRoute;
1127         }
1128
1129         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1130             if (input.getVpnInstanceName() == null) {
1131                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1132                         input.getVrfId());
1133                 return false;
1134             }
1135             return !input.getVpnInstanceName().equals(vpnName);
1136         };
1137
1138         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1139             Iterable<String> commonRTs =
1140                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1141                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1142             return Iterators.size(commonRTs.iterator()) > 0;
1143         };
1144
1145         vpnsToExportRoute =
1146                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1147                         Collectors.toList());
1148         return vpnsToExportRoute;
1149     }
1150
1151     // TODO Clean up the exception handling
1152     @SuppressWarnings("checkstyle:IllegalCatch")
1153     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1154         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1155         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1156             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1157             if (vrfEntries != null) {
1158                 ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1159                     for (VrfEntry vrfEntry : vrfEntries) {
1160                         try {
1161                             if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1162                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1163                                 LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1164                                                 + " is not a controller managed non intervpn link route. Ignoring.",
1165                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1166                                 continue;
1167                             }
1168                             String prefix = vrfEntry.getDestPrefix();
1169                             String gwMac = vrfEntry.getGatewayMacAddress();
1170                             vrfEntry.getRoutePaths().forEach(routePath -> {
1171                                 String nh = routePath.getNexthopAddress();
1172                                 int label = routePath.getLabel().intValue();
1173                                 if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1174                                         vrfEntry.getOrigin()))) {
1175                                     LOG.info("handleVpnsExportingRoutesImporting: Importing fib entry rd {} prefix {}"
1176                                                     + " nexthop {} label {} to vpn {} vpnRd {}",
1177                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1178                                     fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1179                                             Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1180                                             0 /*l3vni*/, gwMac,  vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1181                                             confTx);
1182                                 } else {
1183                                     LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry rd {} "
1184                                                     + "prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1185                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1186                                     SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1187                                     importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1188                                             confTx);
1189                                 }
1190                             });
1191                         } catch (RuntimeException e) {
1192                             LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1193                                             + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1194                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1195                         }
1196                     }
1197                 }), LOG, "Error handing VPN exporting routes");
1198             } else {
1199                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1200                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1201             }
1202         }
1203     }
1204
1205     @Override
1206     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1207         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1208         final String interfaceName = key.getName();
1209         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
1210             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1211             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1212         }
1213     }
1214
1215     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1216                                 final VpnInterface vpnInterface, final String vpnName,
1217                                 final String interfaceName) {
1218         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1219             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1220                 ListenableFuture<Void> future = txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1221                     deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1222                     LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1223                 });
1224                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1225                         vpnInterface.getName(), vpnName);
1226                 return Collections.singletonList(future);
1227             }, DJC_MAX_RETRIES);
1228         } else {
1229             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName);
1230         }
1231     }
1232
1233     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1234     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1235                                 final VpnInterface vpnInterface, final String vpnName,
1236                                 final String interfaceName) {
1237         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1238                 vpnName, vpnInterface.getDpnId());
1239         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1240         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1241             () -> {
1242                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1243                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1244                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1245                         writeOperTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeInvTxn -> {
1246                             LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1247                                     interfaceName, vpnName);
1248                             BigInteger dpId = BigInteger.ZERO;
1249                             int ifIndex = 0;
1250                             String gwMacAddress = null;
1251                             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1252                                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1253                             Optional<VpnInterfaceOpDataEntry> optVpnInterface = Optional.absent();
1254                             try {
1255                                 optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1256                                         LogicalDatastoreType.OPERATIONAL, interfaceId);
1257                             } catch (ReadFailedException e) {
1258                                 LOG.error("remove: Failed to read data store for interface {} vpn {}", interfaceName,
1259                                         vpnName);
1260                                 return;
1261                             }
1262                             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1263                                     interfaceName);
1264                             if (interfaceState != null) {
1265                                 try {
1266                                     dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1267                                 } catch (NumberFormatException | IllegalStateException e) {
1268                                     LOG.error("remove: Unable to retrieve dpnId from interface operational"
1269                                             + " data store for interface {} on dpn {} for vpn {} Fetching"
1270                                                     + " from vpn interface op data store. ", interfaceName,
1271                                             vpnInterface.getDpnId(), vpnName, e);
1272                                     dpId = BigInteger.ZERO;
1273                                 }
1274                                 ifIndex = interfaceState.getIfIndex();
1275                                 gwMacAddress = interfaceState.getPhysAddress().getValue();
1276                             } else {
1277                                 LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1278                                         + " from vpn interface op.", interfaceName);
1279                                 if (optVpnInterface.isPresent()) {
1280                                     VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1281                                     dpId = vpnOpInterface.getDpnId();
1282                                     ifIndex = vpnOpInterface.getLportTag().intValue();
1283                                     gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1284                                 } else {
1285                                     LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1286                                                     + " as interfaceState and vpn interface op is not"
1287                                             + " available", interfaceName, vpnName);
1288                                     return;
1289                                 }
1290                             }
1291                             processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1292                                     optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1293                                     writeConfigTxn, writeOperTxn, writeInvTxn);
1294                             LOG.info(
1295                                     "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1296                                             + "successfully",
1297                                     interfaceName, vpnInterface.getDpnId(), vpnName);
1298                         })))));
1299                 futures.add(configFuture);
1300                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1301                 return futures;
1302             }, DJC_MAX_RETRIES);
1303     }
1304
1305     protected void processVpnInterfaceDown(BigInteger dpId,
1306                                            String interfaceName,
1307                                            int lportTag,
1308                                            String gwMac,
1309                                            VpnInterfaceOpDataEntry vpnOpInterface,
1310                                            boolean isInterfaceStateDown,
1311                                            WriteTransaction writeConfigTxn,
1312                                            WriteTransaction writeOperTxn,
1313                                            WriteTransaction writeInvTxn) {
1314         if (vpnOpInterface == null) {
1315             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1316                     + " as it is not available in operational data store", interfaceName, dpId);
1317             return;
1318         }
1319         final String vpnName = vpnOpInterface.getVpnInstanceName();
1320         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1321                                                     interfaceName, vpnName);
1322         if (!isInterfaceStateDown) {
1323             final long vpnId = vpnUtil.getVpnId(vpnName);
1324             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName,  null);
1325             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1326             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1327                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1328             if (interfaceManager.isExternalInterface(interfaceName)) {
1329                 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1330                     NwConstants.DEL_FLOW);
1331             }
1332             if (!isBgpVpnInternetVpn) {
1333                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1334             }
1335             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1336                     + " successful", interfaceName, dpId, vpnName);
1337         } else {
1338             // Interface is retained in the DPN, but its Link Down.
1339             // Only withdraw the prefixes for this interface from BGP
1340             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1341         }
1342     }
1343
1344     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1345                                           final String vpnName, final long vpnId, String gwMac,
1346                                           WriteTransaction writeConfigTxn, final WriteTransaction writeOperTxn,
1347                                           final WriteTransaction writeInvTxn) {
1348         //Read NextHops
1349         try {
1350             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1351                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1352             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1353             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1354                     LogicalDatastoreType.OPERATIONAL, path);
1355             String primaryRd = vpnUtil.getVpnRd(vpnName);
1356             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1357                     interfaceName, dpnId, vpnName, primaryRd);
1358             if (adjacencies.isPresent() && !adjacencies.get().getAdjacency().isEmpty()) {
1359                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1360                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1361                         interfaceName, dpnId, vpnName, nextHops);
1362                 for (Adjacency nextHop : nextHops) {
1363                     if (nextHop.isPhysNetworkFunc()) {
1364                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1365                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1366                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1367                                 null/*writeCfgTxn*/);
1368                     } else {
1369                         String rd = nextHop.getVrfId();
1370                         List<String> nhList;
1371                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1372                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1373                         } else {
1374                             // This is a primary adjacency
1375                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1376                                     : Collections.emptyList();
1377                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1378                                     interfaceName, writeInvTxn);
1379                         }
1380                         if (!nhList.isEmpty()) {
1381                             if (rd.equals(vpnName)) {
1382                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1383                                 //remove from FIB directly
1384                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1385                                         interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1386                             } else {
1387                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1388                                         interfaceName, writeConfigTxn, writeOperTxn);
1389                             }
1390                         } else {
1391                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1392                                             + " interface {}", nextHop.getIpAddress(), rd,
1393                                     nextHop.getAdjacencyType().toString(), interfaceName);
1394                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1395                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1396                         }
1397                     }
1398                     String ip = nextHop.getIpAddress().split("/")[0];
1399                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1400                     if (vpnVipToPort != null) {
1401                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1402                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1403                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1404                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1405                     }
1406                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1407                     if (vpnPortipToPort != null) {
1408                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1409                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1410                                  + "Interface {} ip {} on dpn {} for vpn {}",
1411                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1412                     }
1413                 }
1414             } else {
1415                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1416                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1417                         + " Removing it.", interfaceName, vpnName, dpnId);
1418                 writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, identifier);
1419             }
1420         } catch (ReadFailedException e) {
1421             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1422                     interfaceName, dpnId, vpnName);
1423         }
1424     }
1425
1426     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName, String interfaceName,
1427                                                             BigInteger dpnId, WriteTransaction writeConfigTxn,
1428                                                             WriteTransaction writeOperTx) {
1429         return (nh) -> {
1430             String primaryRd = vpnUtil.getVpnRd(vpnName);
1431             String prefix = nextHop.getIpAddress();
1432             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1433             LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1434                     nextHop, vpnName, interfaceName, dpnId);
1435             synchronized (vpnNamePrefixKey.intern()) {
1436                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1437                         prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1438                     //If extra-route is present behind at least one VM, then do not remove or update
1439                     //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1440                     //prefixtointerface DS
1441                     return;
1442                 }
1443                 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1444                         writeConfigTxn);
1445             }
1446             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1447                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1448                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1449         };
1450     }
1451
1452     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1453                                            BigInteger dpnId, String rd, String interfaceName,
1454                                            WriteTransaction writeConfigTxn, WriteTransaction writeOperTx) {
1455         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1456                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1457         nhList.forEach((nh) -> {
1458             //IRT: remove routes from other vpns importing it
1459             vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1460                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1461             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1462                 String vpnRd = vpn.getVrfId();
1463                 if (vpnRd != null) {
1464                     fibManager.removeOrUpdateFibEntry(vpnRd,
1465                             nextHop.getIpAddress(), nh, writeConfigTxn);
1466                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1467                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1468                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1469                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1470                 }
1471             }
1472         });
1473     }
1474
1475     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1476                                                  int lportTag, String gwMac, String interfaceName,
1477                                                  WriteTransaction writeInvTxn) {
1478         final Uuid subnetId = nextHop.getSubnetId();
1479         if (nextHop.getSubnetGatewayMacAddress() == null) {
1480             // A valid mac-address was not available for this subnet-gateway-ip
1481             // So a connected-mac-address was used for this subnet and we need
1482             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1483             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1484         }
1485         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1486                 subnetId);
1487     }
1488
1489     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1490                                                   String interfaceName) {
1491         // This is either an extra-route (or) a learned IP via subnet-route
1492         List<String> nhList = null;
1493         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1494         if (nextHopIp == null || nextHopIp.isEmpty()) {
1495             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1496                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1497                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1498                     vpnName);
1499             nhList = Collections.emptyList();
1500         } else {
1501             nhList = Collections.singletonList(nextHopIp);
1502         }
1503         return nhList;
1504     }
1505
1506     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1507         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1508         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1509         // else use connected interface
1510         if (gwPort != null && gwPort.isSubnetIp()) {
1511             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1512                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1513             return Optional.of(gwPort.getMacAddress());
1514         }
1515         return Optional.absent();
1516     }
1517
1518     @Override
1519     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1520         final VpnInterface update) {
1521         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}" ,update.getName(),
1522                 update.getDpnId(), original.getVpnInstanceNames(),
1523                 update.getVpnInstanceNames());
1524         final String vpnInterfaceName = update.getName();
1525         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1526         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1527         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1528             != null ? origAdjs.getAdjacency() : new ArrayList<>();
1529         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1530         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1531             != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1532
1533         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1534         //handles switching between <internal VPN - external VPN>
1535         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1536             if (handleVpnSwapForVpnInterface(identifier, original, update)) {
1537                 LOG.info("update: handled VPNInterface {} on dpn {} update"
1538                                 + "upon VPN swap from oldVpn(s) {} to newVpn(s) {}",
1539                         original.getName(), dpnId,
1540                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1541                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1542                 return Collections.emptyList();
1543             }
1544             List<ListenableFuture<Void>> futures = new ArrayList<>();
1545             for (VpnInstanceNames vpnInterfaceVpnInstance : update.getVpnInstanceNames()) {
1546                 String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1547                 List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1548                 List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1549                 String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1550                 if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1551                     // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1552                     futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1553                         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(operTx -> {
1554                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1555                                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1556                             LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven",
1557                                     update.getName(), newVpnName);
1558                             //handle both addition and removal of adjacencies
1559                             //currently, new adjacency may be an extra route
1560                             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1561                             if (!oldAdjs.equals(newAdjs)) {
1562                                 for (Adjacency adj : copyNewAdjs) {
1563                                     if (copyOldAdjs.contains(adj)) {
1564                                         copyOldAdjs.remove(adj);
1565                                     } else {
1566                                         // add new adjacency - right now only extra route will hit this path
1567                                         if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1568                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1569                                                     dpnId, operTx, confTx);
1570                                         }
1571                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to"
1572                                                 + " vpn interface {} on vpn {} dpnId {}",
1573                                                 adj.getIpAddress(), adj.getNextHopIpList(),
1574                                                 adj.getLabel(), adj.getSubnetId(), update.getName(),
1575                                                 newVpnName, dpnId);
1576                                     }
1577                                 }
1578                                 for (Adjacency adj : copyOldAdjs) {
1579                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1580                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1581                                                 && !adj.isPhysNetworkFunc()) {
1582                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1583                                                     operTx, confTx);
1584                                             Optional<VpnInterfaceOpDataEntry> optVpnInterface = operTx.read(
1585                                                     LogicalDatastoreType.OPERATIONAL, vpnInterfaceOpIdentifier).get();
1586                                             if (optVpnInterface.isPresent()) {
1587                                                 VpnInterfaceOpDataEntry vpnInterfaceOpDataEntry = optVpnInterface.get();
1588                                                 long vpnId = vpnUtil.getVpnId(newVpnName);
1589                                                 vpnUtil.removePrefixToInterfaceAdj(adj, vpnId,
1590                                                         vpnInterfaceOpDataEntry, operTx);
1591                                             } else {
1592                                                 LOG.info("update: Vpninterface {} not present in Operational",
1593                                                         vpnInterfaceName);
1594                                             }
1595                                             //remove FIB entry
1596                                             String vpnRd = vpnUtil.getVpnRd(newVpnName);
1597                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1598                                                     + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1599                                             //remove BGP entry
1600                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1601                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1602                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1603                                             }
1604                                         } else {
1605                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1606                                                     operTx, confTx);
1607                                         }
1608                                     }
1609                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1610                                                     + " vpn interface {} on vpn {}", adj.getIpAddress(), adj
1611                                                     .getNextHopIpList(),
1612                                             adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1613                                 }
1614                             }
1615                         }));
1616                     }));
1617                     for (ListenableFuture<Void> future : futures) {
1618                         ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1619                                 update.getName(), update.getVpnInstanceNames());
1620                     }
1621                 } else {
1622                     LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1623                             + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1624                 }
1625             }
1626             return futures;
1627         });
1628     }
1629
1630     private boolean handleVpnSwapForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1631                                                  VpnInterface original, VpnInterface update) {
1632         boolean isSwap = Boolean.FALSE;
1633         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1634         final String interfaceName = key.getName();
1635         List<String> oldVpnList = original.getVpnInstanceNames().stream()
1636             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1637         List<String> oldVpnListCopy = new ArrayList<>();
1638         oldVpnListCopy.addAll(oldVpnList);
1639         List<String> newVpnList = update.getVpnInstanceNames().stream()
1640             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1641         oldVpnList.removeAll(newVpnList);
1642         newVpnList.removeAll(oldVpnListCopy);
1643         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1644             for (String oldVpnName: oldVpnList) {
1645                 isSwap = Boolean.TRUE;
1646                 LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} remove vpnName {}"
1647                         + " running config-driven swap removal", interfaceName, oldVpnName);
1648                 removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1649                 LOG.info("handleVpnSwapForVpnInterface: Processed Remove for update on VPNInterface {} upon VPN swap"
1650                         + "from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName, newVpnList);
1651             }
1652             //Wait for previous interface bindings to be removed
1653             try {
1654                 Thread.sleep(2000);
1655             } catch (InterruptedException e) {
1656                 //Ignore
1657             }
1658
1659             final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1660             final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1661                     ? origAdjs.getAdjacency() : new ArrayList<>();
1662             final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1663             final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1664                     ? updateAdjs.getAdjacency() : new ArrayList<>();
1665             for (String newVpnName: newVpnList) {
1666                 String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1667                 isSwap = Boolean.TRUE;
1668                 if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1669                     LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} onto vpnName {}"
1670                             + "running config-driven swap addition", interfaceName, newVpnName);
1671                     addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1672                     LOG.info("handleVpnSwapForVpnInterface: Processed Add for update on VPNInterface {}"
1673                                     + "from oldVpn(s) {} to newVpn {} upon VPN swap",
1674                             interfaceName, oldVpnListCopy, newVpnName);
1675                 }
1676             }
1677         }
1678         return isSwap;
1679     }
1680
1681     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1682         try {
1683             Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1684             synchronized (label.toString().intern()) {
1685                 InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1686                         .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1687                 Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1688                         LogicalDatastoreType.OPERATIONAL, lriIid);
1689                 if (opResult.isPresent()) {
1690                     LabelRouteInfo labelRouteInfo =
1691                             new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1692                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1693                             labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1694                 }
1695             }
1696             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1697                     nextHopIpList);
1698         } catch (ReadFailedException e) {
1699             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1700                     nextHopIpList);
1701         } catch (TransactionCommitFailedException e) {
1702             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1703                     nextHopIpList);
1704         }
1705     }
1706
1707     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1708         SubnetRoute route, String parentVpnRd, WriteTransaction writeConfigTxn) {
1709
1710         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1711         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1712                 .addAugmentation(SubnetRoute.class, route).build();
1713         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1714         InstanceIdentifierBuilder<VrfTables> idBuilder =
1715             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1716         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1717         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1718         if (writeConfigTxn != null) {
1719             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1720         } else {
1721             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1722         }
1723         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1724                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1725     }
1726
1727     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1728                                            Adjacency adj, BigInteger dpnId, WriteTransaction writeOperTxn,
1729                                            WriteTransaction writeConfigTxn) {
1730         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1731         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1732         try {
1733             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1734                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1735             if (optVpnInterface.isPresent()) {
1736                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1737                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1738                 String vpnName = currVpnIntf.getVpnInstanceName();
1739                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1740                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1741                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1742                         LogicalDatastoreType.OPERATIONAL, adjPath);
1743                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1744                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1745                 long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1746                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1747                 List<Adjacency> adjacencies;
1748                 if (optAdjacencies.isPresent()) {
1749                     adjacencies = optAdjacencies.get().getAdjacency();
1750                 } else {
1751                     // This code will be hit in case of first PNF adjacency
1752                     adjacencies = new ArrayList<>();
1753                 }
1754                 long vpnId = vpnUtil.getVpnId(vpnName);
1755                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1756                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1757                 Adjacency operationalAdjacency = null;
1758                 //Handling dual stack neutron port primary adjacency
1759                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1760                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1761                             currVpnIntf.getName(), vpnName);
1762                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1763                             currVpnIntf.getName());
1764                     if (interfaceState != null) {
1765                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1766                                 currVpnIntf.getName(),
1767                                 vpnId, writeConfigTxn, writeOperTxn, null, interfaceState);
1768                     }
1769                 }
1770                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1771                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1772                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
1773                             : RouteOrigin.STATIC;
1774                     String nh = adj.getNextHopIpList().get(0);
1775                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1776                     synchronized (vpnPrefixKey.intern()) {
1777                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1778                                 vpnId, null, prefix, vpnName, nh, dpnId);
1779                         if (rdToAllocate.isPresent()) {
1780                             input.setRd(rdToAllocate.get());
1781                             operationalAdjacency = populator.createOperationalAdjacency(input);
1782                             int label = operationalAdjacency.getLabel().intValue();
1783                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1784                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
1785                                     currVpnIntf.getName(), operationalAdjacency, encapType, writeConfigTxn);
1786                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1787                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1788                                     vpnName, label, currVpnIntf.getName(), dpnId);
1789                         } else {
1790                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
1791                                     vpnName, prefix);
1792                             return;
1793                         }
1794                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1795                         // Keeping the MPLS check for now.
1796                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1797                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1798                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1799                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
1800                             vpnsToImportRoute.forEach(vpn -> {
1801                                 if (vpn.getVrfId() != null) {
1802                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(), vpnId, prefix,
1803                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
1804                                             .ifPresent(
1805                                                 rds -> vpnManager.addExtraRoute(
1806                                                         vpnUtil.getVpnName(vpn.getVpnId()),
1807                                                         adj.getIpAddress(), nh, rds,
1808                                                         currVpnIntf.getVpnInstanceName(), l3vni,
1809                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(),
1810                                                         opAdjacency, encapType, writeConfigTxn));
1811                                 }
1812                             });
1813                         }
1814                     }
1815                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1816                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1817                             currVpnIntf.getName(), vpnName);
1818
1819                     InstanceIdentifier<VpnInterface> vpnIfaceConfigidentifier = VpnUtil
1820                             .getVpnInterfaceIdentifier(currVpnIntf.getName());
1821                     Optional<VpnInterface> vpnIntefaceConfig = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1822                             LogicalDatastoreType.CONFIGURATION, vpnIfaceConfigidentifier);
1823                     Prefixes pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1824                             Prefixes.PrefixCue.PhysNetFunc);
1825                     if (vpnIntefaceConfig.isPresent()) {
1826                         pnfPrefix = VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(), prefix,
1827                                 vpnIntefaceConfig.get().getNetworkId(), vpnIntefaceConfig.get().getNetworkType(),
1828                                 vpnIntefaceConfig.get().getSegmentationId(), Prefixes.PrefixCue.PhysNetFunc);
1829                     }
1830
1831                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1832
1833                     writeOperTxn.merge(
1834                             LogicalDatastoreType.OPERATIONAL,
1835                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
1836                                     prefix), pnfPrefix, true);
1837
1838                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1839                             adj.getIpAddress(), Collections.emptyList(), null /* EncapType */, 0 /* label */,
1840                             0 /*l3vni*/, null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1841
1842                     input.setRd(adj.getVrfId());
1843                 }
1844                 if (operationalAdjacency == null) {
1845                     operationalAdjacency = populator.createOperationalAdjacency(input);
1846                 }
1847                 adjacencies.add(operationalAdjacency);
1848                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1849                 VpnInterfaceOpDataEntry newVpnIntf =
1850                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
1851                                 aug, dpnId, currVpnIntf.getLportTag(),
1852                                 currVpnIntf.getGatewayMacAddress());
1853
1854                 writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1855             }
1856         } catch (ReadFailedException e) {
1857             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
1858                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
1859         }
1860     }
1861
1862     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
1863         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
1864         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
1865     }
1866
1867     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
1868             BigInteger dpnId, WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) {
1869         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1870         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1871         try {
1872             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
1873                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1874             if (optVpnInterface.isPresent()) {
1875                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1876                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1877                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1878                         LogicalDatastoreType.OPERATIONAL, path);
1879                 if (optAdjacencies.isPresent()) {
1880                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1881
1882                     if (!adjacencies.isEmpty()) {
1883                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
1884                         Iterator<Adjacency> adjIt = adjacencies.iterator();
1885                         while (adjIt.hasNext()) {
1886                             Adjacency adjElem = adjIt.next();
1887                             if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1888                                 String rd = adjElem.getVrfId();
1889                                 if (adj.getNextHopIpList() != null) {
1890                                     for (String nh : adj.getNextHopIpList()) {
1891                                         deleteExtraRouteFromCurrentAndImportingVpns(
1892                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
1893                                                 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
1894                                     }
1895                                 } else if (adj.isPhysNetworkFunc()) {
1896                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
1897                                             adj.getIpAddress(), adj.getSubnetId());
1898                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
1899                                             writeConfigTxn);
1900                                 }
1901                                 break;
1902                             }
1903
1904                         }
1905                     }
1906                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
1907                             dpnId, adj.getVrfId());
1908                 } else {
1909                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
1910                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
1911                 }
1912             }
1913         } catch (ReadFailedException e) {
1914             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
1915                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
1916         }
1917     }
1918
1919     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
1920         String rd, String intfName, WriteTransaction writeConfigTxn, WriteTransaction writeOperTx) {
1921         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
1922         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1923         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1924             String vpnRd = vpn.getVrfId();
1925             if (vpnRd != null) {
1926                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
1927                         writeOperTx);
1928             }
1929         }
1930     }
1931
1932     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1933         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1934             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1935             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1936     }
1937
1938     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1939         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1940             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1941     }
1942
1943     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
1944                                                     WriteTransaction writeConfigTxn, String vpnName) {
1945         if (vpnInterface == null) {
1946             return;
1947         }
1948         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
1949         if (adjs == null) {
1950             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
1951                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
1952             return;
1953         }
1954         for (Adjacency adj : adjs) {
1955             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1956                 String primaryInterfaceIp = adj.getIpAddress();
1957                 String macAddress = adj.getMacAddress();
1958                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1959
1960                 long label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
1961                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
1962
1963                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
1964                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
1965                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
1966                         routerInt, label, writeConfigTxn);
1967                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
1968                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
1969                         macAddress);
1970                 return;
1971             }
1972         }
1973         LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
1974                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}", interfaceName,
1975                 primaryRd, vpnName);
1976     }
1977
1978     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
1979                                                     WriteTransaction writeConfigTxn, String vpnName) {
1980         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
1981         String rd = vpnUtil.getVpnRd(vpnName);
1982         if (adjs != null) {
1983             List<Adjacency> adjsList = adjs.getAdjacency();
1984             for (Adjacency adj : adjsList) {
1985                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1986                     String primaryInterfaceIp = adj.getIpAddress();
1987                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1988                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
1989                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
1990                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
1991                     return;
1992                 }
1993             }
1994         } else {
1995             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
1996                     vpnInterface.getName(), rd);
1997         }
1998     }
1999
2000     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
2001         if (!canHandleNewVpnInterface(intefaceData.identifier, intefaceData.vpnInterface, vpnName)) {
2002             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
2003                   vpnName, intefaceData.vpnInterface.getName());
2004             return;
2005         }
2006         final VpnInterfaceKey key = intefaceData.identifier
2007                .firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
2008         final String interfaceName = key.getName();
2009         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
2010                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
2011         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
2012                   intefaceData.identifier, vpnName);
2013         return;
2014     }
2015
2016     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
2017                                               VpnInterface vpnInterface, String vpnName) {
2018         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
2019                .get(vpnName);
2020         if (vpnInterfaces == null) {
2021             vpnInterfaces = new ConcurrentLinkedQueue<>();
2022         }
2023         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2024         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2025         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2026                 vpnInterface.getName(), vpnName);
2027     }
2028
2029     public boolean isVpnInstanceReady(String vpnInstanceName) {
2030         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2031         if (vpnRd == null) {
2032             return false;
2033         }
2034         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2035
2036         return vpnInstanceOpDataEntry != null;
2037     }
2038
2039     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2040         synchronized (vpnInstanceName.intern()) {
2041             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2042                     unprocessedVpnInterfaces.get(vpnInstanceName);
2043             if (vpnInterfaces != null) {
2044                 while (!vpnInterfaces.isEmpty()) {
2045                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2046                     if (hasVpnInstanceCreatedSuccessfully) {
2047                         processSavedInterface(savedInterface, vpnInstanceName);
2048                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2049                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2050                     } else {
2051                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2052                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2053                     }
2054                 }
2055             } else {
2056                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2057             }
2058         }
2059     }
2060
2061     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2062             VpnInterface vpnInterface) {
2063         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2064             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2065                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2066             if (vpnInterfaces != null) {
2067                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2068                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2069                             + "unprocessed list", vpnInterface.getName(),
2070                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2071                 }
2072             } else {
2073                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2074                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2075             }
2076         }
2077     }
2078
2079     public void vpnInstanceIsReady(String vpnInstanceName) {
2080         processSavedInterfaces(vpnInstanceName, true);
2081     }
2082
2083     public void vpnInstanceFailed(String vpnInstanceName) {
2084         processSavedInterfaces(vpnInstanceName, false);
2085     }
2086
2087     private static class UnprocessedVpnInterfaceData {
2088         InstanceIdentifier<VpnInterface> identifier;
2089         VpnInterface vpnInterface;
2090
2091         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2092             this.identifier = identifier;
2093             this.vpnInterface = vpnInterface;
2094         }
2095
2096         @Override
2097         public int hashCode() {
2098             final int prime = 31;
2099             int result = 1;
2100             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2101             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2102             return result;
2103         }
2104
2105         @Override
2106         public boolean equals(Object obj) {
2107             if (this == obj) {
2108                 return true;
2109             }
2110             if (obj == null) {
2111                 return false;
2112             }
2113             if (getClass() != obj.getClass()) {
2114                 return false;
2115             }
2116             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2117             if (identifier == null) {
2118                 if (other.identifier != null) {
2119                     return false;
2120                 }
2121             } else if (!identifier.equals(other.identifier)) {
2122                 return false;
2123             }
2124             if (vpnInterface == null) {
2125                 if (other.vpnInterface != null) {
2126                     return false;
2127                 }
2128             } else if (!vpnInterface.equals(other.vpnInterface)) {
2129                 return false;
2130             }
2131             return true;
2132         }
2133     }
2134
2135     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2136         String primaryRd = vpnUtil.getVpnRd(vpnName);
2137         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2138         if (vpnInstanceOpData == null) {
2139             return;
2140         }
2141         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2142         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2143             return;
2144         }
2145         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2146         vpnToDpnLists.forEach(vpnToDpnList -> {
2147             if (vpnToDpnList.getVpnInterfaces() == null) {
2148                 return;
2149             }
2150             vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2151                 try {
2152                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2153                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2154                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2155                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2156                     if (!vpnInterfaceOptional.isPresent()) {
2157                         return;
2158                     }
2159                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2160                             vpnInterface.getInterfaceName());
2161                     if (configVpnAdjacencies == null) {
2162                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2163                         return;
2164                     }
2165                     List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2166                             .augmentation(AdjacenciesOp.class).getAdjacency();
2167                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2168                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2169                     // in operational DS. These unprocessed adjacencies will be handled below.
2170                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2171                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2172                     configVpnAdjacencies.stream()
2173                         .filter(adjacency -> operationVpnAdjacencies.stream()
2174                                 .noneMatch(operationalAdjacency ->
2175                                         operationalAdjacency.getIpAddress().equals(adjacency.getIpAddress())))
2176                         .forEach(adjacency -> {
2177                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2178                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2179                                 () -> {
2180                                     // TODO Deal with sequencing — the config tx must only submitted
2181                                     // if the oper tx goes in
2182                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2183                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2184                                         futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(operTx ->
2185                                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
2186                                                 confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2187                                                     primaryRd, adjacency, vpnInterfaceOptional.get()
2188                                                         .getDpnId(), confTx, operTx)))));
2189                                         return futures;
2190                                     } else {
2191                                         return Collections.emptyList();
2192                                     }
2193                                 });
2194                         });
2195                 } catch (ReadFailedException e) {
2196                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2197                             vpnName, primaryRd);
2198                 }
2199             });
2200         });
2201     }
2202
2203     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2204         private final String interfaceName;
2205         private final boolean add;
2206         private final String txnDestination;
2207
2208         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2209             this.interfaceName = interfaceName;
2210             this.add = add;
2211             this.txnDestination = transactionDest;
2212         }
2213
2214         @Override
2215         public void onSuccess(Void voidObj) {
2216             if (add) {
2217                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2218                         interfaceName, txnDestination);
2219             } else {
2220                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2221             }
2222         }
2223
2224         @Override
2225         public void onFailure(Throwable throwable) {
2226             if (add) {
2227                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2228                         interfaceName, txnDestination, throwable);
2229             } else {
2230                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2231                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2232             }
2233         }
2234     }
2235 }