vpnmanager/impl/src/main/java/org/opendaylight/netvirt/vpnmanager/VpnInterfaceManager.java

   1 /*
   2  * Copyright (c) 2016 - 2017 Ericsson India Global Services Pvt Ltd. and others.  All rights reserved.
   3  *
   4  * This program and the accompanying materials are made available under the
   5  * terms of the Eclipse Public License v1.0 which accompanies this distribution,
   6  * and is available at http://www.eclipse.org/legal/epl-v10.html
   7  */
   8 package org.opendaylight.netvirt.vpnmanager;
   9
  10 import com.google.common.base.Optional;
  11 import com.google.common.base.Preconditions;
  12 import com.google.common.collect.Iterators;
  13 import com.google.common.util.concurrent.FutureCallback;
  14 import com.google.common.util.concurrent.Futures;
  15 import com.google.common.util.concurrent.ListenableFuture;
  16 import com.google.common.util.concurrent.MoreExecutors;
  17 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
  18 import java.math.BigInteger;
  19 import java.util.ArrayList;
  20 import java.util.Collections;
  21 import java.util.Iterator;
  22 import java.util.List;
  23 import java.util.Map;
  24 import java.util.Objects;
  25 import java.util.concurrent.ConcurrentHashMap;
  26 import java.util.concurrent.ConcurrentLinkedQueue;
  27 import java.util.function.Consumer;
  28 import java.util.function.Predicate;
  29 import java.util.stream.Collectors;
  30 import javax.annotation.PostConstruct;
  31 import javax.annotation.PreDestroy;
  32 import javax.inject.Inject;
  33 import javax.inject.Singleton;
  34 import org.opendaylight.controller.md.sal.binding.api.DataBroker;
  35 import org.opendaylight.controller.md.sal.binding.api.WriteTransaction;
  36 import org.opendaylight.controller.md.sal.common.api.data.LogicalDatastoreType;
  37 import org.opendaylight.controller.md.sal.common.api.data.ReadFailedException;
  38 import org.opendaylight.controller.md.sal.common.api.data.TransactionCommitFailedException;
  39 import org.opendaylight.genius.datastoreutils.AsyncDataTreeChangeListenerBase;
  40 import org.opendaylight.genius.datastoreutils.SingleTransactionDataBroker;
  41 import org.opendaylight.genius.infra.ManagedNewTransactionRunner;
  42 import org.opendaylight.genius.infra.ManagedNewTransactionRunnerImpl;
  43 import org.opendaylight.genius.interfacemanager.interfaces.IInterfaceManager;
  44 import org.opendaylight.genius.mdsalutil.NWUtil;
  45 import org.opendaylight.genius.mdsalutil.NwConstants;
  46 import org.opendaylight.genius.mdsalutil.cache.InstanceIdDataObjectCache;
  47 import org.opendaylight.genius.mdsalutil.interfaces.IMdsalApiManager;
  48 import org.opendaylight.infrautils.caches.CacheProvider;
  49 import org.opendaylight.infrautils.jobcoordinator.JobCoordinator;
  50 import org.opendaylight.infrautils.utils.concurrent.ListenableFutures;
  51 import org.opendaylight.netvirt.bgpmanager.api.IBgpManager;
  52 import org.opendaylight.netvirt.fibmanager.api.FibHelper;
  53 import org.opendaylight.netvirt.fibmanager.api.IFibManager;
  54 import org.opendaylight.netvirt.fibmanager.api.RouteOrigin;
  55 import org.opendaylight.netvirt.vpnmanager.api.IVpnManager;
  56 import org.opendaylight.netvirt.vpnmanager.api.InterfaceUtils;
  57 import org.opendaylight.netvirt.vpnmanager.api.VpnHelper;
  58 import org.opendaylight.netvirt.vpnmanager.arp.responder.ArpResponderHandler;
  59 import org.opendaylight.netvirt.vpnmanager.populator.input.L3vpnInput;
  60 import org.opendaylight.netvirt.vpnmanager.populator.intfc.VpnPopulator;
  61 import org.opendaylight.netvirt.vpnmanager.populator.registry.L3vpnRegistry;
  62 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.VpnInterfaces;
  63 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterface;
  64 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.VpnInterfaceKey;
  65 import org.opendaylight.yang.gen.v1.urn.huawei.params.xml.ns.yang.l3vpn.rev140815.vpn.interfaces.vpn._interface.VpnInstanceNames;
  66 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.interfaces.rev140508.interfaces.state.Interface;
  67 import org.opendaylight.yang.gen.v1.urn.ietf.params.xml.ns.yang.ietf.yang.types.rev130715.Uuid;
  68 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.idmanager.rev160406.IdManagerService;
  69 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.interfacemanager.rpcs.rev160406.OdlInterfaceRpcService;
  70 import org.opendaylight.yang.gen.v1.urn.opendaylight.genius.itm.op.rev160406.tunnels_state.StateTunnelList;
  71 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.FibEntries;
  72 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.LabelRouteMap;
  73 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterface;
  74 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.RouterInterfaceBuilder;
  75 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute;
  76 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.VrfEntryBase;
  77 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTables;
  78 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesBuilder;
  79 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.fibentries.VrfTablesKey;
  80 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfo;
  81 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoBuilder;
  82 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.label.route.map.LabelRouteInfoKey;
  83 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.vrfentries.VrfEntry;
  84 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.Adjacencies;
  85 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.AdjacenciesOp;
  86 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.NeutronRouterDpns;
  87 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.VpnInstanceOpData;
  88 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency;
  89 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.Adjacency.AdjacencyType;
  90 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.adjacency.list.AdjacencyBuilder;
  91 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.learnt.vpn.vip.to.port.data.LearntVpnVipToPort;
  92 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnList;
  93 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.RouterDpnListKey;
  94 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesList;
  95 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.neutron.router.dpns.router.dpn.list.DpnVpninterfacesListKey;
  96 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.prefix.to._interface.vpn.ids.Prefixes;
  97 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntry;
  98 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryBuilder;
  99 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn._interface.op.data.VpnInterfaceOpDataEntryKey;
 100 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.VpnInstanceOpDataEntry;
 101 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.VpnToDpnList;
 102 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.l3vpn.rev130911.vpn.instance.op.data.vpn.instance.op.data.entry.vpntargets.VpnTarget;
 103 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.ext.routers.Routers;
 104 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.natservice.rev160111.external.subnets.Subnets;
 105 import org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.neutronvpn.rev150602.neutron.vpn.portip.port.data.VpnPortipToPort;
 106 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier;
 107 import org.opendaylight.yangtools.yang.binding.InstanceIdentifier.InstanceIdentifierBuilder;
 108 import org.slf4j.Logger;
 109 import org.slf4j.LoggerFactory;
 110
 111 @Singleton
 112 public class VpnInterfaceManager extends AsyncDataTreeChangeListenerBase<VpnInterface, VpnInterfaceManager> {
 113
 114     private static final Logger LOG = LoggerFactory.getLogger(VpnInterfaceManager.class);
 115     private static final short DJC_MAX_RETRIES = 3;
 116
 117     private final DataBroker dataBroker;
 118     private final ManagedNewTransactionRunner txRunner;
 119     private final IBgpManager bgpManager;
 120     private final IFibManager fibManager;
 121     private final IMdsalApiManager mdsalManager;
 122     private final IdManagerService idManager;
 123     private final OdlInterfaceRpcService ifaceMgrRpcService;
 124     private final VpnFootprintService vpnFootprintService;
 125     private final IInterfaceManager interfaceManager;
 126     private final IVpnManager vpnManager;
 127     private final ArpResponderHandler arpResponderHandler;
 128     private final JobCoordinator jobCoordinator;
 129     private final VpnUtil vpnUtil;
 130
 131     private final ConcurrentHashMap<String, Runnable> vpnIntfMap = new ConcurrentHashMap<>();
 132
 133     private final Map<String, ConcurrentLinkedQueue<UnprocessedVpnInterfaceData>> unprocessedVpnInterfaces =
 134             new ConcurrentHashMap<>();
 135
 136     private final InstanceIdDataObjectCache<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryCache;
 137
 138     @Inject
 139     public VpnInterfaceManager(final DataBroker dataBroker,
 140                                final IBgpManager bgpManager,
 141                                final IdManagerService idManager,
 142                                final IMdsalApiManager mdsalManager,
 143                                final IFibManager fibManager,
 144                                final OdlInterfaceRpcService ifaceMgrRpcService,
 145                                final VpnFootprintService vpnFootprintService,
 146                                final IInterfaceManager interfaceManager,
 147                                final IVpnManager vpnManager,
 148                                final ArpResponderHandler arpResponderHandler,
 149                                final JobCoordinator jobCoordinator,
 150                                final CacheProvider cacheProvider,
 151                                final VpnUtil vpnUtil) {
 152         super(VpnInterface.class, VpnInterfaceManager.class);
 153
 154         this.dataBroker = dataBroker;
 155         this.txRunner = new ManagedNewTransactionRunnerImpl(dataBroker);
 156         this.bgpManager = bgpManager;
 157         this.idManager = idManager;
 158         this.mdsalManager = mdsalManager;
 159         this.fibManager = fibManager;
 160         this.ifaceMgrRpcService = ifaceMgrRpcService;
 161         this.vpnFootprintService = vpnFootprintService;
 162         this.interfaceManager = interfaceManager;
 163         this.vpnManager = vpnManager;
 164         this.arpResponderHandler = arpResponderHandler;
 165         this.jobCoordinator = jobCoordinator;
 166         this.vpnUtil = vpnUtil;
 167
 168         vpnInstanceOpDataEntryCache = new InstanceIdDataObjectCache<>(VpnInstanceOpDataEntry.class, dataBroker,
 169                 LogicalDatastoreType.OPERATIONAL, InstanceIdentifier.builder(
 170                         VpnInstanceOpData.class).child(VpnInstanceOpDataEntry.class).build(), cacheProvider);
 171     }
 172
 173     public Runnable isNotifyTaskQueued(String intfName) {
 174         return vpnIntfMap.remove(intfName);
 175     }
 176
 177     @PostConstruct
 178     public void start() {
 179         LOG.info("{} start", getClass().getSimpleName());
 180         registerListener(LogicalDatastoreType.CONFIGURATION, dataBroker);
 181     }
 182
 183     @Override
 184     @PreDestroy
 185     public void close() {
 186         super.close();
 187         vpnInstanceOpDataEntryCache.close();
 188     }
 189
 190     @Override
 191     protected InstanceIdentifier<VpnInterface> getWildCardPath() {
 192         return InstanceIdentifier.create(VpnInterfaces.class).child(VpnInterface.class);
 193     }
 194
 195     @Override
 196     protected VpnInterfaceManager getDataTreeChangeListener() {
 197         return VpnInterfaceManager.this;
 198     }
 199
 200     @Override
 201     public void add(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface) {
 202         LOG.info("add: intfName {} onto vpnName {}",
 203                  vpnInterface.getName(),
 204                  VpnHelper.getVpnInterfaceVpnInstanceNamesString(vpnInterface.getVpnInstanceNames()));
 205         addVpnInterface(identifier, vpnInterface, null, null);
 206     }
 207
 208     private boolean canHandleNewVpnInterface(final InstanceIdentifier<VpnInterface> identifier,
 209                           final VpnInterface vpnInterface, String vpnName) {
 210         synchronized (vpnName.intern()) {
 211             if (isVpnInstanceReady(vpnName)) {
 212                 return true;
 213             }
 214             addToUnprocessedVpnInterfaces(identifier, vpnInterface, vpnName);
 215             return false;
 216         }
 217     }
 218
 219     // TODO Clean up the exception handling
 220     @SuppressWarnings("checkstyle:IllegalCatch")
 221     private void addVpnInterface(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 222                              final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs) {
 223         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
 224             String vpnName = vpnInterfaceVpnInstance.getVpnName();
 225             addVpnInterfaceCall(identifier, vpnInterface, oldAdjs, newAdjs, vpnName);
 226         }
 227     }
 228
 229     private void addVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface vpnInterface,
 230                          final List<Adjacency> oldAdjs, final List<Adjacency> newAdjs, String vpnName) {
 231         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 232         final String interfaceName = key.getName();
 233
 234         if (!canHandleNewVpnInterface(identifier, vpnInterface, vpnName)) {
 235             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
 236                   vpnName, vpnInterface.getName());
 237             return;
 238         }
 239         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
 240              .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 241         List<Adjacency> copyOldAdjs = null;
 242         if (oldAdjs != null) {
 243             copyOldAdjs = new ArrayList<>();
 244             copyOldAdjs.addAll(oldAdjs);
 245         }
 246         List<Adjacency> copyNewAdjs = null;
 247         if (newAdjs != null) {
 248             copyNewAdjs = new ArrayList<>();
 249             copyNewAdjs.addAll(newAdjs);
 250         }
 251         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, vpnInterface, copyOldAdjs, copyNewAdjs, identifier, vpnName);
 252     }
 253
 254     private void addVpnInterfaceToVpn(final InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier,
 255                     final VpnInterface vpnInterface, final List<Adjacency> oldAdjs,
 256                     final List<Adjacency> newAdjs,
 257                     final InstanceIdentifier<VpnInterface> identifier, String vpnName) {
 258         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
 259         final String interfaceName = key.getName();
 260         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 261         if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
 262             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker, interfaceName);
 263             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 264             if (interfaceState != null) {
 265                 try {
 266                     final BigInteger dpnId = InterfaceUtils.getDpIdFromInterface(interfaceState);
 267                     final int ifIndex = interfaceState.getIfIndex();
 268                     jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName, () -> {
 269                         // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
 270                         // (the inventory tx goes in last)
 271                         List<ListenableFuture<Void>> futures = new ArrayList<>();
 272                         ListenableFuture<Void> confFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 273                             confTx -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
 274                                 operTx -> futures.add(
 275                                         txRunner.callWithNewWriteOnlyTransactionAndSubmit(invTx -> {
 276                                             LOG.info(
 277                                                     "addVpnInterface: VPN Interface add event - intfName {} vpnName {}"
 278                                                     + " on dpn {}",
 279                                                     vpnInterface.getName(), vpnName, vpnInterface.getDpnId());
 280                                             processVpnInterfaceUp(dpnId, vpnInterface, primaryRd, ifIndex, false,
 281                                                     confTx, operTx, invTx, interfaceState, vpnName);
 282                                             if (oldAdjs != null && !oldAdjs.equals(newAdjs)) {
 283                                                 LOG.info("addVpnInterface: Adjacency changed upon VPNInterface {}"
 284                                                         + " Update for swapping VPN {} case.", interfaceName, vpnName);
 285                                                 if (newAdjs != null) {
 286                                                     for (Adjacency adj : newAdjs) {
 287                                                         if (oldAdjs.contains(adj)) {
 288                                                             oldAdjs.remove(adj);
 289                                                         } else {
 290                                                             if (!isBgpVpnInternetVpn
 291                                                                     || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 292                                                                 addNewAdjToVpnInterface(vpnInterfaceOpIdentifier,
 293                                                                         primaryRd, adj, dpnId, operTx, confTx);
 294                                                             }
 295                                                         }
 296                                                     }
 297                                                 }
 298                                                 for (Adjacency adj : oldAdjs) {
 299                                                     if (!isBgpVpnInternetVpn
 300                                                             || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
 301                                                         delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
 302                                                                 operTx, confTx);
 303                                                     }
 304                                                 }
 305                                             }
 306                                         })))));
 307                         futures.add(confFuture);
 308                         Futures.addCallback(confFuture, new PostVpnInterfaceWorker(interfaceName, true, "Config"),
 309                                 MoreExecutors.directExecutor());
 310                         LOG.info("addVpnInterface: Addition of interface {} in VPN {} on dpn {}"
 311                                 + " processed successfully", interfaceName, vpnName, dpnId);
 312                         return futures;
 313                     });
 314                 } catch (NumberFormatException | IllegalStateException e) {
 315                     LOG.error("addVpnInterface: Unable to retrieve dpnId from interface operational data store for "
 316                                     + "interface {}. Interface addition on vpn {} failed", interfaceName,
 317                                     vpnName, e);
 318                     return;
 319                 }
 320             } else if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
 321                 jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(),
 322                     () -> {
 323                         ListenableFuture<Void> future = txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
 324                             createFibEntryForRouterInterface(primaryRd, vpnInterface, interfaceName,
 325                                     confTx, vpnName);
 326                             LOG.info("addVpnInterface: Router interface {} for vpn {} on dpn {}", interfaceName,
 327                                     vpnName, vpnInterface.getDpnId());
 328                         });
 329                         ListenableFutures.addErrorLogging(future, LOG,
 330                                 "Error creating FIB entry for interface {} on VPN {}", vpnInterface.getName(), vpnName);
 331                         return Collections.singletonList(future);
 332                     });
 333             } else {
 334                 LOG.info("addVpnInterface: Handling addition of VPN interface {} on vpn {} skipped as interfaceState"
 335                         + " is not available", interfaceName, vpnName);
 336             }
 337         } else {
 338             LOG.error("addVpnInterface: Handling addition of VPN interface {} on vpn {} dpn {} skipped"
 339                       + " as vpn is pending delete", interfaceName, vpnName,
 340                     vpnInterface.getDpnId());
 341         }
 342     }
 343
 344     // "Unconditional wait" and "Wait not in loop" wrt the VpnNotifyTask below - suppressing the FB violation -
 345     // see comments below.
 346     @SuppressFBWarnings({"UW_UNCOND_WAIT", "WA_NOT_IN_LOOP"})
 347     protected void processVpnInterfaceUp(final BigInteger dpId, VpnInterface vpnInterface, final String primaryRd,
 348             final int lportTag, boolean isInterfaceUp,
 349             WriteTransaction writeConfigTxn,
 350             WriteTransaction writeOperTxn,
 351             WriteTransaction writeInvTxn,
 352             Interface interfaceState,
 353             final String vpnName) {
 354         final String interfaceName = vpnInterface.getName();
 355         Optional<VpnInterfaceOpDataEntry> optOpVpnInterface = vpnUtil.getVpnInterfaceOpDataEntry(interfaceName,
 356                 vpnName);
 357         VpnInterfaceOpDataEntry opVpnInterface = optOpVpnInterface.isPresent() ? optOpVpnInterface.get() : null;
 358         boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
 359         if (!isInterfaceUp) {
 360             LOG.info("processVpnInterfaceUp: Binding vpn service to interface {} onto dpn {} for vpn {}",
 361                      interfaceName, dpId, vpnName);
 362             long vpnId = vpnUtil.getVpnId(vpnName);
 363             if (vpnId == VpnConstants.INVALID_ID) {
 364                 LOG.warn("processVpnInterfaceUp: VpnInstance to VPNId mapping not available for VpnName {}"
 365                         + " processing vpninterface {} on dpn {}, bailing out now.", vpnName, interfaceName,
 366                         dpId);
 367                 return;
 368             }
 369
 370             boolean waitForVpnInterfaceOpRemoval = false;
 371             if (opVpnInterface != null) {
 372                 String opVpnName = opVpnInterface.getVpnInstanceName();
 373                 String primaryInterfaceIp = null;
 374                 if (opVpnName.equals(vpnName)) {
 375                     // Please check if the primary VRF Entry does not exist for VPNInterface
 376                     // If so, we have to process ADD, as this might be a DPN Restart with Remove and Add triggered
 377                     // back to back
 378                     // However, if the primary VRF Entry for this VPNInterface exists, please continue bailing out !
 379                     List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
 380                     if (adjs == null) {
 381                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} failed as adjacencies"
 382                                 + " for this vpn interface could not be obtained", interfaceName, dpId,
 383                                 vpnName);
 384                         return;
 385                     }
 386                     for (Adjacency adj : adjs) {
 387                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 388                             primaryInterfaceIp = adj.getIpAddress();
 389                             break;
 390                         }
 391                     }
 392                     if (primaryInterfaceIp == null) {
 393                         LOG.error("processVpnInterfaceUp: VPN Interface {} addition on dpn {} for vpn {} failed"
 394                                 + " as primary adjacency for this vpn interface could not be obtained", interfaceName,
 395                                 dpId, vpnName);
 396                         return;
 397                     }
 398                     // Get the rd of the vpn instance
 399                     VrfEntry vrf = vpnUtil.getVrfEntry(primaryRd, primaryInterfaceIp);
 400                     if (vrf != null) {
 401                         LOG.error("processVpnInterfaceUp: VPN Interface {} on dpn {} for vpn {} already provisioned ,"
 402                                 + " bailing out from here.", interfaceName, dpId, vpnName);
 403                         return;
 404                     }
 405                     waitForVpnInterfaceOpRemoval = true;
 406                 } else {
 407                     LOG.error("processVpnInterfaceUp: vpn interface {} to go to configured vpn {} on dpn {},"
 408                             + " but in operational vpn {}", interfaceName, vpnName, dpId, opVpnName);
 409                 }
 410             }
 411             if (!waitForVpnInterfaceOpRemoval) {
 412                 // Add the VPNInterface and quit
 413                 vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 414                         null/*ipAddressSourceValuePair*/,
 415                         true /* add */);
 416                 processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 417                         vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 418                 if (!isBgpVpnInternetVpn) {
 419                     vpnUtil.bindService(vpnName, interfaceName, false /*isTunnelInterface*/);
 420                 }
 421                 LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {}", interfaceName,
 422                         dpId, vpnName);
 423                 if (interfaceManager.isExternalInterface(interfaceName)) {
 424                     processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
 425                         NwConstants.ADD_FLOW);
 426                 }
 427                 return;
 428             }
 429
 430             // FIB didn't get a chance yet to clean up this VPNInterface
 431             // Let us give it a chance here !
 432             LOG.info("processVpnInterfaceUp: Trying to add VPN Interface {} on dpn {} for vpn {},"
 433                     + " but waiting for FIB to clean up! ", interfaceName, dpId, vpnName);
 434             try {
 435                 Runnable notifyTask = new VpnNotifyTask();
 436                 synchronized (notifyTask) {
 437                     // Per FB's "Unconditional wait" violation, the code should really verify that the condition it
 438                     // intends to wait for is not already satisfied before calling wait. However the VpnNotifyTask is
 439                     // published here while holding the lock on it so this path will hit the wait before notify can be
 440                     // invoked.
 441                     vpnIntfMap.put(interfaceName, notifyTask);
 442                     try {
 443                         notifyTask.wait(VpnConstants.MAX_WAIT_TIME_IN_MILLISECONDS);
 444                     } catch (InterruptedException e) {
 445                         // Ignored
 446                     }
 447                 }
 448             } finally {
 449                 vpnIntfMap.remove(interfaceName);
 450             }
 451
 452             if (opVpnInterface != null) {
 453                 LOG.warn("processVpnInterfaceUp: VPN Interface {} removal on dpn {} for vpn {}"
 454                         + " by FIB did not complete on time," + " bailing addition ...", interfaceName,
 455                         dpId, vpnName);
 456                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
 457                 return;
 458             }
 459             // VPNInterface got removed, proceed with Add
 460             LOG.info("processVpnInterfaceUp: Continuing to plumb vpn interface {} onto dpn {} for vpn {}",
 461                     interfaceName, dpId, vpnName);
 462             vpnFootprintService.updateVpnToDpnMapping(dpId, vpnName, primaryRd, interfaceName,
 463                     null/*ipAddressSourceValuePair*/,
 464                     true /* add */);
 465             processVpnInterfaceAdjacencies(dpId, lportTag, vpnName, primaryRd, interfaceName,
 466                     vpnId, writeConfigTxn, writeOperTxn, writeInvTxn, interfaceState);
 467             if (!isBgpVpnInternetVpn) {
 468                 vpnUtil.bindService(vpnName, interfaceName, false/*isTunnelInterface*/);
 469             }
 470             LOG.info("processVpnInterfaceUp: Plumbed vpn interface {} onto dpn {} for vpn {} after waiting for"
 471                     + " FIB to clean up", interfaceName, dpId, vpnName);
 472             if (interfaceManager.isExternalInterface(interfaceName)) {
 473                 processExternalVpnInterface(interfaceName, vpnName, dpId,
 474                                lportTag, NwConstants.ADD_FLOW);
 475             }
 476         } else {
 477             try {
 478                 // Interface is retained in the DPN, but its Link Up.
 479                 // Advertise prefixes again for this interface to BGP
 480                 InstanceIdentifier<VpnInterface> identifier =
 481                         VpnUtil.getVpnInterfaceIdentifier(vpnInterface.getName());
 482                 InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
 483                         VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 484                 advertiseAdjacenciesForVpnToBgp(primaryRd, dpId, vpnInterfaceOpIdentifier, vpnName, interfaceName);
 485                 // Perform similar operation as interface add event for extraroutes.
 486                 InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 487                 Optional<Adjacencies> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 488                         LogicalDatastoreType.CONFIGURATION, path);
 489                 if (!optAdjacencies.isPresent()) {
 490                     LOG.trace("No config adjacencies present for vpninterface {}", vpnInterface);
 491                     return;
 492                 }
 493                 List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
 494                 for (Adjacency adjacency : adjacencies) {
 495                     if (adjacency.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 496                         continue;
 497                     }
 498                     // if BGPVPN Internet, filter only IPv6 Adjacencies
 499                     if (isBgpVpnInternetVpn && !vpnUtil.isAdjacencyEligibleToVpnInternet(adjacency)) {
 500                         continue;
 501                     }
 502                     addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adjacency,
 503                             dpId, writeOperTxn, writeConfigTxn);
 504                 }
 505             } catch (ReadFailedException e) {
 506                 LOG.error("processVpnInterfaceUp: Failed to read data store for interface {} vpn {} rd {} dpn {}",
 507                         interfaceName, vpnName, primaryRd, dpId);
 508             }
 509         }
 510     }
 511
 512     private void processExternalVpnInterface(String interfaceName, String vpnName, BigInteger dpId,
 513         int lportTag, int addOrRemove) {
 514         Uuid extNetworkId;
 515         try {
 516             // vpn instance of ext-net interface is the network-id
 517             extNetworkId = new Uuid(vpnName);
 518         } catch (IllegalArgumentException e) {
 519             LOG.error("processExternalVpnInterface: VPN instance {} is not Uuid. Processing external vpn interface {}"
 520                     + " on dpn {} failed", vpnName, interfaceName, dpId);
 521             return;
 522         }
 523
 524         List<Uuid> routerIds = vpnUtil.getExternalNetworkRouterIds(extNetworkId);
 525         if (routerIds == null || routerIds.isEmpty()) {
 526             LOG.info("processExternalVpnInterface: No router is associated with {}."
 527                     + " Bailing out of processing external vpn interface {} on dpn {} for vpn {}",
 528                     extNetworkId.getValue(), interfaceName, dpId, vpnName);
 529             return;
 530         }
 531
 532         LOG.info("processExternalVpnInterface: Router-ids {} associated with exernal vpn-interface {} on dpn {}"
 533                 + " for vpn {}", routerIds, interfaceName, dpId, vpnName);
 534         for (Uuid routerId : routerIds) {
 535             String routerName = routerId.getValue();
 536             BigInteger primarySwitch = vpnUtil.getPrimarySwitchForRouter(routerName);
 537             if (Objects.equals(primarySwitch, dpId)) {
 538                 Routers router = vpnUtil.getExternalRouter(routerName);
 539                 if (router != null) {
 540                     if (addOrRemove == NwConstants.ADD_FLOW) {
 541                         vpnManager.addArpResponderFlowsToExternalNetworkIps(routerName,
 542                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()), router.getExtGwMacAddress(),
 543                                 dpId, interfaceName, lportTag);
 544                     } else {
 545                         vpnManager.removeArpResponderFlowsToExternalNetworkIps(routerName,
 546                                 VpnUtil.getIpsListFromExternalIps(router.getExternalIps()),
 547                                 dpId, interfaceName, lportTag);
 548                     }
 549                 } else {
 550                     LOG.error("processExternalVpnInterface: No external-router found for router-id {}. Bailing out of"
 551                             + " processing external vpn-interface {} on dpn {} for vpn {}", routerName,
 552                             interfaceName, dpId, vpnName);
 553                 }
 554             }
 555         }
 556     }
 557
 558     // TODO Clean up the exception handling
 559     @SuppressWarnings("checkstyle:IllegalCatch")
 560     private void advertiseAdjacenciesForVpnToBgp(final String rd, BigInteger dpnId,
 561                                                  final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 562                                                   String vpnName, String interfaceName) {
 563         if (rd == null) {
 564             LOG.error("advertiseAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} on dpn {} in vpn {}",
 565                   interfaceName, dpnId, vpnName);
 566             return;
 567         } else {
 568             if (rd.equals(vpnName)) {
 569                 LOG.info("advertiseAdjacenciesForVpnFromBgp: Ignoring BGP advertisement for interface {} on dpn {}"
 570                         + " as it is in internal vpn{} with rd {}", interfaceName, dpnId, vpnName, rd);
 571                 return;
 572             }
 573         }
 574         LOG.info("advertiseAdjacenciesForVpnToBgp: Advertising interface {} on dpn {} in vpn {} with rd {} ",
 575                 interfaceName, dpnId, vpnName, rd);
 576
 577         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 578         if (nextHopIp == null) {
 579             LOG.error("advertiseAdjacenciesForVpnToBgp: NextHop for interface {} on dpn {} is null,"
 580                     + " returning from advertising route with rd {} vpn {} to bgp", interfaceName, dpnId,
 581                     rd, vpnName);
 582             return;
 583         }
 584
 585         try {
 586             //Read NextHops
 587             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 588             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
 589                     LogicalDatastoreType.OPERATIONAL, path);
 590             if (adjacencies.isPresent()) {
 591                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 592                 if (!nextHops.isEmpty()) {
 593                     LOG.debug("advertiseAdjacenciesForVpnToBgp:  NextHops are {} for interface {} on dpn {} for vpn {}"
 594                             + " rd {}", nextHops, interfaceName, dpnId, vpnName, rd);
 595                     VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(rd);
 596                     long l3vni = vpnInstanceOpData.getL3vni();
 597                     VrfEntry.EncapType encapType = VpnUtil.isL3VpnOverVxLan(l3vni)
 598                             ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 599                     for (Adjacency nextHop : nextHops) {
 600                         if (nextHop.getAdjacencyType() == AdjacencyType.ExtraRoute) {
 601                             continue;
 602                         }
 603                         String gatewayMac = null;
 604                         long label = 0;
 605                         if (VpnUtil.isL3VpnOverVxLan(l3vni)) {
 606                             final VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(
 607                                     vpnInstanceOpData.getVpnInstanceName(), nextHop.getIpAddress());
 608                             gatewayMac = arpResponderHandler.getGatewayMacAddressForInterface(gwPort, interfaceName)
 609                                     .get();
 610                         } else {
 611                             label = nextHop.getLabel();
 612                         }
 613                         try {
 614                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Adding Fib Entry rd {} prefix {}"
 615                                     + " nexthop {} label {}", rd, nextHop.getIpAddress(), nextHopIp, label);
 616                             bgpManager.advertisePrefix(rd, nextHop.getMacAddress(), nextHop.getIpAddress(), nextHopIp,
 617                                     encapType, (int)label, l3vni, 0 /*l2vni*/,
 618                                     gatewayMac);
 619                             LOG.info("VPN ADVERTISE: advertiseAdjacenciesForVpnToBgp: Added Fib Entry rd {} prefix {}"
 620                                             + " nexthop {} label {} for interface {} on dpn {} for vpn {}", rd,
 621                                     nextHop.getIpAddress(), nextHopIp, label, interfaceName, dpnId, vpnName);
 622                         } catch (Exception e) {
 623                             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to advertise prefix {} in vpn {}"
 624                                     + " with rd {} for interface {} on dpn {}", nextHop.getIpAddress(), vpnName, rd,
 625                                     interfaceName, dpnId, e);
 626                         }
 627                     }
 628                 }
 629             }
 630         } catch (ReadFailedException e) {
 631             LOG.error("advertiseAdjacenciesForVpnToBgp: Failed to read data store for interface {} dpn {} nexthop {}"
 632                     + "vpn {} rd {}", interfaceName, dpnId, nextHopIp, vpnName, rd);
 633         }
 634     }
 635
 636     // TODO Clean up the exception handling
 637     @SuppressWarnings("checkstyle:IllegalCatch")
 638     private void withdrawAdjacenciesForVpnFromBgp(final InstanceIdentifier<VpnInterfaceOpDataEntry> identifier,
 639                         String vpnName, String interfaceName, WriteTransaction writeConfigTxn,
 640                         WriteTransaction writeOperTx) {
 641         //Read NextHops
 642         InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
 643         String rd = vpnUtil.getVpnRd(interfaceName);
 644         if (rd == null) {
 645             LOG.error("withdrawAdjacenciesForVpnFromBgp: Unable to recover rd for interface {} in vpn {}",
 646                 interfaceName, vpnName);
 647             return;
 648         } else {
 649             if (rd.equals(vpnName)) {
 650                 LOG.info(
 651                         "withdrawAdjacenciesForVpnFromBgp: Ignoring BGP withdrawal for interface {} as it is in "
 652                                 + "internal vpn{} with rd {}", interfaceName, vpnName, rd);
 653                 return;
 654             }
 655         }
 656         LOG.info("withdrawAdjacenciesForVpnFromBgp: For interface {} in vpn {} with rd {}", interfaceName,
 657                vpnName, rd);
 658         Optional<AdjacenciesOp> adjacencies = Optional.absent();
 659         try {
 660             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL,
 661                     path);
 662         } catch (ReadFailedException e) {
 663             LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to read data store for interface {} vpn {}",
 664                     interfaceName, vpnName);
 665         }
 666         if (adjacencies.isPresent()) {
 667             List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 668
 669             if (!nextHops.isEmpty()) {
 670                 LOG.trace("withdrawAdjacenciesForVpnFromBgp: NextHops are {} for interface {} in vpn {} rd {}",
 671                         nextHops, interfaceName, vpnName, rd);
 672                 for (Adjacency nextHop : nextHops) {
 673                     try {
 674                         if (nextHop.getAdjacencyType() != AdjacencyType.ExtraRoute) {
 675                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removing Fib Entry rd {}"
 676                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 677                                     interfaceName, vpnName);
 678                             bgpManager.withdrawPrefix(rd, nextHop.getIpAddress());
 679                             LOG.info("VPN WITHDRAW: withdrawAdjacenciesForVpnFromBgp: Removed Fib Entry rd {}"
 680                                     + " prefix {} for interface {} in vpn {}", rd, nextHop.getIpAddress(),
 681                                     interfaceName, vpnName);
 682                         } else {
 683                             // Perform similar operation as interface delete event for extraroutes.
 684                             String allocatedRd = nextHop.getVrfId();
 685                             for (String nh : nextHop.getNextHopIpList()) {
 686                                 deleteExtraRouteFromCurrentAndImportingVpns(
 687                                     vpnName, nextHop.getIpAddress(), nh, allocatedRd, interfaceName, writeConfigTxn,
 688                                     writeOperTx);
 689                             }
 690                         }
 691                     } catch (Exception e) {
 692                         LOG.error("withdrawAdjacenciesForVpnFromBgp: Failed to withdraw prefix {} in vpn {} with rd {}"
 693                                 + " for interface {} ", nextHop.getIpAddress(), vpnName, rd, interfaceName, e);
 694                     }
 695                 }
 696             }
 697         }
 698     }
 699
 700     @SuppressWarnings("checkstyle:IllegalCatch")
 701     protected void processVpnInterfaceAdjacencies(BigInteger dpnId, final int lportTag, String vpnName,
 702                                                   String primaryRd, String interfaceName, final long vpnId,
 703                                                   WriteTransaction writeConfigTxn,
 704                                                   WriteTransaction writeOperTxn,
 705                                                   final WriteTransaction writeInvTxn,
 706                                                   Interface interfaceState) {
 707         InstanceIdentifier<VpnInterface> identifier = VpnUtil.getVpnInterfaceIdentifier(interfaceName);
 708         // Read NextHops
 709         InstanceIdentifier<Adjacencies> path = identifier.augmentation(Adjacencies.class);
 710         Optional<Adjacencies> adjacencies = Optional.absent();
 711         try {
 712             adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker, LogicalDatastoreType.CONFIGURATION,
 713                     path);
 714         } catch (ReadFailedException e) {
 715             LOG.error("processVpnInterfaceAdjacencies: Failed to read data store for interface {} vpn {} rd {}"
 716                     + "dpn {}", interfaceName, vpnName, primaryRd, dpnId);
 717         }
 718         if (!adjacencies.isPresent()) {
 719             addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, null/*adjacencies*/, lportTag,
 720                     null/*gwMac*/, writeOperTxn);
 721             return;
 722         }
 723         // Get the rd of the vpn instance
 724         String nextHopIp = null;
 725         try {
 726             nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
 727         } catch (Exception e) {
 728             LOG.error("processVpnInterfaceAdjacencies: Unable to retrieve endpoint ip address for "
 729                     + "dpnId {} for vpnInterface {} vpnName {}", dpnId, interfaceName, vpnName);
 730         }
 731         List<String> nhList = new ArrayList<>();
 732         if (nextHopIp != null) {
 733             nhList.add(nextHopIp);
 734             LOG.debug("processVpnInterfaceAdjacencies: NextHop for interface {} on dpn {} in vpn {} is {}",
 735                     interfaceName, dpnId, vpnName, nhList);
 736         }
 737         Optional<String> gwMac = Optional.absent();
 738         String vpnInterfaceSubnetGwMacAddress = null;
 739         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
 740         Long l3vni = vpnInstanceOpData.getL3vni();
 741         boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(l3vni);
 742         VrfEntry.EncapType encapType = isL3VpnOverVxLan ? VrfEntry.EncapType.Vxlan : VrfEntry.EncapType.Mplsgre;
 743         VpnPopulator registeredPopulator = L3vpnRegistry.getRegisteredPopulator(encapType);
 744         List<Adjacency> nextHops = adjacencies.get().getAdjacency();
 745         List<Adjacency> value = new ArrayList<>();
 746         for (Adjacency nextHop : nextHops) {
 747             String rd = primaryRd;
 748             String nexthopIpValue = nextHop.getIpAddress().split("/")[0];
 749             if (vpnInstanceOpData.getBgpvpnType() == VpnInstanceOpDataEntry.BgpvpnType.BGPVPNInternet
 750                     && NWUtil.isIpv4Address(nexthopIpValue)) {
 751                 String prefix = nextHop.getIpAddress() == null ?  "null" :
 752                       VpnUtil.getIpPrefix(nextHop.getIpAddress());
 753                 LOG.debug("processVpnInterfaceAdjacencies: UnsupportedOperation : Not Adding prefix {} to interface {}"
 754                       + " as InternetVpn has an IPV4 address {}", prefix, interfaceName, vpnName);
 755                 continue;
 756             }
 757             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 758                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 759                 Prefixes.PrefixCue prefixCue = nextHop.isPhysNetworkFunc()
 760                         ? Prefixes.PrefixCue.PhysNetFunc : Prefixes.PrefixCue.None;
 761                 LOG.debug("processVpnInterfaceAdjacencies: Adding prefix {} to interface {} with nextHops {} on dpn {}"
 762                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 763                 writeOperTxn.merge(
 764                     LogicalDatastoreType.OPERATIONAL,
 765                     VpnUtil.getPrefixToInterfaceIdentifier(
 766                         vpnUtil.getVpnId(vpnName), prefix),
 767                     VpnUtil.getPrefixToInterface(dpnId, interfaceName, prefix, nextHop.getSubnetId(),
 768                             prefixCue), true);
 769                 final Uuid subnetId = nextHop.getSubnetId();
 770
 771                 String gatewayIp = nextHop.getSubnetGatewayIp();
 772                 if (gatewayIp == null) {
 773                     Optional<String> gatewayIpOptional = vpnUtil.getVpnSubnetGatewayIp(subnetId);
 774                     if (gatewayIpOptional.isPresent()) {
 775                         gatewayIp = gatewayIpOptional.get();
 776                     }
 777                 }
 778
 779                 if (gatewayIp != null) {
 780                     gwMac = getMacAddressForSubnetIp(vpnName, interfaceName, gatewayIp);
 781                     if (gwMac.isPresent()) {
 782                         // A valid mac-address is available for this subnet-gateway-ip
 783                         // Use this for programming ARP_RESPONDER table here.  And save this
 784                         // info into vpnInterface operational, so it can used in VrfEntryProcessor
 785                         // to populate L3_GW_MAC_TABLE there.
 786                         arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 787                                 gatewayIp, gwMac.get());
 788                         vpnInterfaceSubnetGwMacAddress = gwMac.get();
 789                     } else {
 790                         // A valid mac-address is not available for this subnet-gateway-ip
 791                         // Use the connected-mac-address to configure ARP_RESPONDER Table.
 792                         // Save this connected-mac-address as gateway-mac-address for the
 793                         // VrfEntryProcessor to use this later to populate the L3_GW_MAC_TABLE.
 794                         gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 795                         if (gwMac.isPresent()) {
 796                             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn,
 797                                     NwConstants.ADD_FLOW, gwMac.get());
 798                             arpResponderHandler.addArpResponderFlow(dpnId, lportTag, interfaceName,
 799                                     gatewayIp, gwMac.get());
 800                         } else {
 801                             LOG.error("processVpnInterfaceAdjacencies: Gateway MAC for subnet ID {} could not be "
 802                                 + "obtained, cannot create ARP responder flow for interface name {}, vpnName {}, "
 803                                 + "gwIp {}",
 804                                 subnetId, interfaceName, vpnName, gatewayIp);
 805                         }
 806                     }
 807                 } else {
 808                     LOG.warn("processVpnInterfaceAdjacencies: Gateway IP for subnet ID {} could not be obtained, "
 809                         + "cannot create ARP responder flow for interface name {}, vpnName {}",
 810                         subnetId, interfaceName, vpnName);
 811                     gwMac = InterfaceUtils.getMacAddressFromInterfaceState(interfaceState);
 812                 }
 813                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} to interface {} with nextHops {} on dpn {}"
 814                         + " for vpn {}", prefix, interfaceName, nhList, dpnId, vpnName);
 815             } else {
 816                 //Extra route adjacency
 817                 String prefix = VpnUtil.getIpPrefix(nextHop.getIpAddress());
 818                 String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
 819                 synchronized (vpnPrefixKey.intern()) {
 820                     java.util.Optional<String> rdToAllocate = vpnUtil
 821                             .allocateRdForExtraRouteAndUpdateUsedRdsMap(vpnId, null, prefix, vpnName,
 822                                     nextHop.getNextHopIpList().get(0), dpnId);
 823                     if (rdToAllocate.isPresent()) {
 824                         rd = rdToAllocate.get();
 825                         LOG.info("processVpnInterfaceAdjacencies: The rd {} is allocated for the extraroute {}",
 826                             rd, prefix);
 827                     } else {
 828                         LOG.error("processVpnInterfaceAdjacencies: No rds to allocate extraroute {}", prefix);
 829                         continue;
 830                     }
 831                 }
 832                 LOG.info("processVpnInterfaceAdjacencies: Added prefix {} and nextHopList {} as extra-route for vpn{}"
 833                         + " interface {} on dpn {}", nextHop.getIpAddress(), nextHop.getNextHopIpList(), vpnName,
 834                         interfaceName, dpnId);
 835             }
 836             // Please note that primary adjacency will use a subnet-gateway-mac-address that
 837             // can be different from the gateway-mac-address within the VRFEntry as the
 838             // gateway-mac-address is a superset.
 839             RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 840                     : RouteOrigin.STATIC;
 841             L3vpnInput input = new L3vpnInput().setNextHop(nextHop).setRd(rd).setVpnName(vpnName)
 842                 .setInterfaceName(interfaceName).setNextHopIp(nextHopIp).setPrimaryRd(primaryRd)
 843                 .setSubnetGatewayMacAddress(vpnInterfaceSubnetGwMacAddress).setRouteOrigin(origin);
 844             Adjacency operationalAdjacency = null;
 845             try {
 846                 operationalAdjacency = registeredPopulator.createOperationalAdjacency(input);
 847             } catch (NullPointerException e) {
 848                 LOG.error("processVpnInterfaceAdjacencies: failed to create operational adjacency: input: {}, {}",
 849                     input, e.getMessage());
 850                 return;
 851             }
 852             if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
 853                 vpnManager.addExtraRoute(vpnName, nextHop.getIpAddress(), nextHop.getNextHopIpList().get(0), rd,
 854                         vpnName, l3vni, origin,
 855                         interfaceName, operationalAdjacency, encapType, writeConfigTxn);
 856             }
 857             value.add(operationalAdjacency);
 858         }
 859
 860         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 861         addVpnInterfaceToOperational(vpnName, interfaceName, dpnId, aug, lportTag,
 862                 gwMac.isPresent() ? gwMac.get() : null, writeOperTxn);
 863
 864         L3vpnInput input = new L3vpnInput().setNextHopIp(nextHopIp).setL3vni(l3vni).setPrimaryRd(primaryRd)
 865                 .setGatewayMac(gwMac.orNull()).setInterfaceName(interfaceName)
 866                 .setVpnName(vpnName).setDpnId(dpnId).setEncapType(encapType);
 867
 868         for (Adjacency nextHop : aug.getAdjacency()) {
 869             // Adjacencies other than primary Adjacencies are handled in the addExtraRoute call above.
 870             if (nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 871                 RouteOrigin origin = nextHop.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
 872                         : RouteOrigin.STATIC;
 873                 input.setNextHop(nextHop).setRd(nextHop.getVrfId()).setRouteOrigin(origin);
 874                 registeredPopulator.populateFib(input, writeConfigTxn);
 875             }
 876         }
 877     }
 878
 879     private void addVpnInterfaceToOperational(String vpnName, String interfaceName, BigInteger dpnId, AdjacenciesOp aug,
 880                                               long lportTag, String gwMac, WriteTransaction writeOperTxn) {
 881         VpnInterfaceOpDataEntry opInterface =
 882               VpnUtil.getVpnInterfaceOpDataEntry(interfaceName, vpnName, aug, dpnId, lportTag, gwMac);
 883         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId = VpnUtil
 884             .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
 885         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 886                 WriteTransaction.CREATE_MISSING_PARENTS);
 887         LOG.info("addVpnInterfaceToOperational: Added VPN Interface {} on dpn {} vpn {} to operational datastore",
 888                 interfaceName, dpnId, vpnName);
 889     }
 890
 891     // TODO Clean up the exception handling
 892     @SuppressWarnings("checkstyle:IllegalCatch")
 893     public void updateVpnInterfaceOnTepAdd(VpnInterfaceOpDataEntry vpnInterface,
 894                                            StateTunnelList stateTunnelList,
 895                                            WriteTransaction writeConfigTxn,
 896                                            WriteTransaction writeOperTxn) {
 897
 898         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
 899         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
 900         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
 901         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
 902         if (adjList.isEmpty()) {
 903             LOG.trace("updateVpnInterfaceOnTepAdd: Adjacencies are empty for vpnInterface {} on dpn {}",
 904                     vpnInterface, srcDpnId);
 905             return;
 906         }
 907         String prefix = null;
 908         long label = 0;
 909         List<Adjacency> value = new ArrayList<>();
 910         boolean isNextHopAddReqd = false;
 911         String vpnName = vpnInterface.getVpnInstanceName();
 912         long vpnId = vpnUtil.getVpnId(vpnName);
 913         String primaryRd = vpnUtil.getPrimaryRd(vpnName);
 914         LOG.info("updateVpnInterfaceOnTepAdd: AdjacencyList for interface {} on dpn {} vpn {} is {}",
 915                 vpnInterface.getName(), vpnInterface.getDpnId(),
 916                 vpnInterface.getVpnInstanceName(), adjList);
 917         for (Adjacency adj : adjList) {
 918             String rd = adj.getVrfId();
 919             rd = rd != null ? rd : vpnName;
 920             prefix = adj.getIpAddress();
 921             label = adj.getLabel();
 922             List<String> nhList = Collections.singletonList(srcTepIp);
 923             List<String> nextHopList = adj.getNextHopIpList();
 924             // If TEP is added , update the nexthop of primary adjacency.
 925             // Secondary adj nexthop is already pointing to primary adj IP address.
 926             if (nextHopList != null && !nextHopList.isEmpty()) {
 927                 /* everything right already */
 928             } else {
 929                 isNextHopAddReqd = true;
 930             }
 931
 932             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
 933                 value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
 934             } else {
 935                 Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
 936                 if (!vrfEntryOptional.isPresent()) {
 937                     continue;
 938                 }
 939                 nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
 940                 if (!nhList.contains(srcTepIp)) {
 941                     nhList.add(srcTepIp);
 942                     isNextHopAddReqd = true;
 943                 }
 944                 value.add(adj);
 945             }
 946
 947             if (isNextHopAddReqd) {
 948                 updateLabelMapper(label, nhList);
 949                 LOG.info("updateVpnInterfaceOnTepAdd: Updated label mapper : label {} dpn {} prefix {} nexthoplist {}"
 950                         + " vpn {} vpnid {} rd {} interface {}", label, srcDpnId , prefix, nhList,
 951                         vpnInterface.getVpnInstanceName(), vpnId, rd, vpnInterface.getName());
 952                 // Update the VRF entry with nextHop
 953                 fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
 954                         label, true, writeConfigTxn);
 955
 956                 //Get the list of VPN's importing this route(prefix) .
 957                 // Then update the VRF entry with nhList
 958                 List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
 959                 for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
 960                     String vpnRd = vpn.getVrfId();
 961                     if (vpnRd != null) {
 962                         fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
 963                                 srcTepIp, label, true, writeConfigTxn);
 964                         LOG.info("updateVpnInterfaceOnTepAdd: Exported route with rd {} prefix {} nhList {} label {}"
 965                                 + " interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix, nhList, label,
 966                             vpnInterface.getName(), srcDpnId, vpnName,
 967                             vpn.getVpnInstanceName(), vpnRd);
 968                     }
 969                 }
 970                 // Advertise the prefix to BGP only for external vpn
 971                 // since there is a nexthop change.
 972                 try {
 973                     if (!rd.equalsIgnoreCase(vpnName)) {
 974                         bgpManager.advertisePrefix(rd, null /*macAddress*/, prefix, nhList,
 975                                 VrfEntry.EncapType.Mplsgre, (int)label, 0 /*evi*/, 0 /*l2vni*/,
 976                                 null /*gatewayMacAddress*/);
 977                     }
 978                     LOG.info("updateVpnInterfaceOnTepAdd: Advertised rd {} prefix {} nhList {} label {}"
 979                             + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label, vpnInterface.getName(),
 980                             srcDpnId, vpnName);
 981                 } catch (Exception ex) {
 982                     LOG.error("updateVpnInterfaceOnTepAdd: Exception when advertising prefix {} nh {} label {}"
 983                             + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
 984                             vpnInterface.getName(), srcDpnId, vpnName, ex);
 985                 }
 986             }
 987         }
 988         AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
 989         VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
 990                 .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
 991                 .addAugmentation(AdjacenciesOp.class, aug).build();
 992         InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
 993                 VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
 994         writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
 995                 WriteTransaction.CREATE_MISSING_PARENTS);
 996         LOG.info("updateVpnInterfaceOnTepAdd: interface {} updated successully on tep add on dpn {} vpn {}",
 997                 vpnInterface.getName(), srcDpnId, vpnName);
 998
 999     }
1000
1001     // TODO Clean up the exception handling
1002     @SuppressWarnings("checkstyle:IllegalCatch")
1003     public void updateVpnInterfaceOnTepDelete(VpnInterfaceOpDataEntry vpnInterface,
1004                                               StateTunnelList stateTunnelList,
1005                                               WriteTransaction writeConfigTxn,
1006                                               WriteTransaction writeOperTxn) {
1007
1008         AdjacenciesOp adjacencies = vpnInterface.augmentation(AdjacenciesOp.class);
1009         List<Adjacency> adjList = adjacencies != null ? adjacencies.getAdjacency() : new ArrayList<>();
1010         String prefix = null;
1011         long label = 0;
1012         boolean isNextHopRemoveReqd = false;
1013         String srcTepIp = String.valueOf(stateTunnelList.getSrcInfo().getTepIp().getValue());
1014         BigInteger srcDpnId = new BigInteger(stateTunnelList.getSrcInfo().getTepDeviceId());
1015         String vpnName = vpnInterface.getVpnInstanceName();
1016         long vpnId = vpnUtil.getVpnId(vpnName);
1017         String primaryRd = vpnUtil.getVpnRd(vpnName);
1018         if (adjList != null) {
1019             List<Adjacency> value = new ArrayList<>();
1020             LOG.info("updateVpnInterfaceOnTepDelete: AdjacencyList for interface {} on dpn {} vpn {} is {}",
1021                     vpnInterface.getName(), vpnInterface.getDpnId(),
1022                     vpnInterface.getVpnInstanceName(), adjList);
1023             for (Adjacency adj : adjList) {
1024                 List<String> nhList = new ArrayList<>();
1025                 String rd = adj.getVrfId();
1026                 rd = rd != null ? rd : vpnName;
1027                 prefix = adj.getIpAddress();
1028                 List<String> nextHopList = adj.getNextHopIpList();
1029                 label = adj.getLabel();
1030                 if (nextHopList != null && !nextHopList.isEmpty()) {
1031                     isNextHopRemoveReqd = true;
1032                 }
1033                 // If TEP is deleted , remove the nexthop from primary adjacency.
1034                 // Secondary adj nexthop will continue to point to primary adj IP address.
1035                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1036                     value.add(new AdjacencyBuilder(adj).setNextHopIpList(nhList).build());
1037                 } else {
1038                     Optional<VrfEntry> vrfEntryOptional = FibHelper.getVrfEntry(dataBroker, primaryRd, prefix);
1039                     if (!vrfEntryOptional.isPresent()) {
1040                         continue;
1041                     }
1042                     nhList = FibHelper.getNextHopListFromRoutePaths(vrfEntryOptional.get());
1043                     if (nhList.contains(srcTepIp)) {
1044                         nhList.remove(srcTepIp);
1045                         isNextHopRemoveReqd = true;
1046                     }
1047                     value.add(adj);
1048                 }
1049
1050                 if (isNextHopRemoveReqd) {
1051                     updateLabelMapper(label, nhList);
1052                     LOG.info("updateVpnInterfaceOnTepDelete: Updated label mapper : label {} dpn {} prefix {}"
1053                             + " nexthoplist {} vpn {} vpnid {} rd {} interface {}", label, srcDpnId,
1054                             prefix, nhList, vpnName,
1055                             vpnId, rd, vpnInterface.getName());
1056                     // Update the VRF entry with removed nextHop
1057                     fibManager.updateRoutePathForFibEntry(primaryRd, prefix, srcTepIp,
1058                             label, false, writeConfigTxn);
1059
1060                     //Get the list of VPN's importing this route(prefix) .
1061                     // Then update the VRF entry with nhList
1062                     List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1063                     for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1064                         String vpnRd = vpn.getVrfId();
1065                         if (vpnRd != null) {
1066                             fibManager.updateRoutePathForFibEntry(vpnRd, prefix,
1067                                     srcTepIp, label, false, writeConfigTxn);
1068                             LOG.info("updateVpnInterfaceOnTepDelete: Exported route with rd {} prefix {} nhList {}"
1069                                     + " label {} interface {} dpn {} from vpn {} to VPN {} vpnRd {}", rd, prefix,
1070                                     nhList, label, vpnInterface.getName(), srcDpnId,
1071                                     vpnName,
1072                                     vpn.getVpnInstanceName(), vpnRd);
1073                         }
1074                     }
1075
1076                     // Withdraw prefix from BGP only for external vpn.
1077                     try {
1078                         if (!rd.equalsIgnoreCase(vpnName)) {
1079                             bgpManager.withdrawPrefix(rd, prefix);
1080                         }
1081                         LOG.info("updateVpnInterfaceOnTepDelete: Withdrawn rd {} prefix {} nhList {} label {}"
1082                                 + " for interface {} on dpn {} vpn {}", rd, prefix, nhList, label,
1083                                 vpnInterface.getName(), srcDpnId,
1084                                 vpnName);
1085                     } catch (Exception ex) {
1086                         LOG.error("updateVpnInterfaceOnTepDelete: Exception when withdrawing prefix {} nh {} label {}"
1087                                 + " on rd {} for interface {} on dpn {} vpn {}", prefix, nhList, label, rd,
1088                                 vpnInterface.getName(), srcDpnId, vpnName, ex);
1089                     }
1090                 }
1091             }
1092             AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(value);
1093             VpnInterfaceOpDataEntry opInterface = new VpnInterfaceOpDataEntryBuilder(vpnInterface)
1094                     .withKey(new VpnInterfaceOpDataEntryKey(vpnInterface.getName(), vpnName))
1095                     .addAugmentation(AdjacenciesOp.class, aug).build();
1096             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1097                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getName(), vpnName);
1098             writeOperTxn.put(LogicalDatastoreType.OPERATIONAL, interfaceId, opInterface,
1099                     WriteTransaction.CREATE_MISSING_PARENTS);
1100             LOG.info("updateVpnInterfaceOnTepDelete: interface {} updated successully on tep delete on dpn {} vpn {}",
1101                          vpnInterface.getName(), srcDpnId, vpnName);
1102         }
1103     }
1104
1105     private List<VpnInstanceOpDataEntry> getVpnsExportingMyRoute(final String vpnName) {
1106         List<VpnInstanceOpDataEntry> vpnsToExportRoute = new ArrayList<>();
1107
1108         String vpnRd = vpnUtil.getVpnRd(vpnName);
1109         final VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
1110         if (vpnInstanceOpDataEntry == null) {
1111             LOG.debug("getVpnsExportingMyRoute: Could not retrieve vpn instance op data for {}"
1112                     + " to check for vpns exporting the routes", vpnName);
1113             return vpnsToExportRoute;
1114         }
1115
1116         Predicate<VpnInstanceOpDataEntry> excludeVpn = input -> {
1117             if (input.getVpnInstanceName() == null) {
1118                 LOG.error("getVpnsExportingMyRoute.excludeVpn: Received vpn instance with rd {}  without a name",
1119                         input.getVrfId());
1120                 return false;
1121             }
1122             return !input.getVpnInstanceName().equals(vpnName);
1123         };
1124
1125         Predicate<VpnInstanceOpDataEntry> matchRTs = input -> {
1126             Iterable<String> commonRTs =
1127                     VpnUtil.intersection(VpnUtil.getRts(vpnInstanceOpDataEntry, VpnTarget.VrfRTType.ImportExtcommunity),
1128                         VpnUtil.getRts(input, VpnTarget.VrfRTType.ExportExtcommunity));
1129             return Iterators.size(commonRTs.iterator()) > 0;
1130         };
1131
1132         vpnsToExportRoute =
1133                 vpnUtil.getAllVpnInstanceOpData().stream().filter(excludeVpn).filter(matchRTs).collect(
1134                         Collectors.toList());
1135         return vpnsToExportRoute;
1136     }
1137
1138     // TODO Clean up the exception handling
1139     @SuppressWarnings("checkstyle:IllegalCatch")
1140     void handleVpnsExportingRoutes(String vpnName, String vpnRd) {
1141         List<VpnInstanceOpDataEntry> vpnsToExportRoute = getVpnsExportingMyRoute(vpnName);
1142         for (VpnInstanceOpDataEntry vpn : vpnsToExportRoute) {
1143             List<VrfEntry> vrfEntries = vpnUtil.getAllVrfEntries(vpn.getVrfId());
1144             if (vrfEntries != null) {
1145                 ListenableFutures.addErrorLogging(txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1146                     for (VrfEntry vrfEntry : vrfEntries) {
1147                         try {
1148                             if (!FibHelper.isControllerManagedNonInterVpnLinkRoute(
1149                                     RouteOrigin.value(vrfEntry.getOrigin()))) {
1150                                 LOG.info("handleVpnsExportingRoutes: vrfEntry with rd {} prefix {}"
1151                                                 + " is not a controller managed non intervpn link route. Ignoring.",
1152                                         vpn.getVrfId(), vrfEntry.getDestPrefix());
1153                                 continue;
1154                             }
1155                             String prefix = vrfEntry.getDestPrefix();
1156                             String gwMac = vrfEntry.getGatewayMacAddress();
1157                             vrfEntry.getRoutePaths().forEach(routePath -> {
1158                                 String nh = routePath.getNexthopAddress();
1159                                 int label = routePath.getLabel().intValue();
1160                                 if (FibHelper.isControllerManagedVpnInterfaceRoute(RouteOrigin.value(
1161                                         vrfEntry.getOrigin()))) {
1162                                     LOG.info("handleVpnsExportingRoutesImporting: Importing fib entry rd {} prefix {}"
1163                                                     + " nexthop {} label {} to vpn {} vpnRd {}",
1164                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1165                                     fibManager.addOrUpdateFibEntry(vpnRd, null /*macAddress*/, prefix,
1166                                             Collections.singletonList(nh), VrfEntry.EncapType.Mplsgre, label,
1167                                             0 /*l3vni*/, gwMac,  vpn.getVrfId(), RouteOrigin.SELF_IMPORTED,
1168                                             confTx);
1169                                 } else {
1170                                     LOG.info("handleVpnsExportingRoutes: Importing subnet route fib entry rd {} "
1171                                                     + "prefix {} nexthop {} label {} to vpn {} vpnRd {}",
1172                                             vpn.getVrfId(), prefix, nh, label, vpnName, vpnRd);
1173                                     SubnetRoute route = vrfEntry.augmentation(SubnetRoute.class);
1174                                     importSubnetRouteForNewVpn(vpnRd, prefix, nh, label, route, vpn.getVrfId(),
1175                                             confTx);
1176                                 }
1177                             });
1178                         } catch (RuntimeException e) {
1179                             LOG.error("getNextHopAddressList: Exception occurred while importing route with rd {}"
1180                                             + " prefix {} routePaths {} to vpn {} vpnRd {}", vpn.getVrfId(),
1181                                     vrfEntry.getDestPrefix(), vrfEntry.getRoutePaths(), vpnName, vpnRd);
1182                         }
1183                     }
1184                 }), LOG, "Error handing VPN exporting routes");
1185             } else {
1186                 LOG.info("getNextHopAddressList: No vrf entries to import from vpn {} with rd {} to vpn {} with rd {}",
1187                         vpn.getVpnInstanceName(), vpn.getVrfId(), vpnName, vpnRd);
1188             }
1189         }
1190     }
1191
1192     @Override
1193     public void remove(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
1194         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1195         final String interfaceName = key.getName();
1196         for (VpnInstanceNames vpnInterfaceVpnInstance : vpnInterface.getVpnInstanceNames()) {
1197             String vpnName = vpnInterfaceVpnInstance.getVpnName();
1198             removeVpnInterfaceCall(identifier, vpnInterface, vpnName, interfaceName);
1199         }
1200     }
1201
1202     private void removeVpnInterfaceCall(final InstanceIdentifier<VpnInterface> identifier,
1203                                 final VpnInterface vpnInterface, final String vpnName,
1204                                 final String interfaceName) {
1205         if (Boolean.TRUE.equals(vpnInterface.isRouterInterface())) {
1206             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getName(), () -> {
1207                 ListenableFuture<Void> future = txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1208                     deleteFibEntryForRouterInterface(vpnInterface, confTx, vpnName);
1209                     LOG.info("remove: Router interface {} for vpn {}", interfaceName, vpnName);
1210                 });
1211                 ListenableFutures.addErrorLogging(future, LOG, "Error removing call for interface {} on VPN {}",
1212                         vpnInterface.getName(), vpnName);
1213                 return Collections.singletonList(future);
1214             }, DJC_MAX_RETRIES);
1215         } else {
1216             removeVpnInterfaceFromVpn(identifier, vpnInterface, vpnName, interfaceName);
1217         }
1218     }
1219
1220     @SuppressFBWarnings("DLS_DEAD_LOCAL_STORE")
1221     private void removeVpnInterfaceFromVpn(final InstanceIdentifier<VpnInterface> identifier,
1222                                 final VpnInterface vpnInterface, final String vpnName,
1223                                 final String interfaceName) {
1224         LOG.info("remove: VPN Interface remove event - intfName {} vpn {} dpn {}" ,vpnInterface.getName(),
1225                 vpnName, vpnInterface.getDpnId());
1226         removeInterfaceFromUnprocessedList(identifier, vpnInterface);
1227         jobCoordinator.enqueueJob("VPNINTERFACE-" + interfaceName,
1228             () -> {
1229                 List<ListenableFuture<Void>> futures = new ArrayList<>(3);
1230                 ListenableFuture<Void> configFuture = txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1231                     writeConfigTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
1232                         writeOperTxn -> futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(writeInvTxn -> {
1233                             LOG.info("remove: - intfName {} onto vpnName {} running config-driven",
1234                                     interfaceName, vpnName);
1235                             BigInteger dpId = BigInteger.ZERO;
1236                             int ifIndex = 0;
1237                             String gwMacAddress = null;
1238                             InstanceIdentifier<VpnInterfaceOpDataEntry> interfaceId =
1239                                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1240                             Optional<VpnInterfaceOpDataEntry> optVpnInterface = Optional.absent();
1241                             try {
1242                                 optVpnInterface = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1243                                         LogicalDatastoreType.OPERATIONAL, interfaceId);
1244                             } catch (ReadFailedException e) {
1245                                 LOG.error("remove: Failed to read data store for interface {} vpn {}", interfaceName,
1246                                         vpnName);
1247                                 return;
1248                             }
1249                             Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1250                                     interfaceName);
1251                             if (interfaceState != null) {
1252                                 try {
1253                                     dpId = InterfaceUtils.getDpIdFromInterface(interfaceState);
1254                                 } catch (NumberFormatException | IllegalStateException e) {
1255                                     LOG.error("remove: Unable to retrieve dpnId from interface operational"
1256                                             + " data store for interface {} on dpn {} for vpn {} Fetching"
1257                                                     + " from vpn interface op data store. ", interfaceName,
1258                                             vpnInterface.getDpnId(), vpnName, e);
1259                                     dpId = BigInteger.ZERO;
1260                                 }
1261                                 ifIndex = interfaceState.getIfIndex();
1262                                 gwMacAddress = interfaceState.getPhysAddress().getValue();
1263                             } else {
1264                                 LOG.info("remove: Interface state not available for {}. Trying to fetch data"
1265                                         + " from vpn interface op.", interfaceName);
1266                                 if (optVpnInterface.isPresent()) {
1267                                     VpnInterfaceOpDataEntry vpnOpInterface = optVpnInterface.get();
1268                                     dpId = vpnOpInterface.getDpnId();
1269                                     ifIndex = vpnOpInterface.getLportTag().intValue();
1270                                     gwMacAddress = vpnOpInterface.getGatewayMacAddress();
1271                                 } else {
1272                                     LOG.error("remove: Handling removal of VPN interface {} for vpn {} skipped"
1273                                                     + " as interfaceState and vpn interface op is not"
1274                                             + " available", interfaceName, vpnName);
1275                                     return;
1276                                 }
1277                             }
1278                             processVpnInterfaceDown(dpId, interfaceName, ifIndex, gwMacAddress,
1279                                     optVpnInterface.isPresent() ? optVpnInterface.get() : null, false,
1280                                     writeConfigTxn, writeOperTxn, writeInvTxn);
1281                             LOG.info(
1282                                     "remove: Removal of vpn interface {} on dpn {} for vpn {} processed "
1283                                             + "successfully",
1284                                     interfaceName, vpnInterface.getDpnId(), vpnName);
1285                         })))));
1286                 futures.add(configFuture);
1287                 Futures.addCallback(configFuture, new PostVpnInterfaceWorker(interfaceName, false, "Config"));
1288                 return futures;
1289             }, DJC_MAX_RETRIES);
1290     }
1291
1292     protected void processVpnInterfaceDown(BigInteger dpId,
1293                                            String interfaceName,
1294                                            int lportTag,
1295                                            String gwMac,
1296                                            VpnInterfaceOpDataEntry vpnOpInterface,
1297                                            boolean isInterfaceStateDown,
1298                                            WriteTransaction writeConfigTxn,
1299                                            WriteTransaction writeOperTxn,
1300                                            WriteTransaction writeInvTxn) {
1301         if (vpnOpInterface == null) {
1302             LOG.error("processVpnInterfaceDown: Unable to process delete/down for interface {} on dpn {}"
1303                     + " as it is not available in operational data store", interfaceName, dpId);
1304             return;
1305         }
1306         final String vpnName = vpnOpInterface.getVpnInstanceName();
1307         InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil.getVpnInterfaceOpDataEntryIdentifier(
1308                                                     interfaceName, vpnName);
1309         if (!isInterfaceStateDown) {
1310             final long vpnId = vpnUtil.getVpnId(vpnName);
1311             vpnUtil.scheduleVpnInterfaceForRemoval(interfaceName, dpId, vpnName,  null);
1312             final boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(vpnName);
1313             removeAdjacenciesFromVpn(dpId, lportTag, interfaceName, vpnName,
1314                     vpnId, gwMac, writeConfigTxn, writeOperTxn, writeInvTxn);
1315             if (interfaceManager.isExternalInterface(interfaceName)) {
1316                 processExternalVpnInterface(interfaceName, vpnName, dpId, lportTag,
1317                     NwConstants.DEL_FLOW);
1318             }
1319             if (!isBgpVpnInternetVpn) {
1320                 vpnUtil.unbindService(interfaceName, isInterfaceStateDown);
1321             }
1322             LOG.info("processVpnInterfaceDown: Unbound vpn service from interface {} on dpn {} for vpn {}"
1323                     + " successful", interfaceName, dpId, vpnName);
1324         } else {
1325             // Interface is retained in the DPN, but its Link Down.
1326             // Only withdraw the prefixes for this interface from BGP
1327             withdrawAdjacenciesForVpnFromBgp(identifier, vpnName, interfaceName, writeConfigTxn, writeOperTxn);
1328         }
1329     }
1330
1331     private void removeAdjacenciesFromVpn(final BigInteger dpnId, final int lportTag, final String interfaceName,
1332                                           final String vpnName, final long vpnId, String gwMac,
1333                                           WriteTransaction writeConfigTxn, final WriteTransaction writeOperTxn,
1334                                           final WriteTransaction writeInvTxn) {
1335         //Read NextHops
1336         try {
1337             InstanceIdentifier<VpnInterfaceOpDataEntry> identifier = VpnUtil
1338                     .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1339             InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1340             Optional<AdjacenciesOp> adjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1341                     LogicalDatastoreType.OPERATIONAL, path);
1342             String primaryRd = vpnUtil.getVpnRd(vpnName);
1343             LOG.info("removeAdjacenciesFromVpn: For interface {} on dpn {} RD recovered for vpn {} as rd {}",
1344                     interfaceName, dpnId, vpnName, primaryRd);
1345             if (adjacencies.isPresent() && !adjacencies.get().getAdjacency().isEmpty()) {
1346                 List<Adjacency> nextHops = adjacencies.get().getAdjacency();
1347                 LOG.info("removeAdjacenciesFromVpn: NextHops for interface {} on dpn {} for vpn {} are {}",
1348                         interfaceName, dpnId, vpnName, nextHops);
1349                 for (Adjacency nextHop : nextHops) {
1350                     if (nextHop.isPhysNetworkFunc()) {
1351                         LOG.info("removeAdjacenciesFromVpn: Removing PNF FIB entry rd {} prefix {}",
1352                                 nextHop.getSubnetId().getValue(), nextHop.getIpAddress());
1353                         fibManager.removeFibEntry(nextHop.getSubnetId().getValue(), nextHop.getIpAddress(),
1354                                 null/*writeCfgTxn*/);
1355                     } else {
1356                         String rd = nextHop.getVrfId();
1357                         List<String> nhList;
1358                         if (nextHop.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1359                             nhList = getNextHopForNonPrimaryAdjacency(nextHop, vpnName, dpnId, interfaceName);
1360                         } else {
1361                             // This is a primary adjacency
1362                             nhList = nextHop.getNextHopIpList() != null ? nextHop.getNextHopIpList()
1363                                     : Collections.emptyList();
1364                             removeGwMacAndArpResponderFlows(nextHop, vpnId, dpnId, lportTag, gwMac,
1365                                     interfaceName, writeInvTxn);
1366                         }
1367                         if (!nhList.isEmpty()) {
1368                             if (rd.equals(vpnName)) {
1369                                 //this is an internal vpn - the rd is assigned to the vpn instance name;
1370                                 //remove from FIB directly
1371                                 nhList.forEach(removeAdjacencyFromInternalVpn(nextHop, vpnName,
1372                                         interfaceName, dpnId, writeConfigTxn, writeOperTxn));
1373                             } else {
1374                                 removeAdjacencyFromBgpvpn(nextHop, nhList, vpnName, primaryRd, dpnId, rd,
1375                                         interfaceName, writeConfigTxn, writeOperTxn);
1376                             }
1377                         } else {
1378                             LOG.error("removeAdjacenciesFromVpn: nextHop empty for ip {} rd {} adjacencyType {}"
1379                                             + " interface {}", nextHop.getIpAddress(), rd,
1380                                     nextHop.getAdjacencyType().toString(), interfaceName);
1381                             bgpManager.withdrawPrefixIfPresent(rd, nextHop.getIpAddress());
1382                             fibManager.removeFibEntry(primaryRd, nextHop.getIpAddress(), writeConfigTxn);
1383                         }
1384                     }
1385                     String ip = nextHop.getIpAddress().split("/")[0];
1386                     LearntVpnVipToPort vpnVipToPort = vpnUtil.getLearntVpnVipToPort(vpnName, ip);
1387                     if (vpnVipToPort != null) {
1388                         vpnUtil.removeLearntVpnVipToPort(vpnName, ip, null);
1389                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed LearntVpnVipToPort entry"
1390                                  + " for Interface {} ip {} on dpn {} for vpn {}",
1391                                 vpnVipToPort.getPortName(), ip, dpnId, vpnName);
1392                     }
1393                     VpnPortipToPort vpnPortipToPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ip);
1394                     if (vpnPortipToPort != null) {
1395                         VpnUtil.removeVpnPortFixedIpToPort(dataBroker, vpnName, ip, null);
1396                         LOG.info("removeAdjacenciesFromVpn: VpnInterfaceManager removed vpnPortipToPort entry for "
1397                                  + "Interface {} ip {} on dpn {} for vpn {}",
1398                             vpnPortipToPort.getPortName(), ip, dpnId, vpnName);
1399                     }
1400                 }
1401             } else {
1402                 // this vpn interface has no more adjacency left, so clean up the vpn interface from Operational DS
1403                 LOG.info("removeAdjacenciesFromVpn: Vpn Interface {} on vpn {} dpn {} has no adjacencies."
1404                         + " Removing it.", interfaceName, vpnName, dpnId);
1405                 writeOperTxn.delete(LogicalDatastoreType.OPERATIONAL, identifier);
1406             }
1407         } catch (ReadFailedException e) {
1408             LOG.error("removeAdjacenciesFromVpn: Failed to read data store for interface {} dpn {} vpn {}",
1409                     interfaceName, dpnId, vpnName);
1410         }
1411     }
1412
1413     private Consumer<String> removeAdjacencyFromInternalVpn(Adjacency nextHop, String vpnName, String interfaceName,
1414                                                             BigInteger dpnId, WriteTransaction writeConfigTxn,
1415                                                             WriteTransaction writeOperTx) {
1416         return (nh) -> {
1417             String primaryRd = vpnUtil.getVpnRd(vpnName);
1418             String prefix = nextHop.getIpAddress();
1419             String vpnNamePrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1420             LOG.info("remove adjacencies for nexthop {} vpnName {} interfaceName {} dpnId {}",
1421                     nextHop, vpnName, interfaceName, dpnId);
1422             synchronized (vpnNamePrefixKey.intern()) {
1423                 if (vpnUtil.removeOrUpdateDSForExtraRoute(vpnName, primaryRd, dpnId.toString(), interfaceName,
1424                         prefix, nextHop.getNextHopIpList().get(0), nh, writeOperTx)) {
1425                     //If extra-route is present behind at least one VM, then do not remove or update
1426                     //fib entry for route-path representing that CSS nexthop, just update vpntoextraroute and
1427                     //prefixtointerface DS
1428                     return;
1429                 }
1430                 fibManager.removeOrUpdateFibEntry(vpnName, nextHop.getIpAddress(), nh,
1431                         writeConfigTxn);
1432             }
1433             LOG.info("removeAdjacenciesFromVpn: removed/updated FIB with rd {} prefix {}"
1434                             + " nexthop {} for interface {} on dpn {} for internal vpn {}",
1435                     vpnName, nextHop.getIpAddress(), nh, interfaceName, dpnId, vpnName);
1436         };
1437     }
1438
1439     private void removeAdjacencyFromBgpvpn(Adjacency nextHop, List<String> nhList, String vpnName, String primaryRd,
1440                                            BigInteger dpnId, String rd, String interfaceName,
1441                                            WriteTransaction writeConfigTxn, WriteTransaction writeOperTx) {
1442         List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1443                 vpnUtil.getVpnsImportingMyRoute(vpnName);
1444         nhList.forEach((nh) -> {
1445             //IRT: remove routes from other vpns importing it
1446             vpnManager.removePrefixFromBGP(vpnName, primaryRd, rd, interfaceName, nextHop.getIpAddress(),
1447                     nextHop.getNextHopIpList().get(0), nh, dpnId, writeConfigTxn, writeOperTx);
1448             for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1449                 String vpnRd = vpn.getVrfId();
1450                 if (vpnRd != null) {
1451                     fibManager.removeOrUpdateFibEntry(vpnRd,
1452                             nextHop.getIpAddress(), nh, writeConfigTxn);
1453                     LOG.info("removeAdjacenciesFromVpn: Removed Exported route with rd {}"
1454                                     + " prefix {} nextHop {} from VPN {} parentVpn {}"
1455                                     + " for interface {} on dpn {}", vpnRd, nextHop.getIpAddress(), nh,
1456                             vpn.getVpnInstanceName(), vpnName, interfaceName, dpnId);
1457                 }
1458             }
1459         });
1460     }
1461
1462     private void removeGwMacAndArpResponderFlows(Adjacency nextHop, long vpnId, BigInteger dpnId,
1463                                                  int lportTag, String gwMac, String interfaceName,
1464                                                  WriteTransaction writeInvTxn) {
1465         final Uuid subnetId = nextHop.getSubnetId();
1466         if (nextHop.getSubnetGatewayMacAddress() == null) {
1467             // A valid mac-address was not available for this subnet-gateway-ip
1468             // So a connected-mac-address was used for this subnet and we need
1469             // to remove the flows for the same here from the L3_GW_MAC_TABLE.
1470             vpnUtil.setupGwMacIfExternalVpn(dpnId, interfaceName, vpnId, writeInvTxn, NwConstants.DEL_FLOW, gwMac);
1471         }
1472         arpResponderHandler.removeArpResponderFlow(dpnId, lportTag, interfaceName, nextHop.getSubnetGatewayIp(),
1473                 subnetId);
1474     }
1475
1476     private List<String> getNextHopForNonPrimaryAdjacency(Adjacency nextHop, String vpnName, BigInteger dpnId,
1477                                                   String interfaceName) {
1478         // This is either an extra-route (or) a learned IP via subnet-route
1479         List<String> nhList = null;
1480         String nextHopIp = InterfaceUtils.getEndpointIpAddressForDPN(dataBroker, dpnId);
1481         if (nextHopIp == null || nextHopIp.isEmpty()) {
1482             LOG.error("removeAdjacenciesFromVpn: Unable to obtain nextHopIp for"
1483                             + " extra-route/learned-route in rd {} prefix {} interface {} on dpn {}"
1484                             + " for vpn {}", nextHop.getVrfId(), nextHop.getIpAddress(), interfaceName, dpnId,
1485                     vpnName);
1486             nhList = Collections.emptyList();
1487         } else {
1488             nhList = Collections.singletonList(nextHopIp);
1489         }
1490         return nhList;
1491     }
1492
1493     private Optional<String> getMacAddressForSubnetIp(String vpnName, String ifName, String ipAddress) {
1494         VpnPortipToPort gwPort = vpnUtil.getNeutronPortFromVpnPortFixedIp(vpnName, ipAddress);
1495         //Check if a router gateway interface is available for the subnet gw is so then use Router interface
1496         // else use connected interface
1497         if (gwPort != null && gwPort.isSubnetIp()) {
1498             LOG.info("getGatewayMacAddressForSubnetIp: Retrieved gw Mac as {} for ip {} interface {} vpn {}",
1499                     gwPort.getMacAddress(), ipAddress, ifName, vpnName);
1500             return Optional.of(gwPort.getMacAddress());
1501         }
1502         return Optional.absent();
1503     }
1504
1505     @Override
1506     protected void update(final InstanceIdentifier<VpnInterface> identifier, final VpnInterface original,
1507         final VpnInterface update) {
1508         LOG.info("update: VPN Interface update event - intfName {} on dpn {} oldVpn {} newVpn {}" ,update.getName(),
1509                 update.getDpnId(), original.getVpnInstanceNames(),
1510                 update.getVpnInstanceNames());
1511         final String vpnInterfaceName = update.getName();
1512         final BigInteger dpnId = InterfaceUtils.getDpnForInterface(ifaceMgrRpcService, vpnInterfaceName);
1513         final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1514         final List<Adjacency> oldAdjs = origAdjs != null && origAdjs.getAdjacency()
1515             != null ? origAdjs.getAdjacency() : new ArrayList<>();
1516         final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1517         final List<Adjacency> newAdjs = updateAdjs != null && updateAdjs.getAdjacency()
1518             != null ? updateAdjs.getAdjacency() : new ArrayList<>();
1519
1520         LOG.info("VPN Interface update event - intfName {}", vpnInterfaceName);
1521         //handles switching between <internal VPN - external VPN>
1522         jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterfaceName, () -> {
1523             if (handleVpnSwapForVpnInterface(identifier, original, update)) {
1524                 LOG.info("update: handled VPNInterface {} on dpn {} update"
1525                                 + "upon VPN swap from oldVpn(s) {} to newVpn(s) {}",
1526                         original.getName(), dpnId,
1527                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(original.getVpnInstanceNames()),
1528                         VpnHelper.getVpnInterfaceVpnInstanceNamesString(update.getVpnInstanceNames()));
1529                 return Collections.emptyList();
1530             }
1531             for (VpnInstanceNames vpnInterfaceVpnInstance : update.getVpnInstanceNames()) {
1532                 String newVpnName = vpnInterfaceVpnInstance.getVpnName();
1533                 List<Adjacency> copyNewAdjs = new ArrayList<>(newAdjs);
1534                 List<Adjacency> copyOldAdjs = new ArrayList<>(oldAdjs);
1535                 String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1536                 if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1537                     // TODO Deal with sequencing — the config tx must only submitted if the oper tx goes in
1538                     List<ListenableFuture<Void>> futures = new ArrayList<>();
1539                     futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(confTx -> {
1540                         futures.add(txRunner.callWithNewReadWriteTransactionAndSubmit(operTx -> {
1541                             InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier =
1542                                     VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterfaceName, newVpnName);
1543                             LOG.info("VPN Interface update event - intfName {} onto vpnName {} running config-driven",
1544                                     update.getName(), newVpnName);
1545                             //handle both addition and removal of adjacencies
1546                             //currently, new adjacency may be an extra route
1547                             boolean isBgpVpnInternetVpn = vpnUtil.isBgpVpnInternet(newVpnName);
1548                             if (!oldAdjs.equals(newAdjs)) {
1549                                 for (Adjacency adj : copyNewAdjs) {
1550                                     if (copyOldAdjs.contains(adj)) {
1551                                         copyOldAdjs.remove(adj);
1552                                     } else {
1553                                         // add new adjacency - right now only extra route will hit this path
1554                                         if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1555                                             addNewAdjToVpnInterface(vpnInterfaceOpIdentifier, primaryRd, adj,
1556                                                     dpnId, operTx, confTx);
1557                                         }
1558                                         LOG.info("update: new Adjacency {} with nextHop {} label {} subnet {} added to"
1559                                                 + " vpn interface {} on vpn {} dpnId {}",
1560                                                 adj.getIpAddress(), adj.getNextHopIpList(),
1561                                                 adj.getLabel(), adj.getSubnetId(), update.getName(),
1562                                                 newVpnName, dpnId);
1563                                     }
1564                                 }
1565                                 for (Adjacency adj : copyOldAdjs) {
1566                                     if (!isBgpVpnInternetVpn || vpnUtil.isAdjacencyEligibleToVpnInternet(adj)) {
1567                                         if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency
1568                                                 && !adj.isPhysNetworkFunc()) {
1569                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1570                                                     operTx, confTx);
1571                                             Optional<VpnInterfaceOpDataEntry> optVpnInterface = operTx.read(
1572                                                     LogicalDatastoreType.OPERATIONAL, vpnInterfaceOpIdentifier).get();
1573                                             if (optVpnInterface.isPresent()) {
1574                                                 VpnInterfaceOpDataEntry vpnInterfaceOpDataEntry = optVpnInterface.get();
1575                                                 long vpnId = vpnUtil.getVpnId(newVpnName);
1576                                                 vpnUtil.removePrefixToInterfaceAdj(adj, vpnId,
1577                                                         vpnInterfaceOpDataEntry, operTx);
1578                                             } else {
1579                                                 LOG.info("update: Vpninterface {} not present in Operational",
1580                                                         vpnInterfaceName);
1581                                             }
1582                                             //remove FIB entry
1583                                             String vpnRd = vpnUtil.getVpnRd(newVpnName);
1584                                             LOG.debug("update: remove prefix {} from the FIB and BGP entry "
1585                                                     + "for the Vpn-Rd {} ", adj.getIpAddress(), vpnRd);
1586                                             //remove BGP entry
1587                                             fibManager.removeFibEntry(vpnRd, adj.getIpAddress(), confTx);
1588                                             if (vpnRd != null && !vpnRd.equalsIgnoreCase(newVpnName)) {
1589                                                 bgpManager.withdrawPrefix(vpnRd, adj.getIpAddress());
1590                                             }
1591                                         } else {
1592                                             delAdjFromVpnInterface(vpnInterfaceOpIdentifier, adj, dpnId,
1593                                                     operTx, confTx);
1594                                         }
1595                                     }
1596                                     LOG.info("update: Adjacency {} with nextHop {} label {} subnet {} removed from"
1597                                                     + " vpn interface {} on vpn {}", adj.getIpAddress(), adj
1598                                                     .getNextHopIpList(),
1599                                             adj.getLabel(), adj.getSubnetId(), update.getName(), newVpnName);
1600                                 }
1601                             }
1602                         }));
1603                     }));
1604                     for (ListenableFuture<Void> future : futures) {
1605                         ListenableFutures.addErrorLogging(future, LOG, "update: failed for interface {} on vpn {}",
1606                                 update.getName(), update.getVpnInstanceNames());
1607                     }
1608                     return futures;
1609                 } else {
1610                     LOG.error("update: Ignoring update of vpnInterface {}, as newVpnInstance {} with primaryRd {}"
1611                             + " is already marked for deletion", vpnInterfaceName, newVpnName, primaryRd);
1612                 }
1613             }
1614             return Collections.emptyList();
1615         });
1616     }
1617
1618     private boolean handleVpnSwapForVpnInterface(InstanceIdentifier<VpnInterface> identifier,
1619                                                  VpnInterface original, VpnInterface update) {
1620         boolean isSwap = Boolean.FALSE;
1621         final VpnInterfaceKey key = identifier.firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1622         final String interfaceName = key.getName();
1623         List<String> oldVpnList = original.getVpnInstanceNames().stream()
1624             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1625         List<String> oldVpnListCopy = new ArrayList<>();
1626         oldVpnListCopy.addAll(oldVpnList);
1627         List<String> newVpnList = update.getVpnInstanceNames().stream()
1628             .map(VpnInstanceNames::getVpnName).collect(Collectors.toList());
1629         oldVpnList.removeAll(newVpnList);
1630         newVpnList.removeAll(oldVpnListCopy);
1631         if (!oldVpnList.isEmpty() || !newVpnList.isEmpty()) {
1632             for (String oldVpnName: oldVpnList) {
1633                 isSwap = Boolean.TRUE;
1634                 LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} remove vpnName {}"
1635                         + " running config-driven swap removal", interfaceName, oldVpnName);
1636                 removeVpnInterfaceCall(identifier, original, oldVpnName, interfaceName);
1637                 LOG.info("handleVpnSwapForVpnInterface: Processed Remove for update on VPNInterface {} upon VPN swap"
1638                         + "from old vpn {} to newVpn(s) {}", interfaceName, oldVpnName, newVpnList);
1639             }
1640             //Wait for previous interface bindings to be removed
1641             try {
1642                 Thread.sleep(2000);
1643             } catch (InterruptedException e) {
1644                 //Ignore
1645             }
1646
1647             final Adjacencies origAdjs = original.augmentation(Adjacencies.class);
1648             final List<Adjacency> oldAdjs = (origAdjs != null && origAdjs.getAdjacency() != null)
1649                     ? origAdjs.getAdjacency() : new ArrayList<>();
1650             final Adjacencies updateAdjs = update.augmentation(Adjacencies.class);
1651             final List<Adjacency> newAdjs = (updateAdjs != null && updateAdjs.getAdjacency() != null)
1652                     ? updateAdjs.getAdjacency() : new ArrayList<>();
1653
1654             for (String newVpnName: newVpnList) {
1655                 String primaryRd = vpnUtil.getPrimaryRd(newVpnName);
1656                 isSwap = Boolean.TRUE;
1657                 if (!vpnUtil.isVpnPendingDelete(primaryRd)) {
1658                     LOG.info("handleVpnSwapForVpnInterface: VPN Interface update event - intfName {} onto vpnName {}"
1659                             + "running config-driven swap addition", interfaceName, newVpnName);
1660                     addVpnInterfaceCall(identifier, update, oldAdjs, newAdjs, newVpnName);
1661                     LOG.info("handleVpnSwapForVpnInterface: Processed Add for update on VPNInterface {}"
1662                                     + "from oldVpn(s) {} to newVpn {} upon VPN swap",
1663                             interfaceName, oldVpnListCopy, newVpnName);
1664                 }
1665             }
1666         }
1667         return isSwap;
1668     }
1669
1670     private void updateLabelMapper(Long label, List<String> nextHopIpList) {
1671         try {
1672             Preconditions.checkNotNull(label, "updateLabelMapper: label cannot be null or empty!");
1673             synchronized (label.toString().intern()) {
1674                 InstanceIdentifier<LabelRouteInfo> lriIid = InstanceIdentifier.builder(LabelRouteMap.class)
1675                         .child(LabelRouteInfo.class, new LabelRouteInfoKey(label)).build();
1676                 Optional<LabelRouteInfo> opResult = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1677                         LogicalDatastoreType.OPERATIONAL, lriIid);
1678                 if (opResult.isPresent()) {
1679                     LabelRouteInfo labelRouteInfo =
1680                             new LabelRouteInfoBuilder(opResult.get()).setNextHopIpList(nextHopIpList).build();
1681                     SingleTransactionDataBroker.syncWrite(dataBroker, LogicalDatastoreType.OPERATIONAL, lriIid,
1682                             labelRouteInfo, VpnUtil.SINGLE_TRANSACTION_BROKER_NO_RETRY);
1683                 }
1684             }
1685             LOG.info("updateLabelMapper: Updated label rotue info for label {} with nextHopList {}", label,
1686                     nextHopIpList);
1687         } catch (ReadFailedException e) {
1688             LOG.error("updateLabelMapper: Failed to read data store for label {} nexthopList {}", label,
1689                     nextHopIpList);
1690         } catch (TransactionCommitFailedException e) {
1691             LOG.error("updateLabelMapper: Failed to commit to data store for label {} nexthopList {}", label,
1692                     nextHopIpList);
1693         }
1694     }
1695
1696     public synchronized void importSubnetRouteForNewVpn(String rd, String prefix, String nextHop, int label,
1697         SubnetRoute route, String parentVpnRd, WriteTransaction writeConfigTxn) {
1698
1699         RouteOrigin origin = RouteOrigin.SELF_IMPORTED;
1700         VrfEntry vrfEntry = FibHelper.getVrfEntryBuilder(prefix, label, nextHop, origin, parentVpnRd)
1701                 .addAugmentation(SubnetRoute.class, route).build();
1702         List<VrfEntry> vrfEntryList = Collections.singletonList(vrfEntry);
1703         InstanceIdentifierBuilder<VrfTables> idBuilder =
1704             InstanceIdentifier.builder(FibEntries.class).child(VrfTables.class, new VrfTablesKey(rd));
1705         InstanceIdentifier<VrfTables> vrfTableId = idBuilder.build();
1706         VrfTables vrfTableNew = new VrfTablesBuilder().setRouteDistinguisher(rd).setVrfEntry(vrfEntryList).build();
1707         if (writeConfigTxn != null) {
1708             writeConfigTxn.merge(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew, true);
1709         } else {
1710             vpnUtil.syncUpdate(LogicalDatastoreType.CONFIGURATION, vrfTableId, vrfTableNew);
1711         }
1712         LOG.info("SUBNETROUTE: importSubnetRouteForNewVpn: Created vrfEntry for rd {} prefix {} nexthop {} label {}"
1713                 + " and elantag {}", rd, prefix, nextHop, label, route.getElantag());
1714     }
1715
1716     protected void addNewAdjToVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, String primaryRd,
1717                                            Adjacency adj, BigInteger dpnId, WriteTransaction writeOperTxn,
1718                                            WriteTransaction writeConfigTxn) {
1719         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1720         String configVpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1721         try {
1722             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker
1723                     .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1724             if (optVpnInterface.isPresent()) {
1725                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1726                 String prefix = VpnUtil.getIpPrefix(adj.getIpAddress());
1727                 String vpnName = currVpnIntf.getVpnInstanceName();
1728                 VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
1729                 InstanceIdentifier<AdjacenciesOp> adjPath = identifier.augmentation(AdjacenciesOp.class);
1730                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1731                         LogicalDatastoreType.OPERATIONAL, adjPath);
1732                 boolean isL3VpnOverVxLan = VpnUtil.isL3VpnOverVxLan(vpnInstanceOpData.getL3vni());
1733                 VrfEntry.EncapType encapType = VpnUtil.getEncapType(isL3VpnOverVxLan);
1734                 long l3vni = vpnInstanceOpData.getL3vni() == null ? 0L :  vpnInstanceOpData.getL3vni();
1735                 VpnPopulator populator = L3vpnRegistry.getRegisteredPopulator(encapType);
1736                 List<Adjacency> adjacencies;
1737                 if (optAdjacencies.isPresent()) {
1738                     adjacencies = optAdjacencies.get().getAdjacency();
1739                 } else {
1740                     // This code will be hit in case of first PNF adjacency
1741                     adjacencies = new ArrayList<>();
1742                 }
1743                 long vpnId = vpnUtil.getVpnId(vpnName);
1744                 L3vpnInput input = new L3vpnInput().setNextHop(adj).setVpnName(vpnName)
1745                         .setInterfaceName(currVpnIntf.getName()).setPrimaryRd(primaryRd).setRd(primaryRd);
1746                 Adjacency operationalAdjacency = null;
1747                 //Handling dual stack neutron port primary adjacency
1748                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency && !adj.isPhysNetworkFunc()) {
1749                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to existing interface {} for vpn {}", prefix,
1750                             currVpnIntf.getName(), vpnName);
1751                     Interface interfaceState = InterfaceUtils.getInterfaceStateFromOperDS(dataBroker,
1752                             currVpnIntf.getName());
1753                     if (interfaceState != null) {
1754                         processVpnInterfaceAdjacencies(dpnId, currVpnIntf.getLportTag().intValue(), vpnName, primaryRd,
1755                                 currVpnIntf.getName(),
1756                                 vpnId, writeConfigTxn, writeOperTxn, null, interfaceState);
1757                     }
1758                 }
1759                 if (adj.getNextHopIpList() != null && !adj.getNextHopIpList().isEmpty()
1760                         && adj.getAdjacencyType() != AdjacencyType.PrimaryAdjacency) {
1761                     RouteOrigin origin = adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency ? RouteOrigin.LOCAL
1762                             : RouteOrigin.STATIC;
1763                     String nh = adj.getNextHopIpList().get(0);
1764                     String vpnPrefixKey = VpnUtil.getVpnNamePrefixKey(vpnName, prefix);
1765                     synchronized (vpnPrefixKey.intern()) {
1766                         java.util.Optional<String> rdToAllocate = vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(
1767                                 vpnId, null, prefix, vpnName, nh, dpnId);
1768                         if (rdToAllocate.isPresent()) {
1769                             input.setRd(rdToAllocate.get());
1770                             operationalAdjacency = populator.createOperationalAdjacency(input);
1771                             int label = operationalAdjacency.getLabel().intValue();
1772                             vpnManager.addExtraRoute(vpnName, adj.getIpAddress(), nh, rdToAllocate.get(),
1773                                     currVpnIntf.getVpnInstanceName(), l3vni, origin,
1774                                     currVpnIntf.getName(), operationalAdjacency, encapType, writeConfigTxn);
1775                             LOG.info("addNewAdjToVpnInterface: Added extra route ip {} nh {} rd {} vpnname {} label {}"
1776                                             + " Interface {} on dpn {}", adj.getIpAddress(), nh, rdToAllocate.get(),
1777                                     vpnName, label, currVpnIntf.getName(), dpnId);
1778                         } else {
1779                             LOG.error("addNewAdjToVpnInterface: No rds to allocate extraroute vpn {} prefix {}",
1780                                     vpnName, prefix);
1781                             return;
1782                         }
1783                         // iRT/eRT use case Will be handled in a new patchset for L3VPN Over VxLAN.
1784                         // Keeping the MPLS check for now.
1785                         if (encapType.equals(VrfEntryBase.EncapType.Mplsgre)) {
1786                             final Adjacency opAdjacency = new AdjacencyBuilder(operationalAdjacency).build();
1787                             List<VpnInstanceOpDataEntry> vpnsToImportRoute =
1788                                     vpnUtil.getVpnsImportingMyRoute(vpnName);
1789                             vpnsToImportRoute.forEach(vpn -> {
1790                                 if (vpn.getVrfId() != null) {
1791                                     vpnUtil.allocateRdForExtraRouteAndUpdateUsedRdsMap(vpn.getVpnId(), vpnId, prefix,
1792                                             vpnUtil.getVpnName(vpn.getVpnId()), nh, dpnId)
1793                                             .ifPresent(
1794                                                 rds -> vpnManager.addExtraRoute(
1795                                                         vpnUtil.getVpnName(vpn.getVpnId()),
1796                                                         adj.getIpAddress(), nh, rds,
1797                                                         currVpnIntf.getVpnInstanceName(), l3vni,
1798                                                         RouteOrigin.SELF_IMPORTED, currVpnIntf.getName(),
1799                                                         opAdjacency, encapType, writeConfigTxn));
1800                                 }
1801                             });
1802                         }
1803                     }
1804                 } else if (adj.isPhysNetworkFunc()) { // PNF adjacency.
1805                     LOG.trace("addNewAdjToVpnInterface: Adding prefix {} to interface {} for vpn {}", prefix,
1806                             currVpnIntf.getName(), vpnName);
1807
1808                     String parentVpnRd = getParentVpnRdForExternalSubnet(adj);
1809
1810                     writeOperTxn.merge(
1811                             LogicalDatastoreType.OPERATIONAL,
1812                             VpnUtil.getPrefixToInterfaceIdentifier(vpnUtil.getVpnId(adj.getSubnetId().getValue()),
1813                                     prefix), VpnUtil.getPrefixToInterface(BigInteger.ZERO, currVpnIntf.getName(),
1814                                     prefix, adj.getSubnetId(), Prefixes.PrefixCue.PhysNetFunc), true);
1815
1816                     fibManager.addOrUpdateFibEntry(adj.getSubnetId().getValue(), adj.getMacAddress(),
1817                             adj.getIpAddress(), Collections.emptyList(), null /* EncapType */, 0 /* label */,
1818                             0 /*l3vni*/, null /* gw-mac */, parentVpnRd, RouteOrigin.LOCAL, writeConfigTxn);
1819
1820                     input.setRd(adj.getVrfId());
1821                 }
1822                 if (operationalAdjacency == null) {
1823                     operationalAdjacency = populator.createOperationalAdjacency(input);
1824                 }
1825                 adjacencies.add(operationalAdjacency);
1826                 AdjacenciesOp aug = VpnUtil.getVpnInterfaceOpDataEntryAugmentation(adjacencies);
1827                 VpnInterfaceOpDataEntry newVpnIntf =
1828                         VpnUtil.getVpnInterfaceOpDataEntry(currVpnIntf.getName(), currVpnIntf.getVpnInstanceName(),
1829                                 aug, dpnId, currVpnIntf.getLportTag(),
1830                                 currVpnIntf.getGatewayMacAddress());
1831
1832                 writeOperTxn.merge(LogicalDatastoreType.OPERATIONAL, identifier, newVpnIntf, true);
1833             }
1834         } catch (ReadFailedException e) {
1835             LOG.error("addNewAdjToVpnInterface: Failed to read data store for interface {} dpn {} vpn {} rd {} ip "
1836                     + "{}", interfaceName, dpnId, configVpnName, primaryRd, adj.getIpAddress());
1837         }
1838     }
1839
1840     private String getParentVpnRdForExternalSubnet(Adjacency adj) {
1841         Subnets subnets = vpnUtil.getExternalSubnet(adj.getSubnetId());
1842         return subnets != null ? subnets.getExternalNetworkId().getValue() : null;
1843     }
1844
1845     protected void delAdjFromVpnInterface(InstanceIdentifier<VpnInterfaceOpDataEntry> identifier, Adjacency adj,
1846             BigInteger dpnId, WriteTransaction writeOperTxn, WriteTransaction writeConfigTxn) {
1847         String interfaceName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getName();
1848         String vpnName = identifier.firstKeyOf(VpnInterfaceOpDataEntry.class).getVpnInstanceName();
1849         try {
1850             Optional<VpnInterfaceOpDataEntry> optVpnInterface = SingleTransactionDataBroker.syncReadOptional(
1851                     dataBroker, LogicalDatastoreType.OPERATIONAL, identifier);
1852             if (optVpnInterface.isPresent()) {
1853                 VpnInterfaceOpDataEntry currVpnIntf = optVpnInterface.get();
1854                 InstanceIdentifier<AdjacenciesOp> path = identifier.augmentation(AdjacenciesOp.class);
1855                 Optional<AdjacenciesOp> optAdjacencies = SingleTransactionDataBroker.syncReadOptional(dataBroker,
1856                         LogicalDatastoreType.OPERATIONAL, path);
1857                 if (optAdjacencies.isPresent()) {
1858                     List<Adjacency> adjacencies = optAdjacencies.get().getAdjacency();
1859
1860                     if (!adjacencies.isEmpty()) {
1861                         LOG.trace("delAdjFromVpnInterface: Adjacencies are {}", adjacencies);
1862                         Iterator<Adjacency> adjIt = adjacencies.iterator();
1863                         while (adjIt.hasNext()) {
1864                             Adjacency adjElem = adjIt.next();
1865                             if (adjElem.getIpAddress().equals(adj.getIpAddress())) {
1866                                 String rd = adjElem.getVrfId();
1867                                 if (adj.getNextHopIpList() != null) {
1868                                     for (String nh : adj.getNextHopIpList()) {
1869                                         deleteExtraRouteFromCurrentAndImportingVpns(
1870                                                 currVpnIntf.getVpnInstanceName(), adj.getIpAddress(), nh, rd,
1871                                                 currVpnIntf.getName(), writeConfigTxn, writeOperTxn);
1872                                     }
1873                                 } else if (adj.isPhysNetworkFunc()) {
1874                                     LOG.info("delAdjFromVpnInterface: deleting PNF adjacency prefix {} subnet {}",
1875                                             adj.getIpAddress(), adj.getSubnetId());
1876                                     fibManager.removeFibEntry(adj.getSubnetId().getValue(), adj.getIpAddress(),
1877                                             writeConfigTxn);
1878                                 }
1879                                 break;
1880                             }
1881
1882                         }
1883                     }
1884                     LOG.info("delAdjFromVpnInterface: Removed adj {} on dpn {} rd {}", adj.getIpAddress(),
1885                             dpnId, adj.getVrfId());
1886                 } else {
1887                     LOG.error("delAdjFromVpnInterface: Cannnot DEL adjacency, since operational interface is "
1888                             + "unavailable dpnId {} adjIP {} rd {}", dpnId, adj.getIpAddress(), adj.getVrfId());
1889                 }
1890             }
1891         } catch (ReadFailedException e) {
1892             LOG.error("delAdjFromVpnInterface: Failed to read data store for ip {} interface {} dpn {} vpn {}",
1893                     adj.getIpAddress(), interfaceName, dpnId, vpnName);
1894         }
1895     }
1896
1897     private void deleteExtraRouteFromCurrentAndImportingVpns(String vpnName, String destination, String nextHop,
1898         String rd, String intfName, WriteTransaction writeConfigTxn, WriteTransaction writeOperTx) {
1899         vpnManager.delExtraRoute(vpnName, destination, nextHop, rd, vpnName, intfName, writeConfigTxn, writeOperTx);
1900         List<VpnInstanceOpDataEntry> vpnsToImportRoute = vpnUtil.getVpnsImportingMyRoute(vpnName);
1901         for (VpnInstanceOpDataEntry vpn : vpnsToImportRoute) {
1902             String vpnRd = vpn.getVrfId();
1903             if (vpnRd != null) {
1904                 vpnManager.delExtraRoute(vpnName, destination, nextHop, vpnRd, vpnName, intfName, writeConfigTxn,
1905                         writeOperTx);
1906             }
1907         }
1908     }
1909
1910     InstanceIdentifier<DpnVpninterfacesList> getRouterDpnId(String routerName, BigInteger dpnId) {
1911         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1912             .child(RouterDpnList.class, new RouterDpnListKey(routerName))
1913             .child(DpnVpninterfacesList.class, new DpnVpninterfacesListKey(dpnId)).build();
1914     }
1915
1916     InstanceIdentifier<RouterDpnList> getRouterId(String routerName) {
1917         return InstanceIdentifier.builder(NeutronRouterDpns.class)
1918             .child(RouterDpnList.class, new RouterDpnListKey(routerName)).build();
1919     }
1920
1921     protected void createFibEntryForRouterInterface(String primaryRd, VpnInterface vpnInterface, String interfaceName,
1922                                                     WriteTransaction writeConfigTxn, String vpnName) {
1923         if (vpnInterface == null) {
1924             return;
1925         }
1926         List<Adjacency> adjs = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(interfaceName);
1927         if (adjs == null) {
1928             LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as adjacencies for"
1929                     + " this vpn interface could not be obtained. vpn {}", interfaceName, vpnName);
1930             return;
1931         }
1932         for (Adjacency adj : adjs) {
1933             if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1934                 String primaryInterfaceIp = adj.getIpAddress();
1935                 String macAddress = adj.getMacAddress();
1936                 String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1937
1938                 long label = vpnUtil.getUniqueId(VpnConstants.VPN_IDPOOL_NAME,
1939                         VpnUtil.getNextHopLabelKey(primaryRd, prefix));
1940
1941                 RouterInterface routerInt = new RouterInterfaceBuilder().setUuid(vpnName)
1942                         .setIpAddress(primaryInterfaceIp).setMacAddress(macAddress).build();
1943                 fibManager.addFibEntryForRouterInterface(primaryRd, prefix,
1944                         routerInt, label, writeConfigTxn);
1945                 LOG.info("createFibEntryForRouterInterface: Router interface {} for vpn {} rd {} prefix {} label {}"
1946                         + " macAddress {} processed successfully;", interfaceName, vpnName, primaryRd, prefix, label,
1947                         macAddress);
1948                 return;
1949             }
1950         }
1951         LOG.error("createFibEntryForRouterInterface: VPN Interface {} of router addition failed as primary"
1952                 + " adjacency for this vpn interface could not be obtained. rd {} vpnName {}", interfaceName,
1953                 primaryRd, vpnName);
1954     }
1955
1956     protected void deleteFibEntryForRouterInterface(VpnInterface vpnInterface,
1957                                                     WriteTransaction writeConfigTxn, String vpnName) {
1958         Adjacencies adjs = vpnInterface.augmentation(Adjacencies.class);
1959         String rd = vpnUtil.getVpnRd(vpnName);
1960         if (adjs != null) {
1961             List<Adjacency> adjsList = adjs.getAdjacency();
1962             for (Adjacency adj : adjsList) {
1963                 if (adj.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
1964                     String primaryInterfaceIp = adj.getIpAddress();
1965                     String prefix = VpnUtil.getIpPrefix(primaryInterfaceIp);
1966                     fibManager.removeFibEntry(rd, prefix, writeConfigTxn);
1967                     LOG.info("deleteFibEntryForRouterInterface: FIB for router interface {} deleted for vpn {} rd {}"
1968                             + " prefix {}", vpnInterface.getName(), vpnName, rd, prefix);
1969                     return;
1970                 }
1971             }
1972         } else {
1973             LOG.error("deleteFibEntryForRouterInterface: Adjacencies for vpninterface {} is null, rd: {}",
1974                     vpnInterface.getName(), rd);
1975         }
1976     }
1977
1978     private void processSavedInterface(UnprocessedVpnInterfaceData intefaceData, String vpnName) {
1979         if (!canHandleNewVpnInterface(intefaceData.identifier, intefaceData.vpnInterface, vpnName)) {
1980             LOG.error("add: VpnInstance {} for vpnInterface {} not ready, holding on ",
1981                   vpnName, intefaceData.vpnInterface.getName());
1982             return;
1983         }
1984         final VpnInterfaceKey key = intefaceData.identifier
1985                .firstKeyOf(VpnInterface.class, VpnInterfaceKey.class);
1986         final String interfaceName = key.getName();
1987         InstanceIdentifier<VpnInterfaceOpDataEntry> vpnInterfaceOpIdentifier = VpnUtil
1988                  .getVpnInterfaceOpDataEntryIdentifier(interfaceName, vpnName);
1989         addVpnInterfaceToVpn(vpnInterfaceOpIdentifier, intefaceData.vpnInterface, null, null,
1990                   intefaceData.identifier, vpnName);
1991         return;
1992     }
1993
1994     private void addToUnprocessedVpnInterfaces(InstanceIdentifier<VpnInterface> identifier,
1995                                               VpnInterface vpnInterface, String vpnName) {
1996         ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces = unprocessedVpnInterfaces
1997                .get(vpnName);
1998         if (vpnInterfaces == null) {
1999             vpnInterfaces = new ConcurrentLinkedQueue<>();
2000         }
2001         vpnInterfaces.add(new UnprocessedVpnInterfaceData(identifier, vpnInterface));
2002         unprocessedVpnInterfaces.put(vpnName, vpnInterfaces);
2003         LOG.info("addToUnprocessedVpnInterfaces: Saved unhandled vpn interface {} in vpn instance {}",
2004                 vpnInterface.getName(), vpnName);
2005     }
2006
2007     public boolean isVpnInstanceReady(String vpnInstanceName) {
2008         String vpnRd = vpnUtil.getVpnRd(vpnInstanceName);
2009         if (vpnRd == null) {
2010             return false;
2011         }
2012         VpnInstanceOpDataEntry vpnInstanceOpDataEntry = vpnUtil.getVpnInstanceOpData(vpnRd);
2013
2014         return vpnInstanceOpDataEntry != null;
2015     }
2016
2017     public void processSavedInterfaces(String vpnInstanceName, boolean hasVpnInstanceCreatedSuccessfully) {
2018         synchronized (vpnInstanceName.intern()) {
2019             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2020                     unprocessedVpnInterfaces.get(vpnInstanceName);
2021             if (vpnInterfaces != null) {
2022                 while (!vpnInterfaces.isEmpty()) {
2023                     UnprocessedVpnInterfaceData savedInterface = vpnInterfaces.poll();
2024                     if (hasVpnInstanceCreatedSuccessfully) {
2025                         processSavedInterface(savedInterface, vpnInstanceName);
2026                         LOG.info("processSavedInterfaces: Handle saved vpn interfaces {} in vpn instance {}",
2027                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2028                     } else {
2029                         LOG.error("processSavedInterfaces: Cannot process vpn interface {} in vpn instance {}",
2030                                 savedInterface.vpnInterface.getName(), vpnInstanceName);
2031                     }
2032                 }
2033             } else {
2034                 LOG.info("processSavedInterfaces: No interfaces in queue for VPN {}", vpnInstanceName);
2035             }
2036         }
2037     }
2038
2039     private void removeInterfaceFromUnprocessedList(InstanceIdentifier<VpnInterface> identifier,
2040             VpnInterface vpnInterface) {
2041         synchronized (VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface).intern()) {
2042             ConcurrentLinkedQueue<UnprocessedVpnInterfaceData> vpnInterfaces =
2043                 unprocessedVpnInterfaces.get(VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2044             if (vpnInterfaces != null) {
2045                 if (vpnInterfaces.remove(new UnprocessedVpnInterfaceData(identifier, vpnInterface))) {
2046                     LOG.info("removeInterfaceFromUnprocessedList: Removed vpn interface {} in vpn instance {} from "
2047                             + "unprocessed list", vpnInterface.getName(),
2048                             VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2049                 }
2050             } else {
2051                 LOG.info("removeInterfaceFromUnprocessedList: No interfaces in queue for VPN {}",
2052                         VpnHelper.getFirstVpnNameFromVpnInterface(vpnInterface));
2053             }
2054         }
2055     }
2056
2057     public void vpnInstanceIsReady(String vpnInstanceName) {
2058         processSavedInterfaces(vpnInstanceName, true);
2059     }
2060
2061     public void vpnInstanceFailed(String vpnInstanceName) {
2062         processSavedInterfaces(vpnInstanceName, false);
2063     }
2064
2065     private static class UnprocessedVpnInterfaceData {
2066         InstanceIdentifier<VpnInterface> identifier;
2067         VpnInterface vpnInterface;
2068
2069         UnprocessedVpnInterfaceData(InstanceIdentifier<VpnInterface> identifier, VpnInterface vpnInterface) {
2070             this.identifier = identifier;
2071             this.vpnInterface = vpnInterface;
2072         }
2073
2074         @Override
2075         public int hashCode() {
2076             final int prime = 31;
2077             int result = 1;
2078             result = prime * result + (identifier == null ? 0 : identifier.hashCode());
2079             result = prime * result + (vpnInterface == null ? 0 : vpnInterface.hashCode());
2080             return result;
2081         }
2082
2083         @Override
2084         public boolean equals(Object obj) {
2085             if (this == obj) {
2086                 return true;
2087             }
2088             if (obj == null) {
2089                 return false;
2090             }
2091             if (getClass() != obj.getClass()) {
2092                 return false;
2093             }
2094             UnprocessedVpnInterfaceData other = (UnprocessedVpnInterfaceData) obj;
2095             if (identifier == null) {
2096                 if (other.identifier != null) {
2097                     return false;
2098                 }
2099             } else if (!identifier.equals(other.identifier)) {
2100                 return false;
2101             }
2102             if (vpnInterface == null) {
2103                 if (other.vpnInterface != null) {
2104                     return false;
2105                 }
2106             } else if (!vpnInterface.equals(other.vpnInterface)) {
2107                 return false;
2108             }
2109             return true;
2110         }
2111     }
2112
2113     public void updateVpnInterfacesForUnProcessAdjancencies(String vpnName) {
2114         String primaryRd = vpnUtil.getVpnRd(vpnName);
2115         VpnInstanceOpDataEntry vpnInstanceOpData = vpnUtil.getVpnInstanceOpData(primaryRd);
2116         if (vpnInstanceOpData == null) {
2117             return;
2118         }
2119         List<VpnToDpnList> vpnToDpnLists = vpnInstanceOpData.getVpnToDpnList();
2120         if (vpnToDpnLists == null || vpnToDpnLists.isEmpty()) {
2121             return;
2122         }
2123         LOG.debug("Update the VpnInterfaces for Unprocessed Adjancencies for vpnName:{}", vpnName);
2124         vpnToDpnLists.forEach(vpnToDpnList -> {
2125             if (vpnToDpnList.getVpnInterfaces() == null) {
2126                 return;
2127             }
2128             vpnToDpnList.getVpnInterfaces().forEach(vpnInterface -> {
2129                 try {
2130                     InstanceIdentifier<VpnInterfaceOpDataEntry> existingVpnInterfaceId =
2131                             VpnUtil.getVpnInterfaceOpDataEntryIdentifier(vpnInterface.getInterfaceName(), vpnName);
2132                     Optional<VpnInterfaceOpDataEntry> vpnInterfaceOptional = SingleTransactionDataBroker
2133                             .syncReadOptional(dataBroker, LogicalDatastoreType.OPERATIONAL, existingVpnInterfaceId);
2134                     if (!vpnInterfaceOptional.isPresent()) {
2135                         return;
2136                     }
2137                     List<Adjacency> configVpnAdjacencies = vpnUtil.getAdjacenciesForVpnInterfaceFromConfig(
2138                             vpnInterface.getInterfaceName());
2139                     if (configVpnAdjacencies == null) {
2140                         LOG.debug("There is no adjacency available for vpnInterface:{}", vpnInterface);
2141                         return;
2142                     }
2143                     List<Adjacency> operationVpnAdjacencies = vpnInterfaceOptional.get()
2144                             .augmentation(AdjacenciesOp.class).getAdjacency();
2145                     // Due to insufficient rds,  some of the extra route wont get processed when it is added.
2146                     // The unprocessed adjacencies will be present in config vpn interface DS but will be missing
2147                     // in operational DS. These unprocessed adjacencies will be handled below.
2148                     // To obtain unprocessed adjacencies, filtering is done by which the missing adjacencies in
2149                     // operational DS are retrieved which is used to call addNewAdjToVpnInterface method.
2150                     configVpnAdjacencies.stream()
2151                         .filter(adjacency -> operationVpnAdjacencies.stream()
2152                                 .noneMatch(operationalAdjacency ->
2153                                         operationalAdjacency.getIpAddress().equals(adjacency.getIpAddress())))
2154                         .forEach(adjacency -> {
2155                             LOG.debug("Processing the vpnInterface{} for the Ajacency:{}", vpnInterface, adjacency);
2156                             jobCoordinator.enqueueJob("VPNINTERFACE-" + vpnInterface.getInterfaceName(),
2157                                 () -> {
2158                                     // TODO Deal with sequencing — the config tx must only submitted
2159                                     // if the oper tx goes in
2160                                     if (vpnUtil.isAdjacencyEligibleToVpn(adjacency, vpnName)) {
2161                                         List<ListenableFuture<Void>> futures = new ArrayList<>();
2162                                         futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(operTx ->
2163                                             futures.add(txRunner.callWithNewWriteOnlyTransactionAndSubmit(
2164                                                 confTx -> addNewAdjToVpnInterface(existingVpnInterfaceId,
2165                                                     primaryRd, adjacency, vpnInterfaceOptional.get()
2166                                                         .getDpnId(), confTx, operTx)))));
2167                                         return futures;
2168                                     } else {
2169                                         return Collections.emptyList();
2170                                     }
2171                                 });
2172                         });
2173                 } catch (ReadFailedException e) {
2174                     LOG.error("updateVpnInterfacesForUnProcessAdjancencies: Failed to read data store for vpn {} rd {}",
2175                             vpnName, primaryRd);
2176                 }
2177             });
2178         });
2179     }
2180
2181     private class PostVpnInterfaceWorker implements FutureCallback<Void> {
2182         private final String interfaceName;
2183         private final boolean add;
2184         private final String txnDestination;
2185
2186         PostVpnInterfaceWorker(String interfaceName, boolean add, String transactionDest) {
2187             this.interfaceName = interfaceName;
2188             this.add = add;
2189             this.txnDestination = transactionDest;
2190         }
2191
2192         @Override
2193         public void onSuccess(Void voidObj) {
2194             if (add) {
2195                 LOG.debug("VpnInterfaceManager: VrfEntries for {} stored into destination {} successfully",
2196                         interfaceName, txnDestination);
2197             } else {
2198                 LOG.debug("VpnInterfaceManager: VrfEntries for {} removed successfully", interfaceName);
2199             }
2200         }
2201
2202         @Override
2203         public void onFailure(Throwable throwable) {
2204             if (add) {
2205                 LOG.error("VpnInterfaceManager: VrfEntries for {} failed to store into destination {}",
2206                         interfaceName, txnDestination, throwable);
2207             } else {
2208                 LOG.error("VpnInterfaceManager: VrfEntries for {} removal failed", interfaceName, throwable);
2209                 vpnUtil.unsetScheduledToRemoveForVpnInterface(interfaceName);
2210             }
2211         }
2212     }
2213 }